Skip to content

Commit 021405e

Browse files
committed
Fix MLTransform vocab CI dependency and formatting issues
1 parent 66cfd4c commit 021405e

2 files changed

Lines changed: 13 additions & 10 deletions

File tree

sdks/python/apache_beam/examples/ml_transform/mltransform_generate_vocab.py

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -258,13 +258,13 @@ def run(argv=None, test_pipeline=None):
258258
_ = (
259259
token_lists
260260
| 'MLTransformInput' >> beam.Map(lambda tokens: {'tokens': tokens})
261-
| 'ApplyMLTransform' >> MLTransform(
262-
write_artifact_location=artifact_location).with_transform(
263-
ComputeAndApplyVocabulary(
264-
columns=['tokens'],
265-
top_k=known_args.vocab_size,
266-
frequency_threshold=known_args.min_frequency,
267-
vocab_filename='vocab'))
261+
| 'ApplyMLTransform' >>
262+
MLTransform(write_artifact_location=artifact_location).with_transform(
263+
ComputeAndApplyVocabulary(
264+
columns=['tokens'],
265+
top_k=known_args.vocab_size,
266+
frequency_threshold=known_args.min_frequency,
267+
vocab_filename='vocab'))
268268
| 'ExtractTransformedTokens' >> beam.Map(lambda row: row.tokens)
269269
| 'FlattenTokens' >> beam.FlatMap(list)
270270
| 'DropEmptyTokens' >> beam.Filter(bool))
@@ -278,8 +278,8 @@ def run(argv=None, test_pipeline=None):
278278
vocab_filename='vocab',
279279
column_name='tokens'))
280280
output_tokens = [known_args.oov_token]
281-
output_tokens.extend(token for token in vocab_tokens
282-
if token != known_args.oov_token)
281+
output_tokens.extend(
282+
token for token in vocab_tokens if token != known_args.oov_token)
283283
if len(output_tokens) == 1:
284284
logging.warning(
285285
'No tokens remained after filtering; writing only reserved token %r.',

sdks/python/apache_beam/examples/ml_transform/mltransform_generate_vocab_test.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,10 @@
2020
import tempfile
2121
import unittest
2222

23-
from apache_beam.examples.ml_transform import mltransform_generate_vocab
23+
try:
24+
from apache_beam.examples.ml_transform import mltransform_generate_vocab
25+
except ImportError: # pylint: disable=bare-except
26+
raise unittest.SkipTest('tensorflow_transform is not installed.')
2427

2528

2629
class MLTransformGenerateVocabUnitTest(unittest.TestCase):

0 commit comments

Comments
 (0)