code: def testVocabularyAnalyzerWithTokenization(self):
      def preprocessing_fn(inputs):
          return {
              "index": tft.compute_and_apply_vocabulary(
                  tf.compat.v1.strings.split(inputs["a"]),
                  file_format=self._VocabFormat(),
                  vocab_filename="my_vocab",
              )
          }
  
      input_data = [
          {"a": "hello hello world"},
          {"a": "hello goodbye world"},
      ]
      input_metadata = tft.DatasetMetadata.from_feature_spec(
          {"a": tf.io.FixedLenFeature([], tf.string)}
      )
      expected_data = [{"index": [0, 0, 1]}, {"index": [0, 2, 1]}]
  
      expected_metadata = tft.DatasetMetadata.from_feature_spec(
          {
              "index": tf.io.VarLenFeature(tf.int64),
          },
          {
              "index": schema_pb2.IntDomain(
                  min=-1, max=2, is_categorical=True
              ),
          },
      )
      expected_vocabulary = {
          "my_vocab": [b"hello", b"world", b"goodbye"]
      }
      self.assertAnalyzeAndTransformResults(
          input_data,
          input_metadata,
          preprocessing_fn,
          expected_data,
          expected_metadata=expected_metadata,
          expected_vocab_file_contents=expected_vocabulary,
      )
  
imports: import os
  import apache_beam
  import tensorflow
  import tensorflow_transform
  from tensorflow_transform.beam import analyzer_impls
  from tensorflow_transform.beam import impl
  from tensorflow_transform.beam.tft_beam_io import transform_fn_io
  from tensorflow_transform.beam import tft_unit
  from tensorflow_metadata.proto.v0 import schema_pb2
label_desc: Use image node_id 11 for calling the VocabularyIntegrationTest obj's underlying member method code with example usage: obj.testVocabularyAnalyzerWithTokenization() and returns: dict