code: def test_tokenization(self):
      encoder = text_encoder.TokenTextEncoder(
          vocab_list=["hi", "bye", ZH_HELLO]
      )
      text = "hi<<>><<>foo!^* bar && bye (%s hi)" % ZH_HELLO
      self.assertEqual(
          ["hi", "foo", "bar", "bye", ZH_HELLO.strip(), "hi"],
          text_encoder.Tokenizer().tokenize(text),
      )
      self.assertEqual(
          [i + 1 for i in [0, 3, 3, 1, 2, 0]], encoder.encode(text)
      )
  
imports: from __future__ import unicode_literals
  import os
  from absl.testing import parameterized
  import numpy
  from tensorflow_datasets import testing
  from tensorflow_datasets.core.deprecated.text import text_encoder
label_desc: Use image node_id 4 for calling the TokenTextEncoderTest obj's underlying member method code with example usage: obj.test_tokenization() without return types