class_docstr: Tokenizer that takes a list of strings and converts to a list of Tokens.
  Useful in cases where tokenizer is run before-hand
class_docstr_tok: ["Tokenizer","that","takes","a","list","of","strings","and","converts","to","a","list","of","Tokens",".","Useful","in","cases","where","tokenizer","is","run","before-hand"]
code: def tokenize(self, tokens: Union[List[str], str]) -> List[Token]:
      if isinstance(tokens, str):
          # parse tokens if in json format
          try:
              tokens = json.loads(tokens)
              # address edge case "true"/"false" converted to Boolean True/False
              if isinstance(tokens, bool):
                  tokens = [str(tokens).lower()]
          # else make it a List
          except Exception:
              tokens = [tokens]
      tokens = [
          Token(token_text, -1, -1)
          for token_text in tokens
          if token_text
      ]
      return tokens
  
imports: import copy
  import json
  import re
  from collections import OrderedDict
  from typing import List, NamedTuple, Optional, Union
  import torch
  from fairseq.data.encoders.gpt2_bpe import get_encoder
  from fairseq.data.encoders.gpt2_bpe_utils import Encoder
  from pytext.config import ConfigBase
  from pytext.config.component import Component, ComponentType, create_component
  from pytext.data.pickleable_gpt2bpe_encoder import PickleableGPT2BPEEncoder
  from pytext.torchscript.tokenizer import ScriptDoNothingTokenizer, ScriptWordTokenizer
  from pytext.utils.file_io import PathManager
  from pytext.utils.usage import log_class_usage
  from transformers.tokenization_bert import BasicTokenizer, WordpieceTokenizer
label_desc: Use image node_id 3 for calling the DoNothingTokenizer obj's underlying member method code with example usage: obj.tokenize(tokens) and returns: tokens