class_docstr: A simple regex-splitting tokenizer.
class_docstr_tok: ["A","simple","regex-splitting","tokenizer","."]
code: def tokenize(self, input: str) -> List[Token]:
      tokens = []
      start = 0
      tokenize_input = input.lower() if self.lowercase else input
      for match in re.finditer(self.split_regex, tokenize_input):
          split_start, split_end = match.span()
          tokens.append(
              Token(
                  tokenize_input[start:split_start], start, split_start
              )
          )
          start = split_end
      tokens.append(
          Token(tokenize_input[start : len(input)], start, len(input))
      )
      if self.use_byte_offsets:
          return [
              self._convert_token(input, token)
              for token in tokens
              if token.value
          ]
      else:
          return [token for token in tokens if token.value]
  
imports: import copy
  import json
  import re
  from collections import OrderedDict
  from typing import List, NamedTuple, Optional, Union
  import torch
  from fairseq.data.encoders.gpt2_bpe import get_encoder
  from fairseq.data.encoders.gpt2_bpe_utils import Encoder
  from pytext.config import ConfigBase
  from pytext.config.component import Component, ComponentType, create_component
  from pytext.data.pickleable_gpt2bpe_encoder import PickleableGPT2BPEEncoder
  from pytext.torchscript.tokenizer import ScriptDoNothingTokenizer, ScriptWordTokenizer
  from pytext.utils.file_io import PathManager
  from pytext.utils.usage import log_class_usage
  from transformers.tokenization_bert import BasicTokenizer, WordpieceTokenizer
label_desc: Use image node_id 3 for calling the Tokenizer obj's underlying member method code with example usage: obj.tokenize(input) and returns: unknown, unknown