class_docstr: This tokenizer allows us to tokenize using both spm and word token boundaries. It works as below:
  1) Splits the sentence into candidate tokens delimited by space
  2) Checks if the given token is part of the word based ontology vocab. If yes, it tokenizes it as a
     full word token
  3) If no, it breaks it down using spm voabulary.
class_docstr_tok: ["This","tokenizer","allows","us","to","tokenize","using","both","spm","and","word","token","boundaries",".","It","works","as","below",":","1",")","Splits","the","sentence","into","candidate","tokens","delimited","by","space","2",")","Checks","if","the","given","token","is","part","of","the","word","based","ontology","vocab",".","If","yes",",","it","tokenizes","it","as","a","full","word","token","3",")","If","no",",","it","breaks","it","down","using","spm","voabulary","."]
code: def tokenize(self, input_str: str) -> List[Token]:
      if (
          hasattr(self, "max_input_text_length")
          and self.max_input_text_length is not None
      ):
          input_str = input_str[: self.max_input_text_length]
      input_str = input_str.lower() if self.lowercase else input_str
  
      # Split the tokens using space
      first_stage_tokens = input_str.split()
  
      final_tokens = []
      end = 0
      for token in first_stage_tokens:
          if token in self.ontology_vocab:
              # Check if part of ontology vocab tokens. If yes, consider it as
              # single word token
              start = input_str.find(token, end)
              end = start + len(token)
              final_tokens.append(Token(token, start, end))
  
          else:
              # Break the token down using spm vocab
              pieces = self.processor.EncodeAsPieces(" " + token)
              for piece in pieces:
                  original_piece = piece.lstrip("\u2581")
                  start = input_str.find(original_piece, end)
                  end = start + len(original_piece)
                  final_tokens.append(Token(piece, start, end))
  
      return final_tokens
  
imports: import copy
  import json
  import re
  from collections import OrderedDict
  from typing import List, NamedTuple, Optional, Union
  import torch
  from fairseq.data.encoders.gpt2_bpe import get_encoder
  from fairseq.data.encoders.gpt2_bpe_utils import Encoder
  from pytext.config import ConfigBase
  from pytext.config.component import Component, ComponentType, create_component
  from pytext.data.pickleable_gpt2bpe_encoder import PickleableGPT2BPEEncoder
  from pytext.torchscript.tokenizer import ScriptDoNothingTokenizer, ScriptWordTokenizer
  from pytext.utils.file_io import PathManager
  from pytext.utils.usage import log_class_usage
  from transformers.tokenization_bert import BasicTokenizer, WordpieceTokenizer
label_desc: Use image node_id 4 for calling the SPEandWordTokenizer obj's underlying member method code with example usage: obj.tokenize(input_str) and returns: final_tokens