docstr: Process a single line of raw inputs into tokens, it supports
  two input formats:
      1) a single line of texts (single sentence or a pair)
      2) a single line of pre-processed tokens (single sentence or a pair)
docstr_tok: ["Process","a","single","line","of","raw","inputs","into","tokens",",","it","supports","two","input","formats",":","1",")","a","single","line","of","texts","(","single","sentence","or","a","pair",")","2",")","a","single","line","of","pre-processed","tokens","(","single","sentence","or","a","pair",")"]
code: def tokenize(
      self,
      text_row: Optional[List[str]],
      token_row: Optional[List[List[str]]],
  ) -> List[List[Tuple[str, int, int]]]:
      """
      Process a single line of raw inputs into tokens, it supports
      two input formats:
          1) a single line of texts (single sentence or a pair)
          2) a single line of pre-processed tokens (single sentence or a pair)
      """
      per_sentence_tokens: List[List[Tuple[str, int, int]]] = []
  
      if text_row is not None:
          for text in text_row:
              per_sentence_tokens.append(self.tokenizer.tokenize(text))
      elif token_row is not None:
          for sentence_raw_tokens in token_row:
              sentence_tokens: List[Tuple[str, int, int]] = []
              for raw_token in sentence_raw_tokens:
                  sentence_tokens.extend(
                      self.tokenizer.tokenize(raw_token)
                  )
              per_sentence_tokens.append(sentence_tokens)
  
      return per_sentence_tokens
  
imports: import random
  import string
  import unittest
  from typing import List, Optional, Tuple
  import torch
  from pytext.data.tensorizers import TensorizerScriptImpl
  from pytext.torchscript.module import PyTextEmbeddingModule, ScriptPyTextEmbeddingModule
  from pytext.torchscript.tensorizer.tensorizer import ScriptTensorizer, VocabLookup
  from pytext.torchscript.utils import pad_2d_mask, ScriptBatchInput
  from pytext.torchscript.vocab import ScriptVocabulary
  from pytorch.text.fb.utils.transform_utils import pad_2d
label_desc: Use image node_id 4 for calling the MyTensorizer obj's underlying member method code with example usage: obj.tokenize(text_row, token_row) and returns: per_sentence_tokens