docstr: Process a single line of raw inputs into numberized result, it supports
  two input formats:
      1) a single line of texts (single sentence or a pair)
      2) a single line of pre-processed tokens (single sentence or a pair)
  
  This function should handle the logic of calling tokenize(), add special
  tokens and vocab lookup.
docstr_tok: ["Process","a","single","line","of","raw","inputs","into","numberized","result",",","it","supports","two","input","formats",":","1",")","a","single","line","of","texts","(","single","sentence","or","a","pair",")","2",")","a","single","line","of","pre-processed","tokens","(","single","sentence","or","a","pair",")","This","function","should","handle","the","logic","of","calling","tokenize","(",")",",","add","special","tokens","and","vocab","lookup","."]
code: def numberize(
      self,
      text_row: Optional[List[str]],
      token_row: Optional[List[List[str]]],
  ) -> Tuple[List[int], List[int], int, List[int]]:
      """
      Process a single line of raw inputs into numberized result, it supports
      two input formats:
          1) a single line of texts (single sentence or a pair)
          2) a single line of pre-processed tokens (single sentence or a pair)
  
      This function should handle the logic of calling tokenize(), add special
      tokens and vocab lookup.
      """
      token_ids: List[int] = []
      segment_labels: List[int] = []
      seq_len: int = 0
      positions: List[int] = []
      per_sentence_tokens: List[
          List[Tuple[str, int, int]]
      ] = self.tokenize(text_row, token_row)
  
      for idx, per_sentence_token in enumerate(per_sentence_tokens):
          lookup_ids: List[int] = self._lookup_tokens(
              per_sentence_token
          )
          lookup_ids = self._wrap_numberized_tokens(lookup_ids, idx)
  
          token_ids.extend(lookup_ids)
          segment_labels.extend([idx] * len(lookup_ids))
      seq_len = len(token_ids)
      positions = list(range(seq_len))
  
      return token_ids, segment_labels, seq_len, positions
  
imports: import random
  import string
  import unittest
  from typing import List, Optional, Tuple
  import torch
  from pytext.data.tensorizers import TensorizerScriptImpl
  from pytext.torchscript.module import PyTextEmbeddingModule, ScriptPyTextEmbeddingModule
  from pytext.torchscript.tensorizer.tensorizer import ScriptTensorizer, VocabLookup
  from pytext.torchscript.utils import pad_2d_mask, ScriptBatchInput
  from pytext.torchscript.vocab import ScriptVocabulary
  from pytorch.text.fb.utils.transform_utils import pad_2d
label_desc: Use image node_id 5 for calling the MyTensorizer obj's underlying member method code with example usage: obj.numberize(text_row, token_row) and returns: token_ids,  segment_labels,  seq_len,  positions