docstr: Process raw inputs into model input tensors, it supports two input
  formats:
      1) multiple rows of texts (single sentence or a pair)
      2) multiple rows of pre-processed tokens (single sentence or a pair)
  
  This function should handle the logic of calling numberize() and also
  padding the numberized result.
docstr_tok: ["Process","raw","inputs","into","model","input","tensors",",","it","supports","two","input","formats",":","1",")","multiple","rows","of","texts","(","single","sentence","or","a","pair",")","2",")","multiple","rows","of","pre-processed","tokens","(","single","sentence","or","a","pair",")","This","function","should","handle","the","logic","of","calling","numberize","(",")","and","also","padding","the","numberized","result","."]
code: def tensorize(
      self,
      texts: Optional[List[List[str]]] = None,
      tokens: Optional[List[List[List[str]]]] = None,
  ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
      """
      Process raw inputs into model input tensors, it supports two input
      formats:
          1) multiple rows of texts (single sentence or a pair)
          2) multiple rows of pre-processed tokens (single sentence or a pair)
  
      This function should handle the logic of calling numberize() and also
      padding the numberized result.
      """
      tokens_2d: List[List[int]] = []
      segment_labels_2d: List[List[int]] = []
      seq_len_2d: List[int] = []
      positions_2d: List[List[int]] = []
  
      for idx in range(self.batch_size(texts, tokens)):
          numberized: Tuple[
              List[int], List[int], int, List[int]
          ] = self.numberize(
              self.get_texts_by_index(texts, idx),
              self.get_tokens_by_index(tokens, idx),
          )
          tokens_2d.append(numberized[0])
          segment_labels_2d.append(numberized[1])
          seq_len_2d.append(numberized[2])
          positions_2d.append(numberized[3])
  
      tokens, pad_mask = pad_2d_mask(
          tokens_2d, pad_value=self.vocab.pad_idx
      )
      segment_labels = torch.tensor(
          pad_2d(
              segment_labels_2d,
              seq_lens=seq_len_2d,
              pad_idx=self.vocab.pad_idx,
          ),
          dtype=torch.long,
      )
      positions = torch.tensor(
          pad_2d(
              positions_2d,
              seq_lens=seq_len_2d,
              pad_idx=self.vocab.pad_idx,
          ),
          dtype=torch.long,
      )
  
      if self.device == "":
          return tokens, pad_mask, segment_labels, positions
      else:
          return (
              tokens.to(self.device),
              pad_mask.to(self.device),
              segment_labels.to(self.device),
              positions.to(self.device),
          )
  
imports: import random
  import string
  import unittest
  from typing import List, Optional, Tuple
  import torch
  from pytext.data.tensorizers import TensorizerScriptImpl
  from pytext.torchscript.module import PyTextEmbeddingModule, ScriptPyTextEmbeddingModule
  from pytext.torchscript.tensorizer.tensorizer import ScriptTensorizer, VocabLookup
  from pytext.torchscript.utils import pad_2d_mask, ScriptBatchInput
  from pytext.torchscript.vocab import ScriptVocabulary
  from pytorch.text.fb.utils.transform_utils import pad_2d
label_desc: Use image node_id 6 for calling the MyTensorizer obj's underlying member method code with example usage: obj.tensorize(texts, tokens) and returns: tokens,  pad_mask,  segment_labels,  positions, tokens,  pad_mask,  segment_labels,  positions