class_docstr: A field that conducts tokenization and numericalization over each token rather the sequence.
  
  This is customized for models requiring character/subword-level inputs, e.g., CharLSTM and BERT.
  
  Args:
      fix_len (int):
          A fixed length that all subword pieces will be padded to.
          This is used for truncating the subword pieces that exceed the length.
          To save the memory, the final length will be the smaller value
          between the max length of subword pieces in a batch and `fix_len`.
  
  Examples:
      >>> from transformers import AutoTokenizer
      >>> tokenizer = AutoTokenizer.from_pretrained('bert-base-cased')
      >>> field = SubwordField('bert',
                               pad=tokenizer.pad_token,
                               unk=tokenizer.unk_token,
                               bos=tokenizer.cls_token,
                               eos=tokenizer.sep_token,
                               fix_len=20,
                               tokenize=tokenizer.tokenize)
      >>> field.vocab = tokenizer.get_vocab()  # no need to re-build the vocab
      >>> field.transform([['This', 'field', 'performs', 'token-level', 'tokenization']])[0]
      tensor([[  101,     0,     0],
              [ 1188,     0,     0],
              [ 1768,     0,     0],
              [10383,     0,     0],
              [22559,   118,  1634],
              [22559,  2734,     0],
              [  102,     0,     0]])
class_docstr_tok: ["A","field","that","conducts","tokenization","and","numericalization","over","each","token","rather","the","sequence",".","This","is","customized","for","models","requiring","character\/subword-level","inputs",",","e.g.",",","CharLSTM","and","BERT",".","Args",":","fix_len","(","int",")",":","A","fixed","length","that","all","subword","pieces","will","be","padded","to",".","This","is","used","for","truncating","the","subword","pieces","that","exceed","the","length",".","To","save","the","memory",",","the","final","length","will","be","the","smaller","value","between","the","max","length","of","subword","pieces","in","a","batch","and","`","fix_len","`",".","Examples",":",">",">",">","from","transformers","import","AutoTokenizer",">",">",">","tokenizer","=","AutoTokenizer.from_pretrained","(","'bert-base-cased","'",")",">",">",">","field","=","SubwordField","(","'bert","'",",","pad=tokenizer.pad_token",",","unk=tokenizer.unk_token",",","bos=tokenizer.cls_token",",","eos=tokenizer.sep_token",",","fix_len=20",",","tokenize=tokenizer.tokenize",")",">",">",">","field.vocab","=","tokenizer.get_vocab","(",")","#","no","need","to","re-build","the","vocab",">",">",">","field.transform","(","[","[","'This","'",",","'field","'",",","'performs","'",",","'token-level","'",",","'tokenization","'","]","]",")","[","0","]","tensor","(","[","[","101",",","0",",","0","]",",","[","1188",",","0",",","0","]",",","[","1768",",","0",",","0","]",",","[","10383",",","0",",","0","]",",","[","22559",",","118",",","1634","]",",","[","22559",",","2734",",","0","]",",","[","102",",","0",",","0","]","]",")"]
code: def transform(self, sequences):
      sequences_ = [self.vi_preprocess(seq) for seq in sequences]
      sequences_ = [self.vi_preprocess(seq) for seq in sequences]
      sequences = sequences_
      if self.fix_len <= 0:
          self.fix_len = max(
              len(token) for seq in sequences for token in seq
          )
  
      def get_vocab(i):
          if i in self.vocab:
              return self.vocab[i]
          else:
              return self.vocab["<unk>"]
  
      if self.use_vocab:
          sequences = [
              [
                  [get_vocab(i) for i in token]
                  if token
                  else [self.unk_index]
                  for token in seq
              ]
              for seq in sequences
          ]
      if self.bos:
          sequences = [[[self.bos_index]] + seq for seq in sequences]
      if self.eos:
          sequences = [seq + [[self.eos_index]] for seq in sequences]
      lens = [
          min(self.fix_len, max(len(ids) for ids in seq))
          for seq in sequences
      ]
      sequences = [
          pad([torch.tensor(ids[:i]) for ids in seq], self.pad_index, i)
          for i, seq in zip(lens, sequences)
      ]
  
      return sequences
  
imports: from collections import Counter
  import torch
  from underthesea.utils.sp_fn import pad
  from underthesea.utils.sp_vocab import Vocab
label_desc: Use image node_id 3 for calling the SubwordField obj's underlying member method code with example usage: obj.transform(sequences) and returns: sequences, self, self