class_docstr: Tokenize text on subtokens, encode subtokens with their indices, create tokens and segment masks.
  
  Args:
      vocab_file: path to vocabulary
      do_lower_case: set True if lowercasing is needed
      max_seq_length: max sequence length in subtokens, including [SEP] and [CLS] tokens
  
  Attributes:
      max_seq_length: max sequence length in subtokens, including [SEP] and [CLS] tokens
      tokenizer: instance of Bert FullTokenizer
class_docstr_tok: ["Tokenize","text","on","subtokens",",","encode","subtokens","with","their","indices",",","create","tokens","and","segment","masks",".","Args",":","vocab_file",":","path","to","vocabulary","do_lower_case",":","set","True","if","lowercasing","is","needed","max_seq_length",":","max","sequence","length","in","subtokens",",","including","[","SEP","]","and","[","CLS","]","tokens","Attributes",":","max_seq_length",":","max","sequence","length","in","subtokens",",","including","[","SEP","]","and","[","CLS","]","tokens","tokenizer",":","instance","of","Bert","FullTokenizer"]
code: def __init__(
      self,
      vocab_file: str,
      do_lower_case: bool = True,
      max_seq_length: int = 512,
      **kwargs,
  ) -> None:
      self.max_seq_length = max_seq_length
      if Path(vocab_file).is_file():
          vocab_file = str(expand_path(vocab_file))
          self.tokenizer = AutoTokenizer(
              vocab_file=vocab_file,
              do_lower_case=do_lower_case,
              **kwargs,
          )
      else:
          self.tokenizer = AutoTokenizer.from_pretrained(
              vocab_file, do_lower_case=do_lower_case, **kwargs
          )
  
imports: import math
  import random
  import re
  from collections import defaultdict
  from dataclasses import dataclass
  from logging import getLogger
  from pathlib import Path
  from typing import Tuple, List, Optional, Union, Dict, Set, Any
  import nltk
  import numpy
  import torch
  from transformers import AutoTokenizer
  from transformers.data.processors.utils import InputFeatures
  from deeppavlov.core.commands.utils import expand_path
  from deeppavlov.core.common.registry import register
  from deeppavlov.core.data.utils import zero_pad
  from deeppavlov.core.models.component import Component
  from deeppavlov.models.preprocessors.mask import Mask
label_desc: Use image node_id 1 to create a new TorchTransformersMultiplechoicePreprocessor object from inherited base classes: Component with example: obj = TorchTransformersMultiplechoicePreprocessor(vocab_file, do_lower_case, max_seq_length)