class_docstr: Class for tokenization of text into subtokens, encoding of subtokens with indices and obtaining positions of
  special [ENT]-tokens
  Args:
      vocab_file: path to vocabulary
      do_lower_case: set True if lowercasing is needed
      max_seq_length: max sequence length in subtokens, including [SEP] and [CLS] tokens
      special_tokens: list of special tokens
      special_token_id: id of special token
      return_special_tokens_pos: whether to return positions of found special tokens
class_docstr_tok: ["Class","for","tokenization","of","text","into","subtokens",",","encoding","of","subtokens","with","indices","and","obtaining","positions","of","special","[","ENT","]","-tokens","Args",":","vocab_file",":","path","to","vocabulary","do_lower_case",":","set","True","if","lowercasing","is","needed","max_seq_length",":","max","sequence","length","in","subtokens",",","including","[","SEP","]","and","[","CLS","]","tokens","special_tokens",":","list","of","special","tokens","special_token_id",":","id","of","special","token","return_special_tokens_pos",":","whether","to","return","positions","of","found","special","tokens"]
docstr: Tokenize and find special tokens positions.
  Args:
      texts_a: list of texts,
  Returns:
      batch of :class:`transformers.data.processors.utils.InputFeatures` with subtokens, subtoken ids,                 subtoken mask, segment mask, or tuple of batch of InputFeatures and Batch of subtokens
      batch of indices of special token ids in input ids sequence
docstr_tok: ["Tokenize","and","find","special","tokens","positions",".","Args",":","texts_a",":","list","of","texts",",","Returns",":","batch","of",":","class",":","`","transformers.data.processors.utils.InputFeatures","`","with","subtokens",",","subtoken","ids",",","subtoken","mask",",","segment","mask",",","or","tuple","of","batch","of","InputFeatures","and","Batch","of","subtokens","batch","of","indices","of","special","token","ids","in","input","ids","sequence"]
code: def __call__(self, texts_a: List[str]) -> Tuple[Any, List[int]]:
      """Tokenize and find special tokens positions.
          Args:
              texts_a: list of texts,
          Returns:
              batch of :class:`transformers.data.processors.utils.InputFeatures` with subtokens, subtoken ids, \
                  subtoken mask, segment mask, or tuple of batch of InputFeatures and Batch of subtokens
              batch of indices of special token ids in input ids sequence
          """
      # in case of iterator's strange behaviour
      if isinstance(texts_a, tuple):
          texts_a = list(texts_a)
      if self.do_lower_case:
          texts_a = [text.lower() for text in texts_a]
      lengths = []
      input_ids_batch = []
      for text_a in texts_a:
          encoding = self.tokenizer.encode_plus(
              text_a,
              add_special_tokens=True,
              pad_to_max_length=True,
              return_attention_mask=True,
          )
          input_ids = encoding["input_ids"]
          input_ids_batch.append(input_ids)
          lengths.append(len(input_ids))
  
      max_length = min(max(lengths), self.max_seq_length)
      input_features = self.tokenizer(
          text=texts_a,
          add_special_tokens=True,
          max_length=max_length,
          padding="max_length",
          return_attention_mask=True,
          truncation=True,
          return_tensors="pt",
      )
      special_tokens_pos = []
      for input_ids_list in input_ids_batch:
          found_n = -1
          for n, input_id in enumerate(input_ids_list):
              if input_id == self.special_token_id:
                  found_n = n
                  break
          if found_n == -1:
              found_n = 0
          special_tokens_pos.append(found_n)
  
      if self.return_special_tokens_pos:
          return input_features, special_tokens_pos
      else:
          return input_features
  
imports: import math
  import random
  import re
  from collections import defaultdict
  from dataclasses import dataclass
  from logging import getLogger
  from pathlib import Path
  from typing import Tuple, List, Optional, Union, Dict, Set, Any
  import nltk
  import numpy
  import torch
  from transformers import AutoTokenizer
  from transformers.data.processors.utils import InputFeatures
  from deeppavlov.core.commands.utils import expand_path
  from deeppavlov.core.common.registry import register
  from deeppavlov.core.data.utils import zero_pad
  from deeppavlov.core.models.component import Component
  from deeppavlov.models.preprocessors.mask import Mask
label_desc: Use image node_id 2 for calling the TorchTransformersEntityRankerPreprocessor obj's underlying member method code with example usage: obj.__call__(texts_a) and returns: input_features,  special_tokens_pos, input_features