class_docstr: Takes tokens and splits them into bert subtokens, encodes subtokens with their indices.
  Creates a mask of subtokens (one for the first subtoken, zero for the others).
  
  If tags are provided, calculates tags for subtokens.
  
  Args:
      vocab_file: path to vocabulary
      do_lower_case: set True if lowercasing is needed
      max_seq_length: max sequence length in subtokens, including [SEP] and [CLS] tokens
      max_subword_length: replace token to <unk> if it's length is larger than this
          (defaults to None, which is equal to +infinity)
      token_masking_prob: probability of masking token while training
      provide_subword_tags: output tags for subwords or for words
      subword_mask_mode: subword to select inside word tokens, can be "first" or "last"
          (default="first")
      return_features: if True, returns answer in features format
  
  Attributes:
      max_seq_length: max sequence length in subtokens, including [SEP] and [CLS] tokens
      max_subword_length: rmax lenght of a bert subtoken
      tokenizer: instance of Bert FullTokenizer
class_docstr_tok: ["Takes","tokens","and","splits","them","into","bert","subtokens",",","encodes","subtokens","with","their","indices",".","Creates","a","mask","of","subtokens","(","one","for","the","first","subtoken",",","zero","for","the","others",")",".","If","tags","are","provided",",","calculates","tags","for","subtokens",".","Args",":","vocab_file",":","path","to","vocabulary","do_lower_case",":","set","True","if","lowercasing","is","needed","max_seq_length",":","max","sequence","length","in","subtokens",",","including","[","SEP","]","and","[","CLS","]","tokens","max_subword_length",":","replace","token","to","<","unk",">","if","it","'s","length","is","larger","than","this","(","defaults","to","None",",","which","is","equal","to","+infinity",")","token_masking_prob",":","probability","of","masking","token","while","training","provide_subword_tags",":","output","tags","for","subwords","or","for","words","subword_mask_mode",":","subword","to","select","inside","word","tokens",",","can","be","``","first","''","or","``","last","''","(","default=","''","first","''",")","return_features",":","if","True",",","returns","answer","in","features","format","Attributes",":","max_seq_length",":","max","sequence","length","in","subtokens",",","including","[","SEP","]","and","[","CLS","]","tokens","max_subword_length",":","rmax","lenght","of","a","bert","subtoken","tokenizer",":","instance","of","Bert","FullTokenizer"]
code: def _ner_bert_tokenize(
      tokens: List[str],
      tags: List[str],
      tokenizer: AutoTokenizer,
      max_subword_len: int = None,
      mode: str = None,
      subword_mask_mode: str = "first",
      token_masking_prob: float = None,
  ) -> Tuple[List[str], List[int], List[str]]:
      do_masking = (mode == "train") and (
          token_masking_prob is not None
      )
      do_cutting = max_subword_len is not None
      tokens_subword = ["[CLS]"]
      startofword_markers = [0]
      tags_subword = ["X"]
      for token, tag in zip(tokens, tags):
          token_marker = int(tag != "X")
          subwords = tokenizer.tokenize(token)
          if not subwords or (
              do_cutting and (len(subwords) > max_subword_len)
          ):
              tokens_subword.append("[UNK]")
              startofword_markers.append(token_marker)
              tags_subword.append(tag)
          else:
              if do_masking and (random.random() < token_masking_prob):
                  tokens_subword.extend(["[MASK]"] * len(subwords))
              else:
                  tokens_subword.extend(subwords)
              if subword_mask_mode == "last":
                  startofword_markers.extend(
                      [0] * (len(subwords) - 1) + [token_marker]
                  )
              else:
                  startofword_markers.extend(
                      [token_marker] + [0] * (len(subwords) - 1)
                  )
              tags_subword.extend([tag] + ["X"] * (len(subwords) - 1))
  
      tokens_subword.append("[SEP]")
      startofword_markers.append(0)
      tags_subword.append("X")
      return tokens_subword, startofword_markers, tags_subword
  
imports: import math
  import random
  import re
  from collections import defaultdict
  from dataclasses import dataclass
  from logging import getLogger
  from pathlib import Path
  from typing import Tuple, List, Optional, Union, Dict, Set, Any
  import nltk
  import numpy
  import torch
  from transformers import AutoTokenizer
  from transformers.data.processors.utils import InputFeatures
  from deeppavlov.core.commands.utils import expand_path
  from deeppavlov.core.common.registry import register
  from deeppavlov.core.data.utils import zero_pad
  from deeppavlov.core.models.component import Component
  from deeppavlov.models.preprocessors.mask import Mask
label_desc: Use image node_id 3 for calling the TorchTransformersNerPreprocessor obj's underlying member method code with example usage: obj._ner_bert_tokenize(tokens, tags, tokenizer, max_subword_len, mode, subword_mask_mode, token_masking_prob) and returns: tokens_subword,  startofword_markers,  tags_subword