code: def __init__(
      self,
      model_path: Optional[str] = None,
      vocab: Optional[str] = None,
  ):
      tokenizers = try_import_huggingface_tokenizers()
      assert parse_version(tokenizers.__version__) >= parse_version(
          "0.8"
      ), (
          "Only support tokenizers>=0.8. You can upgrade tokenizers via "
          "`python3 -m pip install --upgrade tokenizers`."
      )
      self._model_path = model_path
      self._model = tokenizers.Tokenizer.from_file(model_path)
      hf_vocab = self._model.get_vocab()
      with open(model_path, "r", encoding="utf-8") as f:
          model_info = json.load(f)
          self._model_info = model_info
      added_tokens = model_info["added_tokens"]
      if vocab is not None:
          self._vocab = load_vocab(vocab)
      else:
          sorted_hf_vocab_kv = sorted(
              list(hf_vocab.items()), key=lambda x: x[1]
          )
          for i, ele in enumerate(sorted_hf_vocab_kv):
              assert ele[1] == i
          all_tokens = [ele[0] for ele in sorted_hf_vocab_kv]
          special_tokens = [token["content"] for token in added_tokens]
          special_token_keys = []
          no_valid_name_key_cnt = 0
          for special_token in special_tokens:
              if (
                  special_token.startswith("<")
                  and special_token.endswith(">")
                  and len(special_token) > 2
              ):
                  key = special_token[1:-1] + "_token"
              else:
                  key = "special{}_token".format(no_valid_name_key_cnt)
                  no_valid_name_key_cnt += 1
              assert key not in special_token_keys
              special_token_keys.append(key)
          self._vocab = Vocab(
              all_tokens,
              **{
                  key: token
                  for key, token in zip(
                      special_token_keys, special_tokens
                  )
              },
          )
  
      # Verify the special tokens
      for added_token in added_tokens:
          assert (
              self._vocab[added_token["content"]] == added_token["id"]
          )
          assert added_token["content"] in self._vocab.special_tokens
      # Verify all tokens exist
      for token, idx in hf_vocab.items():
          assert self._vocab[token] == idx
      if self._model_info["decoder"]["type"] == "BPEDecoder":
          self._last_subtoken_id_set = frozenset(
              [
                  i
                  for i, ele in enumerate(self._vocab.all_tokens)
                  if ele.endswith("</w>")
              ]
          )
      elif self._model_info["decoder"]["type"] == "WordPiece":
          self._first_subtoken_id_set = frozenset(
              [
                  i
                  for i, ele in enumerate(self._vocab.all_tokens)
                  if not ele.startswith("##")
              ]
          )
  
imports: import os
  import json
  from pkg_resources import parse_version
  from typing import Optional, Union, List, Tuple
  from collections import OrderedDict
  from uuid import uuid4
  from .base import *
  from ..vocab import Vocab, load_vocab
  from ...utils.lazy_imports import try_import_huggingface_tokenizers
label_desc: Use image node_id 1 to create a new HuggingFaceTokenizer object from inherited base classes: BaseTokenizerWithVocab with example: obj = HuggingFaceTokenizer(model_path, vocab)