docstr: Parameters
  ----------
  merges_file
      The merges file saved by HuggingFace
  vocab_file
      Vocabulary file in GluonNLP
  unk_token
      The unknown token
  suffix
      The suffix for sub-tokens. For example, "Sunnyvale" will be "Sunny vale</w>"
  dropout
      Ratio of the BPE-Dropout
  lowercase
      Whether to lowercase the input before tokenizer
docstr_tok: ["Parameters","--","--","--","--","--","merges_file","The","merges","file","saved","by","HuggingFace","vocab_file","Vocabulary","file","in","GluonNLP","unk_token","The","unknown","token","suffix","The","suffix","for","sub-tokens",".","For","example",",","``","Sunnyvale","''","will","be","``","Sunny","vale","<","\/w",">","''","dropout","Ratio","of","the","BPE-Dropout","lowercase","Whether","to","lowercase","the","input","before","tokenizer"]
code: def __init__(
      self,
      merges_file: Optional[str] = None,
      vocab_file: Optional[str] = None,
      unk_token: Optional[str] = Vocab.UNK_TOKEN,
      suffix: Optional[str] = "</w>",
      dropout: Optional[float] = None,
      lowercase: bool = False,
  ):
      """
  
      Parameters
      ----------
      merges_file
          The merges file saved by HuggingFace
      vocab_file
          Vocabulary file in GluonNLP
      unk_token
          The unknown token
      suffix
          The suffix for sub-tokens. For example, "Sunnyvale" will be "Sunny vale</w>"
      dropout
          Ratio of the BPE-Dropout
      lowercase
          Whether to lowercase the input before tokenizer
      """
      super().__init__()
      self._merges_file = merges_file
      self._vocab_file = vocab_file
      self._unk_token = unk_token
      self._suffix = suffix
      self._dropout = dropout
      self._lowercase = lowercase
      self.__rebuild_tokenizer()
      self._last_subword_id_set = frozenset(
          [
              self._vocab[ele]
              for ele in self._vocab.all_tokens
              if ele.endswith(self._suffix)
          ]
      )
  
imports: import os
  import json
  from pkg_resources import parse_version
  from typing import Optional, Union, List, Tuple
  from collections import OrderedDict
  from uuid import uuid4
  from .base import *
  from ..vocab import Vocab, load_vocab
  from ...utils.lazy_imports import try_import_huggingface_tokenizers
label_desc: Use image node_id 1 to create a new HuggingFaceBPETokenizer object from inherited base classes: LegacyHuggingFaceTokenizer with example: obj = HuggingFaceBPETokenizer(merges_file, vocab_file, unk_token, suffix, dropout, lowercase)