class_docstr: Constructs a Ernie-M tokenizer. It uses the `sentencepiece` tools to cut the words to sub-words.
  
  Args:
      sentencepiece_model_file (`str`):
          The file path of sentencepiece model.
      vocab_file (`str`, *optional*):
          The file path of the vocabulary.
      do_lower_case (`str`, *optional*, defaults to `True`):
          Whether or not to lowercase the input when tokenizing.
      unk_token (`str`, *optional*, defaults to `"[UNK]"`):
          A special token representing the `unknown (out-of-vocabulary)` token. An unknown token is set to be
          `unk_token` inorder to be converted to an ID.
      sep_token (`str`, *optional*, defaults to `"[SEP]"`):
          A special token separating two different sentences in the same input.
      pad_token (`str`, *optional*, defaults to `"[PAD]"`):
          A special token used to make arrays of tokens the same size for batching purposes.
      cls_token (`str`, *optional*, defaults to `"[CLS]"`):
          A special token used for sequence classification. It is the last token of the sequence when built with
          special tokens.
      mask_token (`str`, *optional*, defaults to `"[MASK]"`):
          A special token representing a masked token. This is the token used in the masked language modeling task
          which the model tries to predict the original unmasked ones.
class_docstr_tok: ["Constructs","a","Ernie-M","tokenizer",".","It","uses","the","`","sentencepiece","`","tools","to","cut","the","words","to","sub-words",".","Args",":","sentencepiece_model_file","(","`","str","`",")",":","The","file","path","of","sentencepiece","model",".","vocab_file","(","`","str","`",",","*","optional","*",")",":","The","file","path","of","the","vocabulary",".","do_lower_case","(","`","str","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","Whether","or","not","to","lowercase","the","input","when","tokenizing",".","unk_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","UNK","]","''","`",")",":","A","special","token","representing","the","`","unknown","(","out-of-vocabulary",")","`","token",".","An","unknown","token","is","set","to","be","`","unk_token","`","inorder","to","be","converted","to","an","ID",".","sep_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","SEP","]","''","`",")",":","A","special","token","separating","two","different","sentences","in","the","same","input",".","pad_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","PAD","]","''","`",")",":","A","special","token","used","to","make","arrays","of","tokens","the","same","size","for","batching","purposes",".","cls_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","CLS","]","''","`",")",":","A","special","token","used","for","sequence","classification",".","It","is","the","last","token","of","the","sequence","when","built","with","special","tokens",".","mask_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","MASK","]","''","`",")",":","A","special","token","representing","a","masked","token",".","This","is","the","token","used","in","the","masked","language","modeling","task","which","the","model","tries","to","predict","the","original","unmasked","ones","."]
docstr: Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
  special tokens using the tokenizer `encode` method.
  
  Args:
      token_ids_0 (`List[int]`):
          List of ids of the first sequence.
      token_ids_1 (`List[int]`, *optional*):
          Optional second list of IDs for sequence pairs.
      already_has_special_tokens (`str`, *optional*, defaults to `False`):
          Whether or not the token list is already formatted with special tokens for the model.
  Returns:
      `List[int]`:
          The list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
docstr_tok: ["Retrieves","sequence","ids","from","a","token","list","that","has","no","special","tokens","added",".","This","method","is","called","when","adding","special","tokens","using","the","tokenizer","`","encode","`","method",".","Args",":","token_ids_0","(","`","List","[","int","]","`",")",":","List","of","ids","of","the","first","sequence",".","token_ids_1","(","`","List","[","int","]","`",",","*","optional","*",")",":","Optional","second","list","of","IDs","for","sequence","pairs",".","already_has_special_tokens","(","`","str","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","or","not","the","token","list","is","already","formatted","with","special","tokens","for","the","model",".","Returns",":","`","List","[","int","]","`",":","The","list","of","integers","in","the","range","[","0",",","1","]",":","1","for","a","special","token",",","0","for","a","sequence","token","."]
code: def get_special_tokens_mask(
      self,
      token_ids_0,
      token_ids_1=None,
      already_has_special_tokens=False,
  ):
      r"""
      Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
      special tokens using the tokenizer `encode` method.
  
      Args:
          token_ids_0 (`List[int]`):
              List of ids of the first sequence.
          token_ids_1 (`List[int]`, *optional*):
              Optional second list of IDs for sequence pairs.
          already_has_special_tokens (`str`, *optional*, defaults to `False`):
              Whether or not the token list is already formatted with special tokens for the model.
      Returns:
          `List[int]`:
              The list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
      """
  
      if already_has_special_tokens:
          if token_ids_1 is not None:
              raise ValueError(
                  "You should not supply a second sequence if the provided sequence of "
                  "ids is already formatted with special tokens for the model."
              )
          return [
              1 if x in [self.sep_token_id, self.cls_token_id] else 0
              for x in token_ids_0
          ]
  
      if token_ids_1 is not None:
          return (
              [1]
              + ([0] * len(token_ids_0))
              + [1, 1]
              + ([0] * len(token_ids_1))
              + [1]
          )
      return [1] + ([0] * len(token_ids_0)) + [1]
  
imports: import io
  import os
  import unicodedata
  from typing import Any, Dict, List, Optional, Tuple
  import sentencepiece
  from ...tokenization_utils import PreTrainedTokenizer
  from ...utils import logging
label_desc: Use image node_id 15 for calling the ErnieMTokenizer obj's underlying member method code with example usage: obj.get_special_tokens_mask(token_ids_0, token_ids_1, already_has_special_tokens) and returns: unknown, unknown, unknown