class_docstr: Constructs a Ernie-M tokenizer. It uses the `sentencepiece` tools to cut the words to sub-words.
  
  Args:
      sentencepiece_model_file (`str`):
          The file path of sentencepiece model.
      vocab_file (`str`, *optional*):
          The file path of the vocabulary.
      do_lower_case (`str`, *optional*, defaults to `True`):
          Whether or not to lowercase the input when tokenizing.
      unk_token (`str`, *optional*, defaults to `"[UNK]"`):
          A special token representing the `unknown (out-of-vocabulary)` token. An unknown token is set to be
          `unk_token` inorder to be converted to an ID.
      sep_token (`str`, *optional*, defaults to `"[SEP]"`):
          A special token separating two different sentences in the same input.
      pad_token (`str`, *optional*, defaults to `"[PAD]"`):
          A special token used to make arrays of tokens the same size for batching purposes.
      cls_token (`str`, *optional*, defaults to `"[CLS]"`):
          A special token used for sequence classification. It is the last token of the sequence when built with
          special tokens.
      mask_token (`str`, *optional*, defaults to `"[MASK]"`):
          A special token representing a masked token. This is the token used in the masked language modeling task
          which the model tries to predict the original unmasked ones.
class_docstr_tok: ["Constructs","a","Ernie-M","tokenizer",".","It","uses","the","`","sentencepiece","`","tools","to","cut","the","words","to","sub-words",".","Args",":","sentencepiece_model_file","(","`","str","`",")",":","The","file","path","of","sentencepiece","model",".","vocab_file","(","`","str","`",",","*","optional","*",")",":","The","file","path","of","the","vocabulary",".","do_lower_case","(","`","str","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","Whether","or","not","to","lowercase","the","input","when","tokenizing",".","unk_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","UNK","]","''","`",")",":","A","special","token","representing","the","`","unknown","(","out-of-vocabulary",")","`","token",".","An","unknown","token","is","set","to","be","`","unk_token","`","inorder","to","be","converted","to","an","ID",".","sep_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","SEP","]","''","`",")",":","A","special","token","separating","two","different","sentences","in","the","same","input",".","pad_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","PAD","]","''","`",")",":","A","special","token","used","to","make","arrays","of","tokens","the","same","size","for","batching","purposes",".","cls_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","CLS","]","''","`",")",":","A","special","token","used","for","sequence","classification",".","It","is","the","last","token","of","the","sequence","when","built","with","special","tokens",".","mask_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","[","MASK","]","''","`",")",":","A","special","token","representing","a","masked","token",".","This","is","the","token","used","in","the","masked","language","modeling","task","which","the","model","tries","to","predict","the","original","unmasked","ones","."]
code: def get_offset_mapping(self, text):
      if text is None:
          return None
  
      split_tokens = self.tokenize(text)
      normalized_text, char_mapping = "", []
  
      for i, ch in enumerate(text):
          if ch in self.SP_CHAR_MAPPING:
              ch = self.SP_CHAR_MAPPING.get(ch)
          else:
              ch = unicodedata.normalize("NFKC", ch)
          if self.is_whitespace(ch):
              continue
          normalized_text += ch
          char_mapping.extend([i] * len(ch))
  
      text, token_mapping, offset = normalized_text, [], 0
  
      if self.do_lower_case:
          text = text.lower()
  
      for token in split_tokens:
          if token[:1] == "▁":
              token = token[1:]
          start = text[offset:].index(token) + offset
          end = start + len(token)
  
          token_mapping.append(
              (char_mapping[start], char_mapping[end - 1] + 1)
          )
          offset = end
      return token_mapping
  
imports: import io
  import os
  import unicodedata
  from typing import Any, Dict, List, Optional, Tuple
  import sentencepiece
  from ...tokenization_utils import PreTrainedTokenizer
  from ...utils import logging
label_desc: Use image node_id 2 for calling the ErnieMTokenizer obj's underlying member method code with example usage: obj.get_offset_mapping(text) and returns: token_mapping, None