class_docstr: Construct an M2M100 tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
  
  This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
  this superclass for more information regarding those methods.
  
  Args:
      vocab_file (`str`):
          Path to the vocabulary file.
      spm_file (`str`):
          Path to [SentencePiece](https://github.com/google/sentencepiece) file (generally has a .spm extension) that
          contains the vocabulary.
      src_lang (`str`, *optional*):
          A string representing the source language.
      tgt_lang (`str`, *optional*):
          A string representing the target language.
      eos_token (`str`, *optional*, defaults to `"</s>"`):
          The end of sequence token.
      sep_token (`str`, *optional*, defaults to `"</s>"`):
          The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
          sequence classification or for a text and a question for question answering. It is also used as the last
          token of a sequence built with special tokens.
      unk_token (`str`, *optional*, defaults to `"<unk>"`):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      pad_token (`str`, *optional*, defaults to `"<pad>"`):
          The token used for padding, for example when batching sequences of different lengths.
      language_codes (`str`, *optional*, defaults to `"m2m100"`):
          What language codes to use. Should be one of `"m2m100"` or `"wmt21"`.
      sp_model_kwargs (`dict`, *optional*):
          Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
          SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
          to set:
  
          - `enable_sampling`: Enable subword regularization.
          - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
  
            - `nbest_size = {0,1}`: No sampling is performed.
            - `nbest_size > 1`: samples from the nbest_size results.
            - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
              using forward-filtering-and-backward-sampling algorithm.
  
          - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
            BPE-dropout.
  
  Examples:
  
  ```python
  >>> from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer
  
  >>> model = M2M100ForConditionalGeneration.from_pretrained("facebook/m2m100_418M")
  >>> tokenizer = M2M100Tokenizer.from_pretrained("facebook/m2m100_418M", src_lang="en", tgt_lang="ro")
  >>> src_text = " UN Chief Says There Is No Military Solution in Syria"
  >>> tgt_text = "Şeful ONU declară că nu există o soluţie militară în Siria"
  >>> model_inputs = tokenizer(src_text, text_target=tgt_text, return_tensors="pt")
  >>> outputs = model(**model_inputs)  # should work
  ```
class_docstr_tok: ["Construct","an","M2M100","tokenizer",".","Based","on","[","SentencePiece","]","(","https",":","\/\/github.com\/google\/sentencepiece",")",".","This","tokenizer","inherits","from","[","`","PreTrainedTokenizer","`","]","which","contains","most","of","the","main","methods",".","Users","should","refer","to","this","superclass","for","more","information","regarding","those","methods",".","Args",":","vocab_file","(","`","str","`",")",":","Path","to","the","vocabulary","file",".","spm_file","(","`","str","`",")",":","Path","to","[","SentencePiece","]","(","https",":","\/\/github.com\/google\/sentencepiece",")","file","(","generally","has","a",".spm","extension",")","that","contains","the","vocabulary",".","src_lang","(","`","str","`",",","*","optional","*",")",":","A","string","representing","the","source","language",".","tgt_lang","(","`","str","`",",","*","optional","*",")",":","A","string","representing","the","target","language",".","eos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","\/s",">","''","`",")",":","The","end","of","sequence","token",".","sep_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","\/s",">","''","`",")",":","The","separator","token",",","which","is","used","when","building","a","sequence","from","multiple","sequences",",","e.g",".","two","sequences","for","sequence","classification","or","for","a","text","and","a","question","for","question","answering",".","It","is","also","used","as","the","last","token","of","a","sequence","built","with","special","tokens",".","unk_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","unk",">","''","`",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","pad_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","pad",">","''","`",")",":","The","token","used","for","padding",",","for","example","when","batching","sequences","of","different","lengths",".","language_codes","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","m2m100","''","`",")",":","What","language","codes","to","use",".","Should","be","one","of","`","``","m2m100","''","`","or","`","``","wmt21","''","`",".","sp_model_kwargs","(","`","dict","`",",","*","optional","*",")",":","Will","be","passed","to","the","`","SentencePieceProcessor.__init__","(",")","`","method",".","The","[","Python","wrapper","for","SentencePiece","]","(","https",":","\/\/github.com\/google\/sentencepiece\/tree\/master\/python",")","can","be","used",",","among","other","things",",","to","set",":","-","`","enable_sampling","`",":","Enable","subword","regularization",".","-","`","nbest_size","`",":","Sampling","parameters","for","unigram",".","Invalid","for","BPE-Dropout",".","-","`","nbest_size","=","{","0,1","}","`",":","No","sampling","is","performed",".","-","`","nbest_size",">","1","`",":","samples","from","the","nbest_size","results",".","-","`","nbest_size","<","0","`",":","assuming","that","nbest_size","is","infinite","and","samples","from","the","all","hypothesis","(","lattice",")","using","forward-filtering-and-backward-sampling","algorithm",".","-","`","alpha","`",":","Smoothing","parameter","for","unigram","sampling",",","and","dropout","probability","of","merge","operations","for","BPE-dropout",".","Examples",":","``","`","python",">",">",">","from","transformers","import","M2M100ForConditionalGeneration",",","M2M100Tokenizer",">",">",">","model","=","M2M100ForConditionalGeneration.from_pretrained","(","``","facebook\/m2m100_418M","''",")",">",">",">","tokenizer","=","M2M100Tokenizer.from_pretrained","(","``","facebook\/m2m100_418M","''",",","src_lang=","''","en","''",",","tgt_lang=","''","ro","''",")",">",">",">","src_text","=","``","UN","Chief","Says","There","Is","No","Military","Solution","in","Syria","''",">",">",">","tgt_text","=","``","\u015eeful","ONU","declar\u0103","c\u0103","nu","exist\u0103","o","solu\u0163ie","militar\u0103","\u00een","Siria","''",">",">",">","model_inputs","=","tokenizer","(","src_text",",","text_target=tgt_text",",","return_tensors=","''","pt","''",")",">",">",">","outputs","=","model","(","*","*","model_inputs",")","#","should","work","``","`"]
docstr: Converts a sequence of tokens (string) in a single string.
docstr_tok: ["Converts","a","sequence","of","tokens","(","string",")","in","a","single","string","."]
code: def convert_tokens_to_string(self, tokens):
      """Converts a sequence of tokens (string) in a single string."""
      current_sub_tokens = []
      out_string = ""
      for token in tokens:
          # make sure that special tokens are not decoded using sentencepiece model
          if token in self.all_special_tokens:
              out_string += (
                  self.sp_model.decode(current_sub_tokens) + token
              )
              current_sub_tokens = []
          else:
              current_sub_tokens.append(token)
      out_string += self.sp_model.decode(current_sub_tokens)
      return out_string.strip()
  
imports: import json
  import os
  from pathlib import Path
  from shutil import copyfile
  from typing import Any, Dict, List, Optional, Tuple, Union
  import sentencepiece
  from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
  from ...utils import logging
label_desc: Use image node_id 9 for calling the M2M100Tokenizer obj's underlying member method code with example usage: obj.convert_tokens_to_string(tokens) and returns: out_string