class_docstr: Construct an PLBART tokenizer.
  
  Adapted from [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on
  [SentencePiece](https://github.com/google/sentencepiece).
  
  The tokenization method is `<tokens> <eos> <language code>` for source language documents, and `<language code>
  <tokens> <eos>` for target language documents.
  
  Args:
      vocab_file (`str`):
          Path to the vocabulary file.
      src_lang (`str`, *optional*):
          A string representing the source language.
      tgt_lang (`str`, *optional*):
          A string representing the target language.
      bos_token (`str`, *optional*, defaults to `"<s>"`):
          The start of sequence token.
      eos_token (`str`, *optional*, defaults to `"</s>"`):
          The end of sequence token.
      sep_token (`str`, *optional*, defaults to `"</s>"`):
          The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
          sequence classification or for a text and a question for question answering. It is also used as the last
          token of a sequence built with special tokens.
      cls_token (`str`, *optional*, defaults to `"<s>"`):
          The cls token, which is a special token used as the first token for all tasks.
      unk_token (`str`, *optional*, defaults to `"<unk>"`):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      pad_token (`str`, *optional*, defaults to `"<pad>"`):
          The token used for padding, for example when batching sequences of different lengths.
      mask_token(`str`, *optional*, defaults to `"<mask>"`):
          The token used for masking values. This is the token used when training this model with masking tasks. This
          is only used in the `"base"` tokenizer type. For `"multi"` tokenizer, masking is never done for the
          downstream tasks.
      language_codes (`str`, *optional*, defaults to `"base"`):
          What language codes to use. Should be one of `"base"` or `"multi"`.
      sp_model_kwargs (`dict`, *optional*):
          Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
          SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
          to set:
          - `enable_sampling`: Enable subword regularization.
          - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
            - `nbest_size = {0,1}`: No sampling is performed.
            - `nbest_size > 1`: samples from the nbest_size results.
            - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
              using forward-filtering-and-backward-sampling algorithm.
          - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
            BPE-dropout.
  
  Examples:
  
  ```python
  >>> from transformers import PLBartTokenizer
  
  >>> tokenizer = PLBartTokenizer.from_pretrained("uclanlp/plbart-python-en_XX", src_lang="python", tgt_lang="en_XX")
  >>> example_python_phrase = "def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])"
  >>> expected_translation_english = "Returns the maximum value of a b c."
  >>> inputs = tokenizer(example_python_phrase, text_target=expected_translation_english, return_tensors="pt")
  ```
class_docstr_tok: ["Construct","an","PLBART","tokenizer",".","Adapted","from","[","`","RobertaTokenizer","`","]","and","[","`","XLNetTokenizer","`","]",".","Based","on","[","SentencePiece","]","(","https",":","\/\/github.com\/google\/sentencepiece",")",".","The","tokenization","method","is","`","<","tokens",">","<","eos",">","<","language","code",">","`","for","source","language","documents",",","and","`","<","language","code",">","<","tokens",">","<","eos",">","`","for","target","language","documents",".","Args",":","vocab_file","(","`","str","`",")",":","Path","to","the","vocabulary","file",".","src_lang","(","`","str","`",",","*","optional","*",")",":","A","string","representing","the","source","language",".","tgt_lang","(","`","str","`",",","*","optional","*",")",":","A","string","representing","the","target","language",".","bos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","s",">","''","`",")",":","The","start","of","sequence","token",".","eos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","\/s",">","''","`",")",":","The","end","of","sequence","token",".","sep_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","\/s",">","''","`",")",":","The","separator","token",",","which","is","used","when","building","a","sequence","from","multiple","sequences",",","e.g",".","two","sequences","for","sequence","classification","or","for","a","text","and","a","question","for","question","answering",".","It","is","also","used","as","the","last","token","of","a","sequence","built","with","special","tokens",".","cls_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","s",">","''","`",")",":","The","cls","token",",","which","is","a","special","token","used","as","the","first","token","for","all","tasks",".","unk_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","unk",">","''","`",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","pad_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","pad",">","''","`",")",":","The","token","used","for","padding",",","for","example","when","batching","sequences","of","different","lengths",".","mask_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","mask",">","''","`",")",":","The","token","used","for","masking","values",".","This","is","the","token","used","when","training","this","model","with","masking","tasks",".","This","is","only","used","in","the","`","``","base","''","`","tokenizer","type",".","For","`","``","multi","''","`","tokenizer",",","masking","is","never","done","for","the","downstream","tasks",".","language_codes","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","base","''","`",")",":","What","language","codes","to","use",".","Should","be","one","of","`","``","base","''","`","or","`","``","multi","''","`",".","sp_model_kwargs","(","`","dict","`",",","*","optional","*",")",":","Will","be","passed","to","the","`","SentencePieceProcessor.__init__","(",")","`","method",".","The","[","Python","wrapper","for","SentencePiece","]","(","https",":","\/\/github.com\/google\/sentencepiece\/tree\/master\/python",")","can","be","used",",","among","other","things",",","to","set",":","-","`","enable_sampling","`",":","Enable","subword","regularization",".","-","`","nbest_size","`",":","Sampling","parameters","for","unigram",".","Invalid","for","BPE-Dropout",".","-","`","nbest_size","=","{","0,1","}","`",":","No","sampling","is","performed",".","-","`","nbest_size",">","1","`",":","samples","from","the","nbest_size","results",".","-","`","nbest_size","<","0","`",":","assuming","that","nbest_size","is","infinite","and","samples","from","the","all","hypothesis","(","lattice",")","using","forward-filtering-and-backward-sampling","algorithm",".","-","`","alpha","`",":","Smoothing","parameter","for","unigram","sampling",",","and","dropout","probability","of","merge","operations","for","BPE-dropout",".","Examples",":","``","`","python",">",">",">","from","transformers","import","PLBartTokenizer",">",">",">","tokenizer","=","PLBartTokenizer.from_pretrained","(","``","uclanlp\/plbart-python-en_XX","''",",","src_lang=","''","python","''",",","tgt_lang=","''","en_XX","''",")",">",">",">","example_python_phrase","=","``","def","maximum","(","a",",","b",",","c",")",":","NEW_LINE_INDENTreturn","max","(","[","a",",","b",",","c","]",")","''",">",">",">","expected_translation_english","=","``","Returns","the","maximum","value","of","a","b","c.","''",">",">",">","inputs","=","tokenizer","(","example_python_phrase",",","text_target=expected_translation_english",",","return_tensors=","''","pt","''",")","``","`"]
code: def __init__(
      self,
      vocab_file,
      bos_token="<s>",
      eos_token="</s>",
      sep_token="</s>",
      cls_token="<s>",
      unk_token="<unk>",
      pad_token="<pad>",
      mask_token="<mask>",
      language_codes="base",
      tokenizer_file=None,
      src_lang=None,
      tgt_lang=None,
      sp_model_kwargs: Optional[Dict[str, Any]] = None,
      additional_special_tokens=None,
      **kwargs,
  ):
      # Mask token behave like a normal word, i.e. include the space before it
      mask_token = (
          AddedToken(mask_token, lstrip=True, rstrip=False)
          if isinstance(mask_token, str)
          else mask_token
      )
  
      self.sp_model_kwargs = (
          {} if sp_model_kwargs is None else sp_model_kwargs
      )
      src_lang = self._convert_lang_code_special_format(src_lang)
      tgt_lang = self._convert_lang_code_special_format(tgt_lang)
  
      self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.Load(str(vocab_file))
      self.vocab_file = vocab_file
      self.language_codes = language_codes
  
      fairseq_language_codes = FAIRSEQ_LANGUAGE_CODES[
          self.language_codes
      ]
  
      # Original fairseq vocab and spm vocab must be "aligned":
      # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
      # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
      # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
      # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'
  
      # Mimic fairseq token-to-id alignment for the first 4 token
      self.fairseq_tokens_to_ids = {
          "<s>": 0,
          "<pad>": 1,
          "</s>": 2,
          "<unk>": 3,
      }
  
      # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
      self.fairseq_offset = 1
  
      self.sp_model_size = len(self.sp_model)
      self.lang_code_to_id = {
          code: self.sp_model_size + i + self.fairseq_offset
          for i, code in enumerate(fairseq_language_codes)
      }
      self.id_to_lang_code = {
          v: k for k, v in self.lang_code_to_id.items()
      }
  
      if self.language_codes == "base":
          self.fairseq_tokens_to_ids["<mask>"] = (
              len(self.sp_model)
              + len(self.lang_code_to_id)
              + self.fairseq_offset
          )
  
      self.fairseq_tokens_to_ids.update(self.lang_code_to_id)
      self.fairseq_ids_to_tokens = {
          v: k for k, v in self.fairseq_tokens_to_ids.items()
      }
      _additional_special_tokens = list(self.lang_code_to_id.keys())
  
      if additional_special_tokens is not None:
          # Only add those special tokens if they are not already there.
          _additional_special_tokens.extend(
              [
                  t
                  for t in additional_special_tokens
                  if t not in _additional_special_tokens
              ]
          )
  
      if self.language_codes == "base":
          self._src_lang = src_lang
          self.cur_lang_code_id = (
              self.lang_code_to_id[self._src_lang]
              if self._src_lang is not None
              else self._src_lang
          )
      else:
          self._src_lang = (
              src_lang if src_lang is not None else "__en_XX__"
          )
          self.cur_lang_code_id = self.lang_code_to_id[self._src_lang]
  
      super().__init__(
          bos_token=bos_token,
          eos_token=eos_token,
          unk_token=unk_token,
          sep_token=sep_token,
          cls_token=cls_token,
          pad_token=pad_token,
          mask_token=mask_token,
          language_codes=language_codes,
          tokenizer_file=tokenizer_file,
          src_lang=src_lang,
          tgt_lang=tgt_lang,
          additional_special_tokens=_additional_special_tokens,
          sp_model_kwargs=self.sp_model_kwargs,
          **kwargs,
      )
  
      self.tgt_lang = tgt_lang
      self.set_src_lang_special_tokens(self._src_lang)
  
imports: import os
  from shutil import copyfile
  from typing import Any, Dict, List, Optional, Tuple
  import sentencepiece
  from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
  from ...utils import logging
label_desc: Use image node_id 1 to create a new PLBartTokenizer object from inherited base classes: PreTrainedTokenizer with example: obj = PLBartTokenizer(vocab_file, bos_token, eos_token, sep_token, cls_token, unk_token, pad_token, mask_token, language_codes, tokenizer_file, src_lang, tgt_lang, sp_model_kwargs, additional_special_tokens)