class_docstr: Construct a CLIP tokenizer. Based on byte-level Byte-Pair-Encoding.
  
  This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
  this superclass for more information regarding those methods.
  
  Args:
      vocab_file (`str`):
          Path to the vocabulary file.
      merges_file (`str`):
          Path to the merges file.
      errors (`str`, *optional*, defaults to `"replace"`):
          Paradigm to follow when decoding bytes to UTF-8. See
          [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
      unk_token (`str`, *optional*, defaults to `<|endoftext|>`):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      bos_token (`str`, *optional*, defaults to `<|endoftext|>`):
          The beginning of sequence token.
      eos_token (`str`, *optional*, defaults to `<|endoftext|>`):
          The end of sequence token.
class_docstr_tok: ["Construct","a","CLIP","tokenizer",".","Based","on","byte-level","Byte-Pair-Encoding",".","This","tokenizer","inherits","from","[","`","PreTrainedTokenizer","`","]","which","contains","most","of","the","main","methods",".","Users","should","refer","to","this","superclass","for","more","information","regarding","those","methods",".","Args",":","vocab_file","(","`","str","`",")",":","Path","to","the","vocabulary","file",".","merges_file","(","`","str","`",")",":","Path","to","the","merges","file",".","errors","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","replace","''","`",")",":","Paradigm","to","follow","when","decoding","bytes","to","UTF-8",".","See","[","bytes.decode","]","(","https",":","\/\/docs.python.org\/3\/library\/stdtypes.html","#","bytes.decode",")","for","more","information",".","unk_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","<","|endoftext|",">","`",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","bos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","<","|endoftext|",">","`",")",":","The","beginning","of","sequence","token",".","eos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","<","|endoftext|",">","`",")",":","The","end","of","sequence","token","."]
code: def __init__(
      self,
      vocab_file,
      merges_file,
      errors="replace",
      unk_token="<|endoftext|>",
      bos_token="<|startoftext|>",
      eos_token="<|endoftext|>",
      pad_token="<|endoftext|>",  # hack to enable padding
      **kwargs,
  ):
      bos_token = (
          AddedToken(bos_token, lstrip=False, rstrip=False)
          if isinstance(bos_token, str)
          else bos_token
      )
      eos_token = (
          AddedToken(eos_token, lstrip=False, rstrip=False)
          if isinstance(eos_token, str)
          else eos_token
      )
      unk_token = (
          AddedToken(unk_token, lstrip=False, rstrip=False)
          if isinstance(unk_token, str)
          else unk_token
      )
  
      super().__init__(
          errors=errors,
          unk_token=unk_token,
          bos_token=bos_token,
          eos_token=eos_token,
          pad_token=pad_token,
          **kwargs,
      )
  
      try:
          import ftfy
  
          self.fix_text = ftfy.fix_text
      except ImportError:
          logger.warning(
              "ftfy or spacy is not installed using BERT BasicTokenizer instead of ftfy."
          )
          self.nlp = BasicTokenizer(do_lower_case=True)
          self.fix_text = None
  
      with open(vocab_file, encoding="utf-8") as vocab_handle:
          self.encoder = json.load(vocab_handle)
      self.decoder = {v: k for k, v in self.encoder.items()}
      self.errors = errors  # how to handle errors in decoding
      self.byte_encoder = bytes_to_unicode()
      self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
      with open(merges_file, encoding="utf-8") as merges_handle:
          bpe_merges = (
              merges_handle.read()
              .strip()
              .split("\n")[1 : 49152 - 256 - 2 + 1]
          )
      bpe_merges = [tuple(merge.split()) for merge in bpe_merges]
      self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
      self.cache = {
          "<|startoftext|>": "<|startoftext|>",
          "<|endoftext|>": "<|endoftext|>",
      }
  
      self.pat = re.compile(
          r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""",
          re.IGNORECASE,
      )
  
imports: import json
  import os
  import unicodedata
  from functools import lru_cache
  from typing import List, Optional, Tuple
  import regex
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
label_desc: Use image node_id 1 to create a new CLIPTokenizer object from inherited base classes: PreTrainedTokenizer with example: obj = CLIPTokenizer(vocab_file, merges_file, errors, unk_token, bos_token, eos_token, pad_token)