docstr: Creates an Encoding object.
  
  See openai_public.py for examples of how to construct an Encoding object.
  
  Args:
      name: The name of the encoding. It should be clear from the name of the encoding
          what behaviour to expect, in particular, encodings with different special tokens
          should have different names.
      pat_str: A regex pattern string that is used to split the input text.
      mergeable_ranks: A dictionary mapping mergeable token bytes to their ranks. The ranks
          must correspond to merge priority.
      special_tokens: A dictionary mapping special token strings to their token values.
      explicit_n_vocab: The number of tokens in the vocabulary. If provided, it is checked
          that the number of mergeable tokens and special tokens is equal to this number.
docstr_tok: ["Creates","an","Encoding","object",".","See","openai_public.py","for","examples","of","how","to","construct","an","Encoding","object",".","Args",":","name",":","The","name","of","the","encoding",".","It","should","be","clear","from","the","name","of","the","encoding","what","behaviour","to","expect",",","in","particular",",","encodings","with","different","special","tokens","should","have","different","names",".","pat_str",":","A","regex","pattern","string","that","is","used","to","split","the","input","text",".","mergeable_ranks",":","A","dictionary","mapping","mergeable","token","bytes","to","their","ranks",".","The","ranks","must","correspond","to","merge","priority",".","special_tokens",":","A","dictionary","mapping","special","token","strings","to","their","token","values",".","explicit_n_vocab",":","The","number","of","tokens","in","the","vocabulary",".","If","provided",",","it","is","checked","that","the","number","of","mergeable","tokens","and","special","tokens","is","equal","to","this","number","."]
code: def __init__(
      self,
      name: str,
      *,
      pat_str: str,
      mergeable_ranks: dict[bytes, int],
      special_tokens: dict[str, int],
      explicit_n_vocab: Optional[int] = None,
  ):
      """Creates an Encoding object.
  
      See openai_public.py for examples of how to construct an Encoding object.
  
      Args:
          name: The name of the encoding. It should be clear from the name of the encoding
              what behaviour to expect, in particular, encodings with different special tokens
              should have different names.
          pat_str: A regex pattern string that is used to split the input text.
          mergeable_ranks: A dictionary mapping mergeable token bytes to their ranks. The ranks
              must correspond to merge priority.
          special_tokens: A dictionary mapping special token strings to their token values.
          explicit_n_vocab: The number of tokens in the vocabulary. If provided, it is checked
              that the number of mergeable tokens and special tokens is equal to this number.
      """
      self.name = name
  
      self._pat_str = pat_str
      self._mergeable_ranks = mergeable_ranks
      self._special_tokens = special_tokens
  
      self.max_token_value = max(
          max(mergeable_ranks.values()),
          max(special_tokens.values(), default=0),
      )
      if explicit_n_vocab:
          assert (
              len(mergeable_ranks) + len(special_tokens)
              == explicit_n_vocab
          )
          assert self.max_token_value == explicit_n_vocab - 1
  
      self._core_bpe = _tiktoken.CoreBPE(
          mergeable_ranks, special_tokens, pat_str
      )
  
imports: from __future__ import annotations
  import functools
  from concurrent.futures import ThreadPoolExecutor
  from typing import AbstractSet, Collection, Literal, NoReturn, Optional, Union
  import regex
  from tiktoken import _tiktoken
label_desc: Use image node_id 1 to create a new Encoding object  with example: obj = Encoding(name)