class_docstr: Abstract BPE Helper.
  
  BPE Helper subclasses must implement appropriate abstractmethods.
class_docstr_tok: ["Abstract","BPE","Helper",".","BPE","Helper","subclasses","must","implement","appropriate","abstractmethods","."]
docstr: Decode list of tokens into a text string.
  
  NOTE: DO NOT OVERRIDE
  
  :param tokens:
      list of tokens
  :param token_ids:
      list of token ids
  :param delimiter:
      string delimiter for tokens
  
  :return text:
      decoded text
docstr_tok: ["Decode","list","of","tokens","into","a","text","string",".","NOTE",":","DO","NOT","OVERRIDE",":","param","tokens",":","list","of","tokens",":","param","token_ids",":","list","of","token","ids",":","param","delimiter",":","string","delimiter","for","tokens",":","return","text",":","decoded","text"]
code: def decode(
      self,
      tokens: List[str],
      token_ids: List[int],
      delimiter: str = " ",
  ) -> str:
      """
      Decode list of tokens into a text string.
  
      NOTE: DO NOT OVERRIDE
  
      :param tokens:
          list of tokens
      :param token_ids:
          list of token ids
      :param delimiter:
          string delimiter for tokens
  
      :return text:
          decoded text
      """
      if self.debug:
          return delimiter.join(tokens)
  
      for i, token in enumerate(tokens):
          # note, HF ByteLevelBPE tokenizer handles special tokens itself in
          # a special way, so this will be skipped
          if token in self._special_tokens:
              # special token found. to the left, we've already cleared
              left = self.helper_decode(
                  tokens[:i], token_ids[:i], delimiter
              )
              # token itself is easy to map to a string
              center = token
              # to the right, there may still be special tokens
              right = self.decode(
                  tokens[min(len(token_ids), i + 1) :],
                  token_ids[min(len(token_ids), i + 1) :],
                  delimiter,
              )
              return left + center + right
  
      # no special tokens found, we can fall back
      text = self.helper_decode(tokens, token_ids, delimiter)
      if self.add_prefix_space:
          assert text.startswith(" ")
          text = text.lstrip(" ")
      return text
  
imports: from parlai.core.params import ParlaiParser
  from abc import ABC, abstractmethod
  from functools import lru_cache
  import json
  import os
  import random
  import re
  from typing import Dict, List, Optional, Set, Tuple
  from typing_extensions import final
  from parlai.core.build_data import download, make_dir
  from parlai.core.opt import Opt
  from parlai.utils.misc import warn_once
  from parlai.utils.typing import TShared
  from parlai.utils.io import PathManager
  import parlai.utils.logging
label_desc: Use image node_id 6 for calling the BPEHelper obj's underlying member method code with example usage: obj.decode(tokens, token_ids, delimiter) and returns: text, delimiter, unknown