class_docstr: Abstract BPE Helper.
  
  BPE Helper subclasses must implement appropriate abstractmethods.
class_docstr_tok: ["Abstract","BPE","Helper",".","BPE","Helper","subclasses","must","implement","appropriate","abstractmethods","."]
docstr: Tokenize text.
  
  Checks for add_prefix_space; handles accordingly
  
  NOTE: DO NOT OVERRIDE
  
  :param text:
      text to tokenize
  
  :return tokens:
      A list of tokens
docstr_tok: ["Tokenize","text",".","Checks","for","add_prefix_space",";","handles","accordingly","NOTE",":","DO","NOT","OVERRIDE",":","param","text",":","text","to","tokenize",":","return","tokens",":","A","list","of","tokens"]
code: def encode(self, text: str) -> List[str]:
      """
      Tokenize text.
  
      Checks for add_prefix_space; handles accordingly
  
      NOTE: DO NOT OVERRIDE
  
      :param text:
          text to tokenize
  
      :return tokens:
          A list of tokens
      """
      for special_token in self._special_tokens.keys():
          split = text.split(special_token)
          if len(split) > 1:
              output = []
              for i, piece in enumerate(split):
                  if i > 0:
                      output.append(special_token)
                  output += self.encode(piece)
              return output
      if self.add_prefix_space and not isinstance(
          self, HuggingFaceBpeHelper
      ):
          text = f" {text}"
      return self.helper_encode(text)
  
imports: from parlai.core.params import ParlaiParser
  from abc import ABC, abstractmethod
  from functools import lru_cache
  import json
  import os
  import random
  import re
  from typing import Dict, List, Optional, Set, Tuple
  from typing_extensions import final
  from parlai.core.build_data import download, make_dir
  from parlai.core.opt import Opt
  from parlai.utils.misc import warn_once
  from parlai.utils.typing import TShared
  from parlai.utils.io import PathManager
  import parlai.utils.logging
label_desc: Use image node_id 4 for calling the BPEHelper obj's underlying member method code with example usage: obj.encode(text) and returns: self, output