class_docstr: Helper class for performing BPE subword tokenization.
  
  For technical details, please refer to https://arxiv.org/abs/1508.07909.
  This class just wraps around the official subword-nmt repository.
  
  This API expects the user to call tokenize() (encode) onto the training data,
  then call finalize() to learn the encodings, and then iterate over the data
  in a second pass, calling tokenize() again to get processed output.
class_docstr_tok: ["Helper","class","for","performing","BPE","subword","tokenization",".","For","technical","details",",","please","refer","to","https",":","\/\/arxiv.org\/abs\/1508.07909",".","This","class","just","wraps","around","the","official","subword-nmt","repository",".","This","API","expects","the","user","to","call","tokenize","(",")","(","encode",")","onto","the","training","data",",","then","call","finalize","(",")","to","learn","the","encodings",",","and","then","iterate","over","the","data","in","a","second","pass",",","calling","tokenize","(",")","again","to","get","processed","output","."]
docstr: Build the codecs.
  
  :param frequencies:
      dictionary of (token: frequency) pairs
  :param num_symbols:
      Number of BPE symbols. Recommend 30000-40000.  If <= 0, default
      30000 will be used.
  :param minfreq:
      Minimum frequency of a token before forced BPE decomposition. If <=
      0 will use subword-nmt default of 2.
  
  :return did_finalize:
      return whether codecs are finalized this call.
docstr_tok: ["Build","the","codecs",".",":","param","frequencies",":","dictionary","of","(","token",":","frequency",")","pairs",":","param","num_symbols",":","Number","of","BPE","symbols",".","Recommend","30000-40000",".","If","<","=","0",",","default","30000","will","be","used",".",":","param","minfreq",":","Minimum","frequency","of","a","token","before","forced","BPE","decomposition",".","If","<","=","0","will","use","subword-nmt","default","of","2.",":","return","did_finalize",":","return","whether","codecs","are","finalized","this","call","."]
code: def finalize(
      self,
      frequencies: Dict[str, int],
      num_symbols: int = 30000,
      minfreq: int = 2,
  ) -> bool:
      """
      Build the codecs.
  
      :param frequencies:
          dictionary of (token: frequency) pairs
      :param num_symbols:
          Number of BPE symbols. Recommend 30000-40000.  If <= 0, default
          30000 will be used.
      :param minfreq:
          Minimum frequency of a token before forced BPE decomposition. If <=
          0 will use subword-nmt default of 2.
  
      :return did_finalize:
          return whether codecs are finalized this call.
      """
      if hasattr(self, "bpe"):
          # we already finalized the codecs
          return False
  
      logging.debug(f"Saving bpe codecs to {self.codecs}")
  
      dictionary = (
          "{} {}".format(k, v) for k, v in frequencies.items()
      )
  
      if num_symbols <= 0:
          num_symbols = 30000
      if minfreq <= 0:
          minfreq = 2
  
      codec_dir, _ = os.path.split(self.codecs)
      PathManager.mkdirs(codec_dir)
      with PathManager.open(
          self.codecs, "w", encoding="utf-8"
      ) as outstream:
          learn_bpe.learn_bpe(
              dictionary,
              outstream,
              num_symbols=num_symbols,
              min_frequency=minfreq,
              is_dict=True,
          )
  
      self._load_from_codecs()
      return True
  
imports: from parlai.core.params import ParlaiParser
  from abc import ABC, abstractmethod
  from functools import lru_cache
  import json
  import os
  import random
  import re
  from typing import Dict, List, Optional, Set, Tuple
  from typing_extensions import final
  from parlai.core.build_data import download, make_dir
  from parlai.core.opt import Opt
  from parlai.utils.misc import warn_once
  from parlai.utils.typing import TShared
  from parlai.utils.io import PathManager
  import parlai.utils.logging
label_desc: Use image node_id 5 for calling the SubwordBPEHelper obj's underlying member method code with example usage: obj.finalize(frequencies, num_symbols, minfreq) and returns: True, False