class_docstr: Tokenizer for gpt-2 and RoBERTa.
class_docstr_tok: ["Tokenizer","for","gpt-2","and","RoBERTa","."]
code: def tokenize(self, input_str: str) -> List[Token]:
      if self.lowercase:
          bpe_ids = self.bpe.encode(input_str.lower())
      else:
          bpe_ids = self.bpe.encode(input_str)
      char_tokens = [
          self.bpe.decoder[id].lstrip("\u0120") for id in bpe_ids
      ]
      # fix for incorrect decoding of utf-8 chars
      for i, char_token in enumerate(char_tokens):
          try:
              char_tokens[i] = bytearray(
                  [self.bpe.byte_decoder[char] for char in char_token]
              ).decode("utf-8")
          # handles BPE breaking a single multi-byte char into pieces
          except UnicodeDecodeError:
              continue
      lengths = [len(token) for token in char_tokens]
      tokens = []
      end = 0
      for length, id, char_token in zip(lengths, bpe_ids, char_tokens):
          start = input_str.find(char_token, end)
          end = start + length
          tokens.append(Token(str(id), start, end))
          # handles bad start/end indices cascading to subsequent tokens.
          if len(tokens) > 1 and end < tokens[-2].end:
              end = tokens[-2].end
      return [token for token in tokens if token.value]
  
imports: import copy
  import json
  import re
  from collections import OrderedDict
  from typing import List, NamedTuple, Optional, Union
  import torch
  from fairseq.data.encoders.gpt2_bpe import get_encoder
  from fairseq.data.encoders.gpt2_bpe_utils import Encoder
  from pytext.config import ConfigBase
  from pytext.config.component import Component, ComponentType, create_component
  from pytext.data.pickleable_gpt2bpe_encoder import PickleableGPT2BPEEncoder
  from pytext.torchscript.tokenizer import ScriptDoNothingTokenizer, ScriptWordTokenizer
  from pytext.utils.file_io import PathManager
  from pytext.utils.usage import log_class_usage
  from transformers.tokenization_bert import BasicTokenizer, WordpieceTokenizer
label_desc: Use image node_id 3 for calling the GPT2BPETokenizer obj's underlying member method code with example usage: obj.tokenize(input_str) and returns: unknown