class_docstr: Word piece tokenizer for BERT models.
class_docstr_tok: ["Word","piece","tokenizer","for","BERT","models","."]
code: def tokenize(self, input_str: str) -> List[Token]:
      tokens = []
      for token in self.basic_tokenizer.tokenize(input_str):
          start = token.start
          for sub_token in self.wordpiece_tokenizer.tokenize(
              token.value
          ):
              piece_len = (
                  len(sub_token)
                  if not sub_token.startswith("##")
                  else (len(sub_token) - 2)  # account for ##
              )
              if sub_token == "[UNK]":
                  # this fixes the bug wherein piece_len = 5 for all [UNK]
                  piece_len = len(token.value)
              end = start + piece_len
              tokens.append(Token(sub_token, start, end))
              start = end
      return [token for token in tokens if token.value]
  
imports: import copy
  import json
  import re
  from collections import OrderedDict
  from typing import List, NamedTuple, Optional, Union
  import torch
  from fairseq.data.encoders.gpt2_bpe import get_encoder
  from fairseq.data.encoders.gpt2_bpe_utils import Encoder
  from pytext.config import ConfigBase
  from pytext.config.component import Component, ComponentType, create_component
  from pytext.data.pickleable_gpt2bpe_encoder import PickleableGPT2BPEEncoder
  from pytext.torchscript.tokenizer import ScriptDoNothingTokenizer, ScriptWordTokenizer
  from pytext.utils.file_io import PathManager
  from pytext.utils.usage import log_class_usage
  from transformers.tokenization_bert import BasicTokenizer, WordpieceTokenizer
label_desc: Use image node_id 4 for calling the WordPieceTokenizer obj's underlying member method code with example usage: obj.tokenize(input_str) and returns: unknown