class_docstr: Runs WordPiece tokenization.
class_docstr_tok: ["Runs","WordPiece","tokenization","."]
docstr: Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform
  tokenization using the given vocabulary.
  
  For example, `input = "unaffable"` wil return as output `["un", "##aff", "##able"]`.
  
  Args:
      text: A single token or whitespace separated tokens. This should have
          already been passed through *BasicTokenizer*.
  
  Returns:
      A list of wordpiece tokens.
docstr_tok: ["Tokenizes","a","piece","of","text","into","its","word","pieces",".","This","uses","a","greedy","longest-match-first","algorithm","to","perform","tokenization","using","the","given","vocabulary",".","For","example",",","`","input","=","``","unaffable","''","`","wil","return","as","output","`","[","``","un","''",",","``","#","#","aff","''",",","``","#","#","able","''","]","`",".","Args",":","text",":","A","single","token","or","whitespace","separated","tokens",".","This","should","have","already","been","passed","through","*","BasicTokenizer","*",".","Returns",":","A","list","of","wordpiece","tokens","."]
code: def tokenize(self, text):
      """
      Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform
      tokenization using the given vocabulary.
  
      For example, `input = "unaffable"` wil return as output `["un", "##aff", "##able"]`.
  
      Args:
          text: A single token or whitespace separated tokens. This should have
              already been passed through *BasicTokenizer*.
  
      Returns:
          A list of wordpiece tokens.
      """
  
      output_tokens = []
      for token in whitespace_tokenize(text):
          chars = list(token)
          if len(chars) > self.max_input_chars_per_word:
              output_tokens.append(self.unk_token)
              continue
  
          is_bad = False
          start = 0
          sub_tokens = []
          while start < len(chars):
              end = len(chars)
              cur_substr = None
              while start < end:
                  substr = "".join(chars[start:end])
                  if start > 0:
                      substr = "##" + substr
                  if substr in self.vocab:
                      cur_substr = substr
                      break
                  end -= 1
              if cur_substr is None:
                  is_bad = True
                  break
              sub_tokens.append(cur_substr)
              start = end
  
          if is_bad:
              output_tokens.append(self.unk_token)
          else:
              output_tokens.extend(sub_tokens)
      return output_tokens
  
imports: import collections
  import os
  import unicodedata
  from typing import List, Optional, Tuple
  from ....tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ....utils import logging
label_desc: Use image node_id 2 for calling the WordpieceTokenizer obj's underlying member method code with example usage: obj.tokenize(text) and returns: output_tokens