class_docstr: Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).
  
  Args:
      do_lower_case (`bool`, *optional*, defaults to `True`):
          Whether or not to lowercase the input when tokenizing.
      never_split (`Iterable`, *optional*):
          Collection of tokens which will never be split during tokenization. Only has an effect when
          `do_basic_tokenize=True`
      tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
          Whether or not to tokenize Chinese characters.
  
          This should likely be deactivated for Japanese (see this
          [issue](https://github.com/huggingface/transformers/issues/328)).
      strip_accents (`bool`, *optional*):
          Whether or not to strip all accents. If this option is not specified, then it will be determined by the
          value for `lowercase` (as in the original BERT).
class_docstr_tok: ["Constructs","a","BasicTokenizer","that","will","run","basic","tokenization","(","punctuation","splitting",",","lower","casing",",","etc.",")",".","Args",":","do_lower_case","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","Whether","or","not","to","lowercase","the","input","when","tokenizing",".","never_split","(","`","Iterable","`",",","*","optional","*",")",":","Collection","of","tokens","which","will","never","be","split","during","tokenization",".","Only","has","an","effect","when","`","do_basic_tokenize=True","`","tokenize_chinese_chars","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","Whether","or","not","to","tokenize","Chinese","characters",".","This","should","likely","be","deactivated","for","Japanese","(","see","this","[","issue","]","(","https",":","\/\/github.com\/huggingface\/transformers\/issues\/328",")",")",".","strip_accents","(","`","bool","`",",","*","optional","*",")",":","Whether","or","not","to","strip","all","accents",".","If","this","option","is","not","specified",",","then","it","will","be","determined","by","the","value","for","`","lowercase","`","(","as","in","the","original","BERT",")","."]
docstr: Basic Tokenization of a piece of text. Split on "white spaces" only, for sub-word tokenization, see
  WordPieceTokenizer.
  
  Args:
      never_split (`List[str]`, *optional*)
          Kept for backward compatibility purposes. Now implemented directly at the base class level (see
          [`PreTrainedTokenizer.tokenize`]) List of token not to split.
docstr_tok: ["Basic","Tokenization","of","a","piece","of","text",".","Split","on","``","white","spaces","''","only",",","for","sub-word","tokenization",",","see","WordPieceTokenizer",".","Args",":","never_split","(","`","List","[","str","]","`",",","*","optional","*",")","Kept","for","backward","compatibility","purposes",".","Now","implemented","directly","at","the","base","class","level","(","see","[","`","PreTrainedTokenizer.tokenize","`","]",")","List","of","token","not","to","split","."]
code: def tokenize(self, text, never_split=None):
      """
      Basic Tokenization of a piece of text. Split on "white spaces" only, for sub-word tokenization, see
      WordPieceTokenizer.
  
      Args:
          never_split (`List[str]`, *optional*)
              Kept for backward compatibility purposes. Now implemented directly at the base class level (see
              [`PreTrainedTokenizer.tokenize`]) List of token not to split.
      """
      # union() returns a new set by concatenating the two sets.
      never_split = (
          self.never_split.union(set(never_split))
          if never_split
          else self.never_split
      )
      text = self._clean_text(text)
  
      # This was added on November 1st, 2018 for the multilingual and Chinese
      # models. This is also applied to the English models now, but it doesn't
      # matter since the English models were not trained on any Chinese data
      # and generally don't have any Chinese data in them (there are Chinese
      # characters in the vocabulary because Wikipedia does have some Chinese
      # words in the English Wikipedia.).
      if self.tokenize_chinese_chars:
          text = self._tokenize_chinese_chars(text)
      orig_tokens = whitespace_tokenize(text)
      split_tokens = []
      for token in orig_tokens:
          if token not in never_split:
              if self.do_lower_case:
                  token = token.lower()
                  if self.strip_accents is not False:
                      token = self._run_strip_accents(token)
              elif self.strip_accents:
                  token = self._run_strip_accents(token)
          split_tokens.extend(
              self._run_split_on_punc(token, never_split)
          )
  
      output_tokens = whitespace_tokenize(" ".join(split_tokens))
      return output_tokens
  
imports: import json
  import os
  import re
  import unicodedata
  from typing import Optional, Tuple
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
label_desc: Use image node_id 2 for calling the BasicTokenizer obj's underlying member method code with example usage: obj.tokenize(text, never_split) and returns: output_tokens