class_docstr: Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).
  
  Args:
      do_lower_case (`bool`, *optional*, defaults to `True`):
          Whether or not to lowercase the input when tokenizing.
      never_split (`Iterable`, *optional*):
          Collection of tokens which will never be split during tokenization. Only has an effect when
          `do_basic_tokenize=True`
      tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
          Whether or not to tokenize Chinese characters.
  
          This should likely be deactivated for Japanese (see this
          [issue](https://github.com/huggingface/transformers/issues/328)).
      strip_accents (`bool`, *optional*):
          Whether or not to strip all accents. If this option is not specified, then it will be determined by the
          value for `lowercase` (as in the original BERT).
      do_split_on_punc (`bool`, *optional*, defaults to `True`):
          In some instances we want to skip the basic punctuation splitting so that later tokenization can capture
          the full context of the words, such as contractions.
class_docstr_tok: ["Constructs","a","BasicTokenizer","that","will","run","basic","tokenization","(","punctuation","splitting",",","lower","casing",",","etc.",")",".","Args",":","do_lower_case","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","Whether","or","not","to","lowercase","the","input","when","tokenizing",".","never_split","(","`","Iterable","`",",","*","optional","*",")",":","Collection","of","tokens","which","will","never","be","split","during","tokenization",".","Only","has","an","effect","when","`","do_basic_tokenize=True","`","tokenize_chinese_chars","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","Whether","or","not","to","tokenize","Chinese","characters",".","This","should","likely","be","deactivated","for","Japanese","(","see","this","[","issue","]","(","https",":","\/\/github.com\/huggingface\/transformers\/issues\/328",")",")",".","strip_accents","(","`","bool","`",",","*","optional","*",")",":","Whether","or","not","to","strip","all","accents",".","If","this","option","is","not","specified",",","then","it","will","be","determined","by","the","value","for","`","lowercase","`","(","as","in","the","original","BERT",")",".","do_split_on_punc","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","In","some","instances","we","want","to","skip","the","basic","punctuation","splitting","so","that","later","tokenization","can","capture","the","full","context","of","the","words",",","such","as","contractions","."]
docstr: Splits punctuation on a piece of text.
docstr_tok: ["Splits","punctuation","on","a","piece","of","text","."]
code: def _run_split_on_punc(self, text, never_split=None):
      """Splits punctuation on a piece of text."""
      if not self.do_split_on_punc or (
          never_split is not None and text in never_split
      ):
          return [text]
      chars = list(text)
      i = 0
      start_new_word = True
      output = []
      while i < len(chars):
          char = chars[i]
          if _is_punctuation(char):
              output.append([char])
              start_new_word = True
          else:
              if start_new_word:
                  output.append([])
              start_new_word = False
              output[-1].append(char)
          i += 1
  
      return ["".join(x) for x in output]
  
imports: import json
  import os
  import unicodedata
  from functools import lru_cache
  from typing import List, Optional, Tuple
  import regex
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
label_desc: Use image node_id 4 for calling the BasicTokenizer obj's underlying member method code with example usage: obj._run_split_on_punc(text, never_split) and returns: unknown, list