class_docstr: A base tokenizer use a provided delimiter `sep` to split text.
  
  Parameters
  ----------
  sep: str, optional, default: ' '
      Separator string used to split text into tokens.
  
  pre_rules: List[Callable[[str], str]], optional
      List of callable lambda functions to apply on text before tokenization.
  
  stop_words: Union[List, str], optional
      List of stop-words to be ignored during tokenization,
      or key of built-in stop-word lists (e.g., english).
class_docstr_tok: ["A","base","tokenizer","use","a","provided","delimiter","`","sep","`","to","split","text",".","Parameters","--","--","--","--","--","sep",":","str",",","optional",",","default",":","'","'","Separator","string","used","to","split","text","into","tokens",".","pre_rules",":","List","[","Callable","[","[","str","]",",","str","]","]",",","optional","List","of","callable","lambda","functions","to","apply","on","text","before","tokenization",".","stop_words",":","Union","[","List",",","str","]",",","optional","List","of","stop-words","to","be","ignored","during","tokenization",",","or","key","of","built-in","stop-word","lists","(","e.g.",",","english",")","."]
code: def __init__(
      self,
      sep: str = " ",
      pre_rules: List[Callable[[str], str]] = None,
      stop_words: Union[List, str] = None,
  ):
      self.sep = sep
      self.pre_rules = (
          DEFAULT_PRE_RULES if pre_rules is None else pre_rules
      )
      self.stop_words = _validate_stopwords(stop_words)
  
imports: from typing import List, Dict, Callable, Union
  from collections import defaultdict, Counter, OrderedDict
  import string
  import pickle
  import re
  import numpy
  import scipy.sparse
  from .None import FeatureModality
  from .modality import fallback_feature
  from ..utils import normalize
label_desc: Use image node_id 1 to create a new BaseTokenizer object from inherited base classes: Tokenizer with example: obj = BaseTokenizer(sep, pre_rules, stop_words)