class_docstr: A base tokenizer use a provided delimiter `sep` to split text.
  
  Parameters
  ----------
  sep: str, optional, default: ' '
      Separator string used to split text into tokens.
  
  pre_rules: List[Callable[[str], str]], optional
      List of callable lambda functions to apply on text before tokenization.
  
  stop_words: Union[List, str], optional
      List of stop-words to be ignored during tokenization,
      or key of built-in stop-word lists (e.g., english).
class_docstr_tok: ["A","base","tokenizer","use","a","provided","delimiter","`","sep","`","to","split","text",".","Parameters","--","--","--","--","--","sep",":","str",",","optional",",","default",":","'","'","Separator","string","used","to","split","text","into","tokens",".","pre_rules",":","List","[","Callable","[","[","str","]",",","str","]","]",",","optional","List","of","callable","lambda","functions","to","apply","on","text","before","tokenization",".","stop_words",":","Union","[","List",",","str","]",",","optional","List","of","stop-words","to","be","ignored","during","tokenization",",","or","key","of","built-in","stop-word","lists","(","e.g.",",","english",")","."]
docstr: Splitting text into tokens.
  
  Parameters
  ----------
  t: str, required
      Input text to be tokenized.
  
  Returns
  -------
  tokens : ``List[str]``
docstr_tok: ["Splitting","text","into","tokens",".","Parameters","--","--","--","--","--","t",":","str",",","required","Input","text","to","be","tokenized",".","Returns","--","--","--","-","tokens",":","``","List","[","str","]","``"]
code: def tokenize(self, t: str) -> List[str]:
      """
      Splitting text into tokens.
  
      Parameters
      ----------
      t: str, required
          Input text to be tokenized.
  
      Returns
      -------
      tokens : ``List[str]``
      """
      for rule in self.pre_rules:
          t = rule(t)
      tokens = t.split(self.sep)
      if self.stop_words is not None:
          tokens = [tok for tok in tokens if tok not in self.stop_words]
      return tokens
  
imports: from typing import List, Dict, Callable, Union
  from collections import defaultdict, Counter, OrderedDict
  import string
  import pickle
  import re
  import numpy
  import scipy.sparse
  from .None import FeatureModality
  from .modality import fallback_feature
  from ..utils import normalize
label_desc: Use image node_id 2 for calling the BaseTokenizer obj's underlying member method code with example usage: obj.tokenize(t) and returns: tokens