docstr: Modified from pyvi to preserve text and whitespace.
docstr_tok: ["Modified","from","pyvi","to","preserve","text","and","whitespace","."]
code: def pyvi_tokenize(self, text):
      """Modified from pyvi to preserve text and whitespace."""
      if len(text) == 0:
          return []
      elif text.isspace():
          return [text]
      segs = self.pyvi_sylabelize_with_ws(text)
      words = []
      preceding_ws = []
      for i, token in enumerate(segs):
          if not token.isspace():
              words.append(token)
              preceding_ws.append(
                  ""
                  if (i == 0 or not segs[i - 1].isspace())
                  else segs[i - 1]
              )
      labels = self.ViTokenizer.ViTokenizer.model.predict(
          [self.ViTokenizer.ViTokenizer.sent2features(words, False)]
      )
      token = words[0]
      tokens = []
      for i in range(1, len(labels[0])):
          if (
              labels[0][i] == "I_W"
              and words[i] not in string.punctuation
              and words[i - 1] not in string.punctuation
              and not words[i][0].isdigit()
              and not words[i - 1][0].isdigit()
              and not (
                  words[i][0].istitle()
                  and not words[i - 1][0].istitle()
              )
          ):
              token = token + preceding_ws[i] + words[i]
          else:
              tokens.append(token)
              token = words[i]
      tokens.append(token)
      return tokens
  
imports: import re
  import string
  from pathlib import Path
  from typing import Any, Dict, Union
  import srsly
  from ...None import util
  from ...language import BaseDefaults, Language
  from ...tokens import Doc
  from ...util import DummyTokenizer, load_config_from_str, registry
  from ...vocab import Vocab
  from .lex_attrs import LEX_ATTRS
  from .stop_words import STOP_WORDS
label_desc: Use image node_id 5 for calling the VietnameseTokenizer obj's underlying member method code with example usage: obj.pyvi_tokenize(text) and returns: tokens, list, list