code: def tokenize_en(line):
      line = line.strip()
      line = " " + line + " "
      # remove ASCII junk
      line = re.sub(r"\s+", " ", line)
      line = re.sub(r"[\x00-\x1F]", "", line)
      # fix whitespaces
      line = re.sub(r"\ +", " ", line)
      line = re.sub("^ ", "", line)
      line = re.sub(" $", "", line)
      # separate other special characters
      line = re.sub(
          r"([^\s\.\'\`\,\-\w]|[_" + NUMERICS + "])", r" \g<1> ", line
      )
      line = re.sub(r"(\w)\-(?=\w)", r"\g<1> @-@ ", line)
  
      # multidots stay together
      line = re.sub(r"\.([\.]+)", r" DOTMULTI\g<1>", line)
      while re.search(r"DOTMULTI\.", line):
          line = re.sub(
              r"DOTMULTI\.([^\.])", r"DOTDOTMULTI \g<1>", line
          )
          line = re.sub(r"DOTMULTI\.", r"DOTDOTMULTI", line)
  
      # separate out "," except if within numbers (5,300)
      line = re.sub(r"([\D])[,]", r"\g<1> , ", line)
      line = re.sub(r"[,]([\D])", r" , \g<1>", line)
  
      # separate "," after a number if it's the end of sentence
      line = re.sub(r"(\d)[,]$", r"\g<1> ,", line)
  
      # split contractions right
      line = re.sub(r"([\W\d])[\']([\W\d])", r"\g<1> \' \g<2>", line)
      line = re.sub(r"(\W)[\']([\w\D])", r"\g<1> \' \g<2>", line)
      line = re.sub(r"([\w\D])[\']([\W\d])", r"\g<1> \' \g<2>", line)
      line = re.sub(r"([\w\D])[\']([\w\D])", r"\g<1> \'\g<2>", line)
      # special case for "1990's"
      line = re.sub(r"([\W\d])[\']([s])", r"\g<1> \'\g<2>", line)
  
      # apply nonbreaking prefixes
      words = line.split()
      line = ""
      for i in range(len(words)):
          word = words[i]
          match = re.search(r"^(\S+)\.$", word)
          if match:
              pre = match.group(1)
              if i == len(words) - 1:
                  """split last words independently as they are unlikely
                  to be non-breaking prefixes"""
                  word = pre + " ."
              else:
                  word = pre + " ."
  
          word += " "
          line += word
  
      # clean up extraneous spaces
      line = re.sub(" +", " ", line)
      line = re.sub("^ ", "", line)
      line = re.sub(" $", "", line)
  
      # .' at end of sentence is missed
      line = re.sub(r"\.\' ?$", " . ' ", line)
  
      # restore multi-dots
      while re.search("DOTDOTMULTI", line):
          line = re.sub("DOTDOTMULTI", "DOTMULTI.", line)
  
      line = re.sub("DOTMULTI", ".", line)
  
      # escape special characters
      line = re.sub(r"\&", r"&amp;", line)
      line = re.sub(r"\|", r"&#124;", line)
      line = re.sub(r"\<", r"&lt;", line)
      line = re.sub(r"\>", r"&gt;", line)
      line = re.sub(r"\'", r"&apos;", line)
      line = re.sub(r"\"", r"&quot;", line)
      line = re.sub(r"\[", r"&#91;", line)
      line = re.sub(r"\]", r"&#93;", line)
  
      # ensure final line breaks
      # if line[-1] is not '\n':
      #     line += '\n'
  
      return line
  
imports: import re
  import sys
  import unicodedata
  from collections import defaultdict
label_desc: Use image node_id 2 for calling a global function with example usage: tokenize_en(line) and returns: line