code: def token_pre(tokenizer, q):
      st = " ".join(q)
      s = tokenizer.tokenize(st)
  
      out = [0]
      cur = 0
      expect = ""
      first = True
      for i, w in enumerate(s):
          if len(expect) == 0:
              cur += 1
              expect = q[cur - 1].lower()
              first = True
          if w.startswith("##"):
              out.append(-1)
              expect = expect[len(w) - 2 :]
          elif first:
              out.append(cur)
              expect = expect[len(w) :]
              first = False
          else:
              expect = expect[len(w) :]
      out.append(cur + 1)
      # assert cur == len(q)-1, "%s %s \n%s\n%s"%(len(q), cur, q, s)
      if cur != len(q):
          print("error")
          return [0] * (len(q) + 2), [0] * (len(q) + 2)
      return tokenizer.encode(st, add_special_tokens=True), out
  
imports: import torchtext
  import torch
label_desc: Use image node_id 1 for calling a global function with example usage: token_pre(tokenizer, q) and returns: tokenizer,  out, unknown,  unknown