class_docstr: A `Tokenizer` which keeps runs of (unicode) letters and runs of digits together, while
  every other non-whitespace character becomes a separate word.
  
  Registered as a `Tokenizer` with name "letters_digits".
class_docstr_tok: ["A","`","Tokenizer","`","which","keeps","runs","of","(","unicode",")","letters","and","runs","of","digits","together",",","while","every","other","non-whitespace","character","becomes","a","separate","word",".","Registered","as","a","`","Tokenizer","`","with","name","``","letters_digits","''","."]
code: def tokenize(self, text: str) -> List[Token]:
      # We use the [^\W\d_] pattern as a trick to match unicode letters
      tokens = [
          Token(m.group(), idx=m.start())
          for m in re.finditer(r"[^\W\d_]+|\d+|\S", text)
      ]
      return tokens
  
imports: import re
  from typing import List
  from allennlp.data.tokenizers.token_class import Token
  from allennlp.data.tokenizers.tokenizer import Tokenizer
label_desc: Use image node_id 1 for calling the LettersDigitsTokenizer obj's underlying member method code with example usage: obj.tokenize(text) and returns: tokens