code: def read_corpus(fname, tokens_only=False):
      with smart_open.open(fname, encoding="iso-8859-1") as f:
          for i, line in enumerate(f):
              tokens = gensim.utils.simple_preprocess(line)
              if tokens_only:
                  yield tokens
              else:
                  # For training data, add tags
                  yield gensim.models.doc2vec.TaggedDocument(
                      tokens, [i]
                  )
  
imports: import logging
  import os
  import gensim
  import smart_open
  import collections
  import random
label_desc: Use image node_id 1 for calling a global function with example usage: read_corpus(fname, tokens_only) without return types