class_docstr: Base class for Wikisum problems.
class_docstr_tok: ["Base","class","for","Wikisum","problems","."]
code: def generate_lines_for_vocab(
      self, wikis_dir, refs_dir, max_chars=10**7
  ):
      total_chars = 0
      ref_files_by_shard = _references_files_by_shard(refs_dir)
      for shard_id in range(cc_utils.NUM_SHARDS):
          # Wikipedia articles
          for wiki in _wiki_articles(shard_id, wikis_dir):
              yield _normalize_text(wiki.title) + EOT
              for section in wiki.sections:
                  yield _format_title(_normalize_text(section.title))
                  yield _normalize_text(section.text)
                  total_chars += len(section.title)
                  total_chars += len(section.text)
  
          # References
          for i, content in enumerate(
              six.itervalues(
                  _references_content(ref_files_by_shard[shard_id])
              )
          ):
              for line in content.split("\n"):
                  if line:
                      yield _normalize_text(line)
                      total_chars += len(line)
  
              # Make sure we use at least 1k references
              if i >= 1000 and total_chars >= max_chars:
                  break
  
          if total_chars >= max_chars:
              tf.logging.info(
                  "Seen enough chars: %d; finished.", max_chars
              )
              break
      tf.logging.info("Built vocabulary using %d chars", total_chars)
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  import json
  import math
  import os
  import re
  import string
  import tempfile
  import six
  from tensor2tensor.data_generators import generator_utils
  from tensor2tensor.data_generators import problem
  from tensor2tensor.data_generators import text_encoder
  from tensor2tensor.data_generators import tokenizer
  from tensor2tensor.data_generators.wikisum import utils
  from tensor2tensor.layers import modalities
  from tensor2tensor.utils import metrics
  from tensor2tensor.utils import registry
  import tensorflow.compat.v1
label_desc: Use image node_id 7 for calling the WikisumBase obj's underlying member method code with example usage: obj.generate_lines_for_vocab(wikis_dir, refs_dir, max_chars) without return types