code: def produce_examples(
      shard_ids,
      wikis_dir,
      refs_dir,
      urls_dir,
      vocab_path,
      out_filepaths,
  ):
      """Produce examples from shard_ids to out_filepaths."""
      # * Join the Wikipedia articles with their references
      # * Run Tf-idf to sort reference paragraphs
      # * Encode the Wikipedia and reference text with the vocabulary
      # * Write out TFRecords of tensorflow.Example
      tf.logging.info(
          "Processing %d input shards into %d output files.",
          len(shard_ids),
          len(out_filepaths),
      )
  
      vocab = text_encoder.SubwordTextEncoder(vocab_path)
      eot_ids = vocab.encode(EOT)
  
      def example_generator():
          """Generate Example dicts."""
          stats = dict(
              total_original_wikis=0,
              total_original_refs=0,
              total_found_refs=0,
              ref_lengths=[],
              wiki_original_refs=[],
              wiki_found_refs=[],
              wikis_skipped_no_refs=0,
              wikis_skipped_short_lead=0,
              num_wikis_written=0,
          )
          ref_files_by_shard = _references_files_by_shard(refs_dir)
          for shard_id in shard_ids:
              tf.logging.info("Processing shard %d", shard_id)
              wiki_urls = _wiki_urls_for_shard(shard_id, urls_dir)
              tf.logging.info("Loaded wiki URLs for shard")
              refs_content = _references_content(
                  ref_files_by_shard[shard_id]
              )
              tf.logging.info("Loaded reference content for shard")
              for i, wiki in enumerate(
                  _wiki_articles(shard_id, wikis_dir)
              ):
                  if not i % 1000:
                      tf.logging.info(
                          "Processing wiki index %d for shard %d",
                          i,
                          shard_id,
                      )
                  stats["total_original_wikis"] += 1
  
                  # Get reference content
                  wiki_ref_content = []
                  ref_urls = wiki_urls[wiki.url]["refs"]
                  stats["total_original_refs"] += len(ref_urls)
                  stats_wiki_original_refs = len(ref_urls)
                  stats_wiki_found_refs = 0
                  for ref_url in ref_urls:
                      ref_content = refs_content.get(ref_url)
                      if not ref_content:
                          continue
                      stats["total_found_refs"] += 1
                      stats["ref_lengths"].append(len(ref_content))
                      stats_wiki_found_refs += 1
                      wiki_ref_content.append(ref_content)
  
                  stats["wiki_original_refs"].append(
                      stats_wiki_original_refs
                  )
                  stats["wiki_found_refs"].append(stats_wiki_found_refs)
                  if (
                      not wiki_ref_content
                      or len(wiki_ref_content) < _MIN_REFS
                  ):
                      # No/few refs were found
                      stats["wikis_skipped_no_refs"] += 1
                      continue
  
                  # Rank reference paragraphs with TFIDF
                  wiki_title = _normalize_text(wiki.title)
                  ranked_paragraphs = rank_reference_paragraphs(
                      wiki_title, wiki_ref_content
                  )
  
                  # Construct inputs from Wiki title and references
                  inputs = []
                  inputs.extend(vocab.encode(wiki_title))
                  inputs.extend(eot_ids)
                  for paragraph in ranked_paragraphs:
                      if len(inputs) >= 1e6:
                          break
                      paragraph += " "
                      inputs.extend(vocab.encode(paragraph))
  
                  # Construct targets from article sections
                  targets, section_boundaries = _encode_wiki_sections(
                      wiki.sections, vocab
                  )
  
                  # Skip if lead section is too short
                  if (
                      not section_boundaries
                      or section_boundaries[0] < _MIN_LEADSECTION_TOKENS
                  ):
                      stats["wikis_skipped_short_lead"] += 1
                      continue
  
                  inputs.append(text_encoder.EOS_ID)
                  targets.append(text_encoder.EOS_ID)
  
                  stats["num_wikis_written"] += 1
                  yield {
                      "inputs": inputs,
                      "targets": targets,
                      "section_boundaries": section_boundaries,
                  }
  
          tf.logging.info(
              "Total: %d, Skipped: %d",
              stats["num_wikis_written"],
              stats["total_original_wikis"]
              - stats["num_wikis_written"],
          )
          tf.logging.info(
              "Total refs: %d, Skipped refs: %d",
              stats["total_found_refs"],
              stats["total_original_refs"] - stats["total_found_refs"],
          )
          stats_fname = os.path.join(
              os.path.split(out_filepaths[0])[0],
              "stats.%d.json" % shard_ids[0],
          )
          with tf.gfile.Open(stats_fname, "w") as f:
              f.write(json.dumps(stats))
  
      generator_utils.generate_files(example_generator(), out_filepaths)
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  import json
  import math
  import os
  import re
  import string
  import tempfile
  import six
  from tensor2tensor.data_generators import generator_utils
  from tensor2tensor.data_generators import problem
  from tensor2tensor.data_generators import text_encoder
  from tensor2tensor.data_generators import tokenizer
  from tensor2tensor.data_generators.wikisum import utils
  from tensor2tensor.layers import modalities
  from tensor2tensor.utils import metrics
  from tensor2tensor.utils import registry
  import tensorflow.compat.v1
label_desc: Use image node_id 13 for calling a global function with example usage: produce_examples(shard_ids, wikis_dir, refs_dir, urls_dir, vocab_path, out_filepaths) without return types