code: def reprocess(fle):
      # takes in a file of generate.py translation generate_output
      # returns a source dict and hypothesis dict, where keys are the ID num (as a string)
      # and values and the corresponding source and translation. There may be several translations
      # per source, so the values for hypothesis_dict are lists.
      # parses output of generate.py
  
      with open(fle, "r") as f:
          txt = f.read()
  
      """reprocess generate.py output"""
      p = re.compile(r"[STHP][-]\d+\s*")
      hp = re.compile(r"(\s*[-]?\d+[.]?\d+\s*)|(\s*(-inf)\s*)")
      source_dict = {}
      hypothesis_dict = {}
      score_dict = {}
      target_dict = {}
      pos_score_dict = {}
      lines = txt.split("\n")
  
      for line in lines:
          line += "\n"
          prefix = re.search(p, line)
          if prefix is not None:
              assert len(prefix.group()) > 2, "prefix id not found"
              _, j = prefix.span()
              id_num = prefix.group()[2:]
              id_num = int(id_num)
              line_type = prefix.group()[0]
              if line_type == "H":
                  h_txt = line[j:]
                  hypo = re.search(hp, h_txt)
                  assert (
                      hypo is not None
                  ), "regular expression failed to find the hypothesis scoring"
                  _, i = hypo.span()
                  score = hypo.group()
                  if id_num in hypothesis_dict:
                      hypothesis_dict[id_num].append(h_txt[i:])
                      score_dict[id_num].append(float(score))
                  else:
                      hypothesis_dict[id_num] = [h_txt[i:]]
                      score_dict[id_num] = [float(score)]
  
              elif line_type == "S":
                  source_dict[id_num] = line[j:]
              elif line_type == "T":
                  target_dict[id_num] = line[j:]
              elif line_type == "P":
                  pos_scores = (line[j:]).split()
                  pos_scores = [float(x) for x in pos_scores]
                  if id_num in pos_score_dict:
                      pos_score_dict[id_num].append(pos_scores)
                  else:
                      pos_score_dict[id_num] = [pos_scores]
  
      return (
          source_dict,
          hypothesis_dict,
          score_dict,
          target_dict,
          pos_score_dict,
      )
  
imports: import math
  import os
  import re
  import subprocess
  from contextlib import redirect_stdout
  from fairseq import options
  from fairseq_cli import eval_lm, preprocess
label_desc: Use image node_id 1 for calling a global function with example usage: reprocess(fle) and returns: source_dict,  hypothesis_dict,  score_dict,  target_dict,  pos_score_dict