class_docstr: Creates SQuAD dataset for Question Answering.
  Args:
      data_file (str): train.*.json eval.*.json or test.*.json.
      tokenizer (obj): Tokenizer object, e.g. AutoTokenizer.
      version_2_with_negative (bool): True if training should allow
          unanswerable questions.
      doc_stride (int): When splitting up a long document into chunks,
          how much stride to take between chunks.
      max_query_length (iny): All training files which have a duration less
          than min_duration are dropped. Can't be used if the `utt2dur` file
          does not exist. Defaults to None.
      max_seq_length (int): All training files which have a duration more
          than max_duration are dropped. Can't be used if the `utt2dur` file
          does not exist. Defaults to None.
      num_samples: number of samples you want to use for the dataset.
          If -1, use all dataset. Useful for testing.
      mode (str): Use TRAINING_MODE/EVALUATION_MODE/INFERENCE_MODE to define between
          training, evaluation and inference dataset.
      use_cache (bool): Caches preprocessed data for future usage
class_docstr_tok: ["Creates","SQuAD","dataset","for","Question","Answering",".","Args",":","data_file","(","str",")",":","train",".","*",".json","eval",".","*",".json","or","test.","*",".json",".","tokenizer","(","obj",")",":","Tokenizer","object",",","e.g",".","AutoTokenizer",".","version_2_with_negative","(","bool",")",":","True","if","training","should","allow","unanswerable","questions",".","doc_stride","(","int",")",":","When","splitting","up","a","long","document","into","chunks",",","how","much","stride","to","take","between","chunks",".","max_query_length","(","iny",")",":","All","training","files","which","have","a","duration","less","than","min_duration","are","dropped",".","Ca","n't","be","used","if","the","`","utt2dur","`","file","does","not","exist",".","Defaults","to","None",".","max_seq_length","(","int",")",":","All","training","files","which","have","a","duration","more","than","max_duration","are","dropped",".","Ca","n't","be","used","if","the","`","utt2dur","`","file","does","not","exist",".","Defaults","to","None",".","num_samples",":","number","of","samples","you","want","to","use","for","the","dataset",".","If","-1",",","use","all","dataset",".","Useful","for","testing",".","mode","(","str",")",":","Use","TRAINING_MODE\/EVALUATION_MODE\/INFERENCE_MODE","to","define","between","training",",","evaluation","and","inference","dataset",".","use_cache","(","bool",")",":","Caches","preprocessed","data","for","future","usage"]
code: def get_predictions(
      self,
      unique_ids: List[int],
      start_logits: List[List[float]],
      end_logits: List[List[float]],
      n_best_size: int,
      max_answer_length: int,
      do_lower_case: bool,
      version_2_with_negative: bool,
      null_score_diff_threshold: float,
  ):
      example_index_to_features = collections.defaultdict(list)
  
      unique_id_to_pos = {}
      for index, unique_id in enumerate(unique_ids):
          unique_id_to_pos[unique_id] = index
  
      for feature in self.features:
          example_index_to_features[feature.example_index].append(
              feature
          )
  
      _PrelimPrediction = collections.namedtuple(
          "PrelimPrediction",
          [
              "feature_index",
              "start_index",
              "end_index",
              "start_logit",
              "end_logit",
          ],
      )
  
      all_predictions = collections.OrderedDict()
      all_nbest_json = collections.OrderedDict()
      scores_diff_json = collections.OrderedDict()
      for example_index, example in enumerate(self.examples):
          # finish this loop if we went through all batch examples
          if example_index >= len(unique_ids):
              break
  
          features = example_index_to_features[example_index]
  
          (
              doc_tokens,
              _,
              _,
              _,
              _,
          ) = SquadDataset.get_doc_tokens_and_offset_from_context_id(
              example.context_id,
              example.start_position_character,
              example.is_impossible,
              example.answer_text,
              self.processor.doc_id_to_context_text,
          )
          prelim_predictions = []
          # keep track of the minimum score of null start+end of position 0
          # large and positive
          score_null = 1000000
          # the paragraph slice with min null score
          min_null_feature_index = 0
          # start logit at the slice with min null score
          null_start_logit = 0
          # end logit at the slice with min null score
          null_end_logit = 0
          for feature_index, feature in enumerate(features):
              pos = unique_id_to_pos[feature.unique_id]
              start_indexes = get_best_indexes(
                  start_logits[pos], n_best_size
              )
              end_indexes = get_best_indexes(
                  end_logits[pos], n_best_size
              )
              # if we could have irrelevant answers,
              # get the min score of irrelevant
              if version_2_with_negative:
                  feature_null_score = (
                      start_logits[pos][0] + end_logits[pos][0]
                  )
                  if feature_null_score < score_null:
                      score_null = feature_null_score
                      min_null_feature_index = feature_index
                      null_start_logit = start_logits[pos][0]
                      null_end_logit = end_logits[pos][0]
              for start_index in start_indexes:
                  for end_index in end_indexes:
                      # We could hypothetically create invalid predictions,
                      # e.g., predict that the start of the span is in the
                      # question. We throw out all invalid predictions.
                      if start_index >= len(feature.tokens):
                          continue
                      if end_index >= len(feature.tokens):
                          continue
                      if start_index not in feature.token_to_orig_map:
                          continue
                      if end_index not in feature.token_to_orig_map:
                          continue
                      if not feature.token_is_max_context.get(
                          start_index, False
                      ):
                          continue
                      if end_index < start_index:
                          continue
                      length = end_index - start_index + 1
                      if length > max_answer_length:
                          continue
                      prelim_predictions.append(
                          _PrelimPrediction(
                              feature_index=feature_index,
                              start_index=start_index,
                              end_index=end_index,
                              start_logit=start_logits[pos][
                                  start_index
                              ],
                              end_logit=end_logits[pos][end_index],
                          )
                      )
  
          if version_2_with_negative:
              prelim_predictions.append(
                  _PrelimPrediction(
                      feature_index=min_null_feature_index,
                      start_index=0,
                      end_index=0,
                      start_logit=null_start_logit,
                      end_logit=null_end_logit,
                  )
              )
          prelim_predictions = sorted(
              prelim_predictions,
              key=lambda x: (x.start_logit + x.end_logit),
              reverse=True,
          )
  
          _NbestPrediction = collections.namedtuple(
              "NbestPrediction", ["text", "start_logit", "end_logit"]
          )
  
          seen_predictions = {}
          nbest = []
          for pred in prelim_predictions:
              if len(nbest) >= n_best_size:
                  break
              feature = features[pred.feature_index]
              if pred.start_index > 0:  # this is a non-null prediction
                  tok_tokens = feature.tokens[
                      pred.start_index : (pred.end_index + 1)
                  ]
                  orig_doc_start = feature.token_to_orig_map[
                      pred.start_index
                  ]
                  orig_doc_end = feature.token_to_orig_map[
                      pred.end_index
                  ]
                  orig_tokens = doc_tokens[
                      orig_doc_start : (orig_doc_end + 1)
                  ]
                  tok_text = " ".join(tok_tokens)
  
                  # De-tokenize WordPieces that have been split off.
                  tok_text = tok_text.replace(" ##", "")
                  tok_text = tok_text.replace("##", "")
  
                  # Clean whitespace
                  tok_text = tok_text.strip()
                  tok_text = " ".join(tok_text.split())
                  orig_text = " ".join(orig_tokens)
  
                  final_text = get_final_text(
                      tok_text, orig_text, do_lower_case
                  )
                  if final_text in seen_predictions:
                      continue
  
                  seen_predictions[final_text] = True
              else:
                  final_text = ""
                  seen_predictions[final_text] = True
  
              nbest.append(
                  _NbestPrediction(
                      text=final_text,
                      start_logit=pred.start_logit,
                      end_logit=pred.end_logit,
                  )
              )
  
          # if we didn't include the empty option in the n-best, include it
          if version_2_with_negative:
              if "" not in seen_predictions:
                  nbest.append(
                      _NbestPrediction(
                          text="",
                          start_logit=null_start_logit,
                          end_logit=null_end_logit,
                      )
                  )
  
              # In very rare edge cases we could only
              # have single null pred. We just create a nonce prediction
              # in this case to avoid failure.
              if len(nbest) == 1:
                  nbest.insert(
                      0,
                      _NbestPrediction(
                          text="empty", start_logit=0.0, end_logit=0.0
                      ),
                  )
  
          # In very rare edge cases we could have no valid predictions. So we
          # just create a nonce prediction in this case to avoid failure.
          if not nbest:
              nbest.append(
                  _NbestPrediction(
                      text="empty", start_logit=0.0, end_logit=0.0
                  )
              )
  
          assert len(nbest) >= 1
  
          total_scores = []
          best_non_null_entry = None
          for entry in nbest:
              total_scores.append(entry.start_logit + entry.end_logit)
              if not best_non_null_entry:
                  if entry.text:
                      best_non_null_entry = entry
  
          probs = _compute_softmax(total_scores)
  
          nbest_json = []
          for i, entry in enumerate(nbest):
              output = collections.OrderedDict()
              output["question"] = example.question_text
              output["text"] = entry.text
              output["probability"] = probs[i]
              output["start_logit"] = (
                  entry.start_logit
                  if (
                      isinstance(entry.start_logit, float)
                      or isinstance(entry.start_logit, int)
                  )
                  else list(entry.start_logit)
              )
              output["end_logit"] = (
                  entry.end_logit
                  if (
                      isinstance(entry.end_logit, float)
                      or isinstance(entry.end_logit, int)
                  )
                  else list(entry.end_logit)
              )
              nbest_json.append(output)
  
          assert len(nbest_json) >= 1
          if not version_2_with_negative:
              all_predictions[example.qas_id] = nbest_json[0]["text"]
          else:
              # predict "" iff the null score -
              # the score of best non-null > threshold
              score_diff = (
                  score_null
                  - best_non_null_entry.start_logit
                  - (best_non_null_entry.end_logit)
              )
              scores_diff_json[example.qas_id] = score_diff
              if score_diff > null_score_diff_threshold:
                  all_predictions[example.qas_id] = ""
              else:
                  all_predictions[
                      example.qas_id
                  ] = best_non_null_entry.text
          all_nbest_json[example.qas_id] = nbest_json
  
      return all_predictions, all_nbest_json, scores_diff_json
  
imports: import collections
  import os
  import pickle
  from functools import lru_cache
  from typing import Dict, List, Optional
  import numpy
  import psutil
  import torch
  from tqdm import trange
  from nemo.collections.common.parts.utils import _compute_softmax
  from nemo.collections.nlp.data.data_utils import is_whitespace
  from nemo.collections.nlp.data.question_answering_squad.qa_squad_processing import EVALUATION_MODE, INFERENCE_MODE, TRAINING_MODE, SquadProcessor, _improve_answer_span, apply_no_ans_threshold, exact_match_score, f1_score, find_all_best_thresh, get_best_indexes, get_final_text, make_eval_dict, merge_eval, normalize_answer
  from nemo.core.classes import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 11 for calling the SquadDataset obj's underlying member method code with example usage: obj.get_predictions(unique_ids, start_logits, end_logits, n_best_size, max_answer_length, do_lower_case, version_2_with_negative, null_score_diff_threshold) and returns: all_predictions,  all_nbest_json,  scores_diff_json