class_docstr: Creates SQuAD dataset for Question Answering.
  Args:
      data_file (str): train.*.json eval.*.json or test.*.json.
      tokenizer (obj): Tokenizer object, e.g. AutoTokenizer.
      version_2_with_negative (bool): True if training should allow
          unanswerable questions.
      doc_stride (int): When splitting up a long document into chunks,
          how much stride to take between chunks.
      max_query_length (iny): All training files which have a duration less
          than min_duration are dropped. Can't be used if the `utt2dur` file
          does not exist. Defaults to None.
      max_seq_length (int): All training files which have a duration more
          than max_duration are dropped. Can't be used if the `utt2dur` file
          does not exist. Defaults to None.
      num_samples: number of samples you want to use for the dataset.
          If -1, use all dataset. Useful for testing.
      mode (str): Use TRAINING_MODE/EVALUATION_MODE/INFERENCE_MODE to define between
          training, evaluation and inference dataset.
      use_cache (bool): Caches preprocessed data for future usage
class_docstr_tok: ["Creates","SQuAD","dataset","for","Question","Answering",".","Args",":","data_file","(","str",")",":","train",".","*",".json","eval",".","*",".json","or","test.","*",".json",".","tokenizer","(","obj",")",":","Tokenizer","object",",","e.g",".","AutoTokenizer",".","version_2_with_negative","(","bool",")",":","True","if","training","should","allow","unanswerable","questions",".","doc_stride","(","int",")",":","When","splitting","up","a","long","document","into","chunks",",","how","much","stride","to","take","between","chunks",".","max_query_length","(","iny",")",":","All","training","files","which","have","a","duration","less","than","min_duration","are","dropped",".","Ca","n't","be","used","if","the","`","utt2dur","`","file","does","not","exist",".","Defaults","to","None",".","max_seq_length","(","int",")",":","All","training","files","which","have","a","duration","more","than","max_duration","are","dropped",".","Ca","n't","be","used","if","the","`","utt2dur","`","file","does","not","exist",".","Defaults","to","None",".","num_samples",":","number","of","samples","you","want","to","use","for","the","dataset",".","If","-1",",","use","all","dataset",".","Useful","for","testing",".","mode","(","str",")",":","Use","TRAINING_MODE\/EVALUATION_MODE\/INFERENCE_MODE","to","define","between","training",",","evaluation","and","inference","dataset",".","use_cache","(","bool",")",":","Caches","preprocessed","data","for","future","usage"]
docstr: Filters out doc_spans, which might not be relevant to answering question, 
  which can be helpful when document is extremely long leading to many doc_spans with no answers
  
  Args:
      doc_spans: all possible doc_spans
      tok_start_position: start position of answer in document
      tok_end_position: end position of answer in document
      mode:
          all: do not filter
          only_positive: only keep doc_spans containing the answer
          limited_negative: only keep 10 doc_spans that are nearest to answer
  
  Returns:
      doc_spans: doc_spans after filtering
docstr_tok: ["Filters","out","doc_spans",",","which","might","not","be","relevant","to","answering","question",",","which","can","be","helpful","when","document","is","extremely","long","leading","to","many","doc_spans","with","no","answers","Args",":","doc_spans",":","all","possible","doc_spans","tok_start_position",":","start","position","of","answer","in","document","tok_end_position",":","end","position","of","answer","in","document","mode",":","all",":","do","not","filter","only_positive",":","only","keep","doc_spans","containing","the","answer","limited_negative",":","only","keep","10","doc_spans","that","are","nearest","to","answer","Returns",":","doc_spans",":","doc_spans","after","filtering"]
code: def keep_relevant_docspans(
      doc_spans, tok_start_position, tok_end_position, mode
  ):
      """
      Filters out doc_spans, which might not be relevant to answering question,
      which can be helpful when document is extremely long leading to many doc_spans with no answers
  
      Args:
          doc_spans: all possible doc_spans
          tok_start_position: start position of answer in document
          tok_end_position: end position of answer in document
          mode:
              all: do not filter
              only_positive: only keep doc_spans containing the answer
              limited_negative: only keep 10 doc_spans that are nearest to answer
  
      Returns:
          doc_spans: doc_spans after filtering
      """
      if mode == "all":
          return doc_spans
      elif mode == "only_positive":
          if tok_start_position in [-1, None] or tok_end_position in [
              -1,
              None,
          ]:
              return []
          else:
              return [
                  doc_span
                  for doc_span in doc_spans
                  if tok_start_position >= doc_span.start
                  and tok_end_position
                  <= doc_span.start + doc_span.length - 1
              ]
      elif mode == "limited_negative":
          n_candidates = 10
          if tok_start_position in [-1, None] or tok_end_position in [
              -1,
              None,
          ]:
              pass
          else:
              doc_spans.sort(
                  key=lambda doc_span: SquadDataset.get_average_dist_to_tok_start_and_end(
                      doc_span, tok_start_position, tok_end_position
                  )
              )
          return doc_spans[:n_candidates]
      else:
          raise ValueError(
              "mode can only be in {all, only_positive and limited_negative"
          )
  
imports: import collections
  import os
  import pickle
  from functools import lru_cache
  from typing import Dict, List, Optional
  import numpy
  import psutil
  import torch
  from tqdm import trange
  from nemo.collections.common.parts.utils import _compute_softmax
  from nemo.collections.nlp.data.data_utils import is_whitespace
  from nemo.collections.nlp.data.question_answering_squad.qa_squad_processing import EVALUATION_MODE, INFERENCE_MODE, TRAINING_MODE, SquadProcessor, _improve_answer_span, apply_no_ans_threshold, exact_match_score, f1_score, find_all_best_thresh, get_best_indexes, get_final_text, make_eval_dict, merge_eval, normalize_answer
  from nemo.core.classes import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 9 for calling the SquadDataset obj's underlying member method code with example usage: obj.keep_relevant_docspans(doc_spans, tok_start_position, tok_end_position, mode) and returns: doc_spans, list, unknown, doc_spans