class_docstr: Creates SQuAD dataset for Question Answering.
  Args:
      data_file (str): train.*.json eval.*.json or test.*.json.
      tokenizer (obj): Tokenizer object, e.g. AutoTokenizer.
      version_2_with_negative (bool): True if training should allow
          unanswerable questions.
      doc_stride (int): When splitting up a long document into chunks,
          how much stride to take between chunks.
      max_query_length (iny): All training files which have a duration less
          than min_duration are dropped. Can't be used if the `utt2dur` file
          does not exist. Defaults to None.
      max_seq_length (int): All training files which have a duration more
          than max_duration are dropped. Can't be used if the `utt2dur` file
          does not exist. Defaults to None.
      num_samples: number of samples you want to use for the dataset.
          If -1, use all dataset. Useful for testing.
      mode (str): Use TRAINING_MODE/EVALUATION_MODE/INFERENCE_MODE to define between
          training, evaluation and inference dataset.
      use_cache (bool): Caches preprocessed data for future usage
class_docstr_tok: ["Creates","SQuAD","dataset","for","Question","Answering",".","Args",":","data_file","(","str",")",":","train",".","*",".json","eval",".","*",".json","or","test.","*",".json",".","tokenizer","(","obj",")",":","Tokenizer","object",",","e.g",".","AutoTokenizer",".","version_2_with_negative","(","bool",")",":","True","if","training","should","allow","unanswerable","questions",".","doc_stride","(","int",")",":","When","splitting","up","a","long","document","into","chunks",",","how","much","stride","to","take","between","chunks",".","max_query_length","(","iny",")",":","All","training","files","which","have","a","duration","less","than","min_duration","are","dropped",".","Ca","n't","be","used","if","the","`","utt2dur","`","file","does","not","exist",".","Defaults","to","None",".","max_seq_length","(","int",")",":","All","training","files","which","have","a","duration","more","than","max_duration","are","dropped",".","Ca","n't","be","used","if","the","`","utt2dur","`","file","does","not","exist",".","Defaults","to","None",".","num_samples",":","number","of","samples","you","want","to","use","for","the","dataset",".","If","-1",",","use","all","dataset",".","Useful","for","testing",".","mode","(","str",")",":","Use","TRAINING_MODE\/EVALUATION_MODE\/INFERENCE_MODE","to","define","between","training",",","evaluation","and","inference","dataset",".","use_cache","(","bool",")",":","Caches","preprocessed","data","for","future","usage"]
docstr: Get docspans which are sliding window spans from a document
  
  Args:
      all_doc_tokens: list of all tokens in document
      max_tokens_for_doc: maximum number of tokens in each doc span
      doc_stride: stride size which sliding window moves with
  
  Returns:
      doc_spans: all possible doc_spans from document
docstr_tok: ["Get","docspans","which","are","sliding","window","spans","from","a","document","Args",":","all_doc_tokens",":","list","of","all","tokens","in","document","max_tokens_for_doc",":","maximum","number","of","tokens","in","each","doc","span","doc_stride",":","stride","size","which","sliding","window","moves","with","Returns",":","doc_spans",":","all","possible","doc_spans","from","document"]
code: def get_docspans(all_doc_tokens, max_tokens_for_doc, doc_stride):
      """
      Get docspans which are sliding window spans from a document
  
      Args:
          all_doc_tokens: list of all tokens in document
          max_tokens_for_doc: maximum number of tokens in each doc span
          doc_stride: stride size which sliding window moves with
  
      Returns:
          doc_spans: all possible doc_spans from document
      """
      _DocSpan = collections.namedtuple("DocSpan", ["start", "length"])
      doc_spans = []
      start_offset = 0
      while start_offset < len(all_doc_tokens):
          length = len(all_doc_tokens) - start_offset
          if length > max_tokens_for_doc:
              length = max_tokens_for_doc
          doc_spans.append(_DocSpan(start=start_offset, length=length))
          if start_offset + length == len(all_doc_tokens):
              break
          start_offset += min(length, doc_stride)
      return doc_spans
  
imports: import collections
  import os
  import pickle
  from functools import lru_cache
  from typing import Dict, List, Optional
  import numpy
  import psutil
  import torch
  from tqdm import trange
  from nemo.collections.common.parts.utils import _compute_softmax
  from nemo.collections.nlp.data.data_utils import is_whitespace
  from nemo.collections.nlp.data.question_answering_squad.qa_squad_processing import EVALUATION_MODE, INFERENCE_MODE, TRAINING_MODE, SquadProcessor, _improve_answer_span, apply_no_ans_threshold, exact_match_score, f1_score, find_all_best_thresh, get_best_indexes, get_final_text, make_eval_dict, merge_eval, normalize_answer
  from nemo.core.classes import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 6 for calling the SquadDataset obj's underlying member method code with example usage: obj.get_docspans(all_doc_tokens, max_tokens_for_doc, doc_stride) and returns: doc_spans