class_docstr: Creates SQuAD dataset for Question Answering.
  Args:
      data_file (str): train.*.json eval.*.json or test.*.json.
      tokenizer (obj): Tokenizer object, e.g. AutoTokenizer.
      version_2_with_negative (bool): True if training should allow
          unanswerable questions.
      doc_stride (int): When splitting up a long document into chunks,
          how much stride to take between chunks.
      max_query_length (iny): All training files which have a duration less
          than min_duration are dropped. Can't be used if the `utt2dur` file
          does not exist. Defaults to None.
      max_seq_length (int): All training files which have a duration more
          than max_duration are dropped. Can't be used if the `utt2dur` file
          does not exist. Defaults to None.
      num_samples: number of samples you want to use for the dataset.
          If -1, use all dataset. Useful for testing.
      mode (str): Use TRAINING_MODE/EVALUATION_MODE/INFERENCE_MODE to define between
          training, evaluation and inference dataset.
      use_cache (bool): Caches preprocessed data for future usage
class_docstr_tok: ["Creates","SQuAD","dataset","for","Question","Answering",".","Args",":","data_file","(","str",")",":","train",".","*",".json","eval",".","*",".json","or","test.","*",".json",".","tokenizer","(","obj",")",":","Tokenizer","object",",","e.g",".","AutoTokenizer",".","version_2_with_negative","(","bool",")",":","True","if","training","should","allow","unanswerable","questions",".","doc_stride","(","int",")",":","When","splitting","up","a","long","document","into","chunks",",","how","much","stride","to","take","between","chunks",".","max_query_length","(","iny",")",":","All","training","files","which","have","a","duration","less","than","min_duration","are","dropped",".","Ca","n't","be","used","if","the","`","utt2dur","`","file","does","not","exist",".","Defaults","to","None",".","max_seq_length","(","int",")",":","All","training","files","which","have","a","duration","more","than","max_duration","are","dropped",".","Ca","n't","be","used","if","the","`","utt2dur","`","file","does","not","exist",".","Defaults","to","None",".","num_samples",":","number","of","samples","you","want","to","use","for","the","dataset",".","If","-1",",","use","all","dataset",".","Useful","for","testing",".","mode","(","str",")",":","Use","TRAINING_MODE\/EVALUATION_MODE\/INFERENCE_MODE","to","define","between","training",",","evaluation","and","inference","dataset",".","use_cache","(","bool",")",":","Caches","preprocessed","data","for","future","usage"]
docstr: Loads a data file into a list of `InputBatch`s.
docstr_tok: ["Loads","a","data","file","into","a","list","of","`","InputBatch","`","s","."]
code: def convert_examples_to_features(
      self,
      examples: List[object],
      tokenizer: object,
      max_seq_length: int,
      doc_stride: int,
      max_query_length: int,
      has_groundtruth: bool,
  ):
      """Loads a data file into a list of `InputBatch`s."""
  
      unique_id = 1000000000
      features = []
      text_to_tokens_dict = {}
  
      for example_index in trange(len(examples)):
          if example_index % 1000 == 0:
              SquadDataset.check_if_sufficient_memory()
  
          example = examples[example_index]
          if example.question_text not in text_to_tokens_dict:
              text_to_tokens_dict[
                  example.question_text
              ] = tokenizer.text_to_tokens(example.question_text)[
                  :max_query_length
              ]
          query_tokens = text_to_tokens_dict[example.question_text]
  
          # context: index of token -> index of word
          tok_to_orig_index = []
          # context: index of word -> index of first token in token list
          orig_to_tok_index = []
          # context without white spaces after tokenization
          all_doc_tokens = []
          # doc tokens is word separated context
          (
              doc_tokens,
              char_to_word_offset,
              start_position,
              end_position,
              context_text,
          ) = SquadDataset.get_doc_tokens_and_offset_from_context_id(
              example.context_id,
              example.start_position_character,
              example.is_impossible,
              example.answer_text,
              self.processor.doc_id_to_context_text,
          )
  
          example.start_position = start_position
          example.end_position = end_position
          if self.mode != TRAINING_MODE:
              example.doc_tokens = doc_tokens
          # the text to tokens step is the slowest step
          for i, token in enumerate(doc_tokens):
              orig_to_tok_index.append(len(all_doc_tokens))
              if token not in text_to_tokens_dict:
                  text_to_tokens_dict[token] = tokenizer.text_to_tokens(
                      token
                  )
              sub_tokens = text_to_tokens_dict[token]
  
              for sub_token in sub_tokens:
                  tok_to_orig_index.append(i)
                  all_doc_tokens.append(sub_token)
  
          # idx of query token start and end in context
          tok_start_position = None
          tok_end_position = None
          if has_groundtruth and example.is_impossible:
              tok_start_position = -1
              tok_end_position = -1
          if has_groundtruth and not example.is_impossible:
              tok_start_position = orig_to_tok_index[
                  example.start_position
              ]
              if example.end_position < len(doc_tokens) - 1:
                  tok_end_position = (
                      orig_to_tok_index[example.end_position + 1] - 1
                  )
              else:
                  tok_end_position = len(all_doc_tokens) - 1
  
              (
                  tok_start_position,
                  tok_end_position,
              ) = _improve_answer_span(
                  all_doc_tokens,
                  tok_start_position,
                  tok_end_position,
                  tokenizer,
                  example.answer_text,
              )
  
          # The -3 accounts for tokenizer.cls_token, tokenizer.sep_token and tokenizer.sep_token
          # doc_spans contains all possible contexts options of given length
          max_tokens_for_doc = max_seq_length - len(query_tokens) - 3
  
          doc_spans = SquadDataset.get_docspans(
              all_doc_tokens, max_tokens_for_doc, doc_stride
          )
  
          doc_spans = SquadDataset.keep_relevant_docspans(
              doc_spans,
              tok_start_position,
              tok_end_position,
              self.keep_doc_spans,
          )
  
          # make compatible for hashing
          doc_spans = tuple(doc_spans)
  
          for doc_span_index, doc_span in enumerate(doc_spans):
              tokens = (
                  [tokenizer.cls_token]
                  + query_tokens
                  + [tokenizer.sep_token]
              )
              segment_ids = [0 for i in range(len(tokens))]
  
              token_is_max_context = {}
              # maps context tokens idx in final input -> word idx in context
              token_to_orig_map = {}
  
              for i in range(doc_span.length):
                  split_token_index = doc_span.start + i
                  token_to_orig_map[len(tokens)] = tok_to_orig_index[
                      split_token_index
                  ]
                  is_max_context = _check_is_max_context(
                      doc_spans, doc_span_index, split_token_index
                  )
                  token_is_max_context[len(tokens)] = is_max_context
                  tokens.append(all_doc_tokens[split_token_index])
                  segment_ids.append(1)
              tokens.append(tokenizer.sep_token)
              segment_ids.append(1)
  
              input_ids = tokenizer.tokens_to_ids(tokens)
  
              # The mask has 1 for real tokens and 0 for padding tokens.
              # Only real tokens are attended to.
              input_mask = [1] * len(input_ids)
  
              # Zero-pad up to the sequence length.
              while len(input_ids) < max_seq_length:
                  input_ids.append(tokenizer.pad_id)
                  input_mask.append(0)
                  segment_ids.append(0)
  
              assert len(input_ids) == max_seq_length
              assert len(input_mask) == max_seq_length
              assert len(segment_ids) == max_seq_length
  
              # calculate start and end position in final array
              # of tokens in answer if no answer,
              # 0 for both pointing to tokenizer.cls_token
              start_position = 0
              end_position = 0
              if has_groundtruth and not example.is_impossible:
                  doc_start = doc_span.start
                  doc_end = doc_span.start + doc_span.length - 1
                  out_of_span = False
                  if not (
                      tok_start_position >= doc_start
                      and tok_end_position <= doc_end
                  ):
                      out_of_span = True
                  if out_of_span:
                      start_position = 0
                      end_position = 0
                  else:
                      doc_offset = len(query_tokens) + 2
                      start_position = (
                          tok_start_position - doc_start + doc_offset
                      )
                      end_position = (
                          tok_end_position - doc_start + doc_offset
                      )
              if has_groundtruth and example.is_impossible:
                  # if our document chunk does not contain
                  # an annotation we throw it out, since there is nothing
                  # to predict.
                  start_position = 0
                  end_position = 0
  
              if example_index < 1:
                  logging.info("*** Example ***")
                  logging.info("unique_id: %s" % (unique_id))
                  logging.info("example_index: %s" % (example_index))
                  logging.info("doc_span_index: %s" % (doc_span_index))
                  logging.info("tokens: %s" % " ".join(tokens))
                  logging.info(
                      "token_to_orig_map: %s"
                      % " ".join(
                          [
                              "%d:%d" % (x, y)
                              for (x, y) in token_to_orig_map.items()
                          ]
                      )
                  )
                  logging.info(
                      "token_is_max_context: %s"
                      % " ".join(
                          [
                              "%d:%s" % (x, y)
                              for (x, y) in token_is_max_context.items()
                          ]
                      )
                  )
                  logging.info(
                      "input_ids: %s"
                      % " ".join([str(x) for x in input_ids])
                  )
                  logging.info(
                      "input_mask: %s"
                      % " ".join([str(x) for x in input_mask])
                  )
                  logging.info(
                      "segment_ids: %s"
                      % " ".join([str(x) for x in segment_ids])
                  )
                  if has_groundtruth and example.is_impossible:
                      logging.info("impossible example")
                  if has_groundtruth and not example.is_impossible:
                      answer_text = " ".join(
                          tokens[start_position : (end_position + 1)]
                      )
                      logging.info(
                          "start_position: %d" % (start_position)
                      )
                      logging.info("end_position: %d" % (end_position))
                      logging.info("answer: %s" % (answer_text))
  
              # memoization to save CPU memory for large datasets
              input_mask = tuple(input_mask)
              if input_mask in self.input_mask_to_input_mask_id:
                  feature_input_mask_id = (
                      self.input_mask_to_input_mask_id[input_mask]
                  )
              else:
                  self.input_mask_id_to_input_mask[
                      self.input_mask_id
                  ] = input_mask
                  self.input_mask_to_input_mask_id[
                      input_mask
                  ] = self.input_mask_id
                  feature_input_mask_id = self.input_mask_id
                  self.input_mask_id += 1
  
              segment_mask = tuple(segment_ids)
              if segment_mask in self.segment_mask_to_segment_mask_id:
                  feature_segment_mask_id = (
                      self.segment_mask_to_segment_mask_id[segment_mask]
                  )
              else:
                  self.segment_mask_id_to_segment_mask[
                      self.segment_mask_id
                  ] = segment_mask
                  self.segment_mask_to_segment_mask_id[
                      segment_mask
                  ] = self.segment_mask_id
                  feature_segment_mask_id = self.segment_mask_id
                  self.segment_mask_id += 1
              # end memoization
  
              if self.mode == TRAINING_MODE:
                  input_feature = {
                      "unique_id": unique_id,
                      "input_ids": input_ids,
                      "input_mask": feature_input_mask_id,
                      "segment_ids": feature_segment_mask_id,
                      "start_position": start_position,
                      "end_position": end_position,
                  }
              else:
                  input_feature = {
                      "unique_id": unique_id,
                      "input_ids": input_ids,
                      "input_mask": feature_input_mask_id,
                      "segment_ids": feature_segment_mask_id,
                      "start_position": start_position,
                      "end_position": end_position,
                      "example_index": example_index,
                      "doc_span_index": doc_span_index,
                      "tokens": tokens,
                      "token_to_orig_map": token_to_orig_map,
                      "token_is_max_context": token_is_max_context,
                      "is_impossible": example.is_impossible,
                  }
  
              features.append(input_feature)
              unique_id += 1
      return features
  
imports: import collections
  import os
  import pickle
  from functools import lru_cache
  from typing import Dict, List, Optional
  import numpy
  import psutil
  import torch
  from tqdm import trange
  from nemo.collections.common.parts.utils import _compute_softmax
  from nemo.collections.nlp.data.data_utils import is_whitespace
  from nemo.collections.nlp.data.question_answering_squad.qa_squad_processing import EVALUATION_MODE, INFERENCE_MODE, TRAINING_MODE, SquadProcessor, _improve_answer_span, apply_no_ans_threshold, exact_match_score, f1_score, find_all_best_thresh, get_best_indexes, get_final_text, make_eval_dict, merge_eval, normalize_answer
  from nemo.core.classes import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 10 for calling the SquadDataset obj's underlying member method code with example usage: obj.convert_examples_to_features(examples, tokenizer, max_seq_length, doc_stride, max_query_length, has_groundtruth) and returns: features