class_docstr: Perform cross validation or nested cross validation.
  
  For performing cross validation or nested cross validation, we really want to combine all the
  instances from all the sets or just some of the sets, then create a different data silo
  instance for each fold or nested fold.
  Calling DataSiloForCrossVal.make() creates a list of DataSiloForCrossVal instances - one for each fold.
class_docstr_tok: ["Perform","cross","validation","or","nested","cross","validation",".","For","performing","cross","validation","or","nested","cross","validation",",","we","really","want","to","combine","all","the","instances","from","all","the","sets","or","just","some","of","the","sets",",","then","create","a","different","data","silo","instance","for","each","fold","or","nested","fold",".","Calling","DataSiloForCrossVal.make","(",")","creates","a","list","of","DataSiloForCrossVal","instances","-","one","for","each","fold","."]
docstr: Create number of folds data-silo-like objects which can be used for training from the
  original data silo passed on. This function takes into account the characteristics of the
  data for question-answering-
  
  :param datasilo: the data silo that contains the original data
  :type datasilo: DataSilo
  :param sets: which sets to use to create the xval folds (strings)
  :type sets: list
  :param n_splits: number of folds to create
  :type n_splits: int
  :param shuffle: shuffle each class' samples before splitting
  :type shuffle: bool
  :param random_state: random state for shuffling
  :type random_state: int
  :param n_neg_answers_per_question: number of negative answers per question to include for training
  :type n_neg_answers_per_question: int
docstr_tok: ["Create","number","of","folds","data-silo-like","objects","which","can","be","used","for","training","from","the","original","data","silo","passed","on",".","This","function","takes","into","account","the","characteristics","of","the","data","for","question-answering-",":","param","datasilo",":","the","data","silo","that","contains","the","original","data",":","type","datasilo",":","DataSilo",":","param","sets",":","which","sets","to","use","to","create","the","xval","folds","(","strings",")",":","type","sets",":","list",":","param","n_splits",":","number","of","folds","to","create",":","type","n_splits",":","int",":","param","shuffle",":","shuffle","each","class","'","samples","before","splitting",":","type","shuffle",":","bool",":","param","random_state",":","random","state","for","shuffling",":","type","random_state",":","int",":","param","n_neg_answers_per_question",":","number","of","negative","answers","per","question","to","include","for","training",":","type","n_neg_answers_per_question",":","int"]
code: def _make_question_answering(
      cls,
      datasilo,
      sets=["train", "dev", "test"],
      n_splits=5,
      shuffle=True,
      random_state=None,
      n_neg_answers_per_question=1,
  ):
      """
      Create number of folds data-silo-like objects which can be used for training from the
      original data silo passed on. This function takes into account the characteristics of the
      data for question-answering-
  
      :param datasilo: the data silo that contains the original data
      :type datasilo: DataSilo
      :param sets: which sets to use to create the xval folds (strings)
      :type sets: list
      :param n_splits: number of folds to create
      :type n_splits: int
      :param shuffle: shuffle each class' samples before splitting
      :type shuffle: bool
      :param random_state: random state for shuffling
      :type random_state: int
      :param n_neg_answers_per_question: number of negative answers per question to include for training
      :type n_neg_answers_per_question: int
      """
      assert (
          "id" in datasilo.tensor_names
      ), f"Expected tensor 'id' in tensor names, found {datasilo.tensor_names}"
      assert (
          "labels" in datasilo.tensor_names
      ), f"Expected tensor 'labels' in tensor names, found {datasilo.tensor_names}"
  
      id_index = datasilo.tensor_names.index("id")
      label_index = datasilo.tensor_names.index("labels")
  
      sets_to_concat = []
      for setname in sets:
          if datasilo.data[setname]:
              sets_to_concat.extend(datasilo.data[setname])
      all_data = ConcatTensorDataset(sets_to_concat)
  
      documents = []
      keyfunc = lambda x: x[id_index][0]
      all_data = sorted(all_data.datasets, key=keyfunc)
      for key, document in groupby(all_data, key=keyfunc):
          documents.append(list(document))
  
      xval_split = cls._split_for_qa(
          documents=documents,
          id_index=id_index,
          n_splits=n_splits,
          shuffle=shuffle,
          random_state=random_state,
      )
      silos = []
  
      for train_set, test_set in xval_split:
          # Each training set is further divided into actual train and dev set
          if datasilo.processor.dev_split > 0:
              dev_split = datasilo.processor.dev_split
              n_dev = int(np.ceil(dev_split * len(train_set)))
              assert (
                  n_dev > 0
              ), f"dev split of {dev_split} is not large enough to split away a development set"
              n_actual_train = len(train_set) - n_dev
              actual_train_set = train_set[:n_actual_train]
              dev_set = train_set[n_actual_train:]
              ds_dev = [
                  sample for document in dev_set for sample in document
              ]
          else:
              ds_dev = None
              actual_train_set = train_set
  
          train_samples = []
          for doc in actual_train_set:
              keyfunc = lambda x: x[id_index][1]
              doc = sorted(doc, key=keyfunc)
              for key, question in groupby(doc, key=keyfunc):
                  # add all available answrs to train set
                  sample_list = list(question)
                  neg_answer_idx = []
                  for index, sample in enumerate(sample_list):
                      if (
                          sample[label_index][0][0]
                          or sample[label_index][0][1]
                      ):
                          train_samples.append(sample)
                      else:
                          neg_answer_idx.append(index)
                  # add random n_neg_answers_per_question samples to train set
                  if len(neg_answer_idx) <= n_neg_answers_per_question:
                      train_samples.extend(
                          [sample_list[idx] for idx in neg_answer_idx]
                      )
                  else:
                      neg_answer_idx = random.sample(
                          neg_answer_idx, n_neg_answers_per_question
                      )
                      train_samples.extend(
                          [sample_list[idx] for idx in neg_answer_idx]
                      )
  
          ds_train = train_samples
          ds_test = [
              sample for document in test_set for sample in document
          ]
          silos.append(
              DataSiloForCrossVal(datasilo, ds_train, ds_dev, ds_test)
          )
      return silos
  
imports: import copy
  import logging
  import torch.multiprocessing
  from contextlib import ExitStack
  from functools import partial
  import random
  from pathlib import Path
  from itertools import chain, groupby
  import numpy
  from sklearn.utils.class_weight import compute_class_weight
  from torch.utils.data import Dataset, Subset, IterableDataset
  from torch.utils.data.distributed import DistributedSampler
  from torch.utils.data.sampler import RandomSampler, SequentialSampler
  import torch
  from sklearn.model_selection import StratifiedKFold, KFold, ShuffleSplit, StratifiedShuffleSplit
  from tqdm import tqdm
  from farm.data_handler.dataloader import NamedDataLoader
  from farm.data_handler.processor import Processor, BertStyleLMProcessor
  from farm.data_handler.utils import grouper
  from farm.data_handler.dataset import ConcatTensorDataset
  from farm.modeling.tokenization import EmbeddingTokenizer
  from farm.utils import MLFlowLogger
  from farm.utils import log_ascii_workers, calc_chunksize
  from farm.utils import get_dict_checksum
  from farm.visual.ascii.images import TRACTOR_SMALL
label_desc: Use image node_id 4 for calling the DataSiloForCrossVal obj's underlying member method code with example usage: obj._make_question_answering(cls, datasilo, sets, n_splits, shuffle, random_state, n_neg_answers_per_question) and returns: silos