class_docstr: Provide data silos for holdout evaluation.
  
  For this, we first combine the
  instances from all the sets or just some of the sets, then allow to create a different data silo
  instance with a train/test split.
  Calling DataSiloForHoldout.make() creates a list of holdout splits
class_docstr_tok: ["Provide","data","silos","for","holdout","evaluation",".","For","this",",","we","first","combine","the","instances","from","all","the","sets","or","just","some","of","the","sets",",","then","allow","to","create","a","different","data","silo","instance","with","a","train\/test","split",".","Calling","DataSiloForHoldout.make","(",")","creates","a","list","of","holdout","splits"]
docstr: Create number of folds data-silo-like objects which can be used for training from the
  original data silo passed on.
  
  :param datasilo: the data silo that contains the original data
  :param sets: which sets to use to create the xval folds
  :param n_splits: number of folds to create
  :param train_split: portion of all data to use for training, rest is used for evaluation
  :param random_state: random state for shuffling
  :param stratification: True, False or a callable the returns a new train/test dataset pair from
      the original train set and kwarg train_split.
docstr_tok: ["Create","number","of","folds","data-silo-like","objects","which","can","be","used","for","training","from","the","original","data","silo","passed","on",".",":","param","datasilo",":","the","data","silo","that","contains","the","original","data",":","param","sets",":","which","sets","to","use","to","create","the","xval","folds",":","param","n_splits",":","number","of","folds","to","create",":","param","train_split",":","portion","of","all","data","to","use","for","training",",","rest","is","used","for","evaluation",":","param","random_state",":","random","state","for","shuffling",":","param","stratification",":","True",",","False","or","a","callable","the","returns","a","new","train\/test","dataset","pair","from","the","original","train","set","and","kwarg","train_split","."]
code: def _make(
      datasilo,
      sets=["train", "dev", "test"],
      n_splits=5,
      train_split=0.80,
      random_state=None,
      stratification=True,
  ):
      """
      Create number of folds data-silo-like objects which can be used for training from the
      original data silo passed on.
  
      :param datasilo: the data silo that contains the original data
      :param sets: which sets to use to create the xval folds
      :param n_splits: number of folds to create
      :param train_split: portion of all data to use for training, rest is used for evaluation
      :param random_state: random state for shuffling
      :param stratification: True, False or a callable the returns a new train/test dataset pair from
          the original train set and kwarg train_split.
      """
      setstoconcat = [datasilo.data[setname] for setname in sets]
      ds_all = ConcatTensorDataset(setstoconcat)
      idxs = list(range(len(ds_all)))
      dev_split = datasilo.processor.dev_split
      Y = None
      silos = []
      if callable(stratification):
          for i in range(n_splits):
              ds_train, ds_test = stratification(
                  ds_all, train_split=train_split
              )
              silos.append(
                  DataSiloForHoldout(datasilo, ds_train, None, ds_test)
              )
          return silos
      elif stratification:
          ytensors = [t[3][0] for t in ds_all]
          Y = torch.stack(ytensors)
          eval = StratifiedShuffleSplit(
              n_splits=n_splits,
              train_size=train_split,
              random_state=random_state,
          )
          eval_split = eval.split(idxs, Y)
      else:
          eval = ShuffleSplit(
              n_splits=n_splits,
              train_size=train_split,
              random_state=random_state,
          )
          eval_split = eval.split(idxs)
      # for each fold create a DataSilo4Xval instance, where the training set is further
      # divided into actual train and dev set
      for train_idx, test_idx in eval_split:
          ds_train = Subset(ds_all, train_idx)
          ds_test = Subset(ds_all, test_idx)
          silo = DataSiloForHoldout(datasilo, ds_train, None, ds_test)
          silos.append(silo)
      return silos
  
imports: import copy
  import logging
  import torch.multiprocessing
  from contextlib import ExitStack
  from functools import partial
  import random
  from pathlib import Path
  from itertools import chain, groupby
  import numpy
  from sklearn.utils.class_weight import compute_class_weight
  from torch.utils.data import Dataset, Subset, IterableDataset
  from torch.utils.data.distributed import DistributedSampler
  from torch.utils.data.sampler import RandomSampler, SequentialSampler
  import torch
  from sklearn.model_selection import StratifiedKFold, KFold, ShuffleSplit, StratifiedShuffleSplit
  from tqdm import tqdm
  from farm.data_handler.dataloader import NamedDataLoader
  from farm.data_handler.processor import Processor, BertStyleLMProcessor
  from farm.data_handler.utils import grouper
  from farm.data_handler.dataset import ConcatTensorDataset
  from farm.modeling.tokenization import EmbeddingTokenizer
  from farm.utils import MLFlowLogger
  from farm.utils import log_ascii_workers, calc_chunksize
  from farm.utils import get_dict_checksum
  from farm.visual.ascii.images import TRACTOR_SMALL
label_desc: Use image node_id 4 for calling the DataSiloForHoldout obj's underlying member method code with example usage: obj._make(datasilo, sets, n_splits, train_split, random_state, stratification) and returns: silos, silos