class_docstr: Generates and stores PyTorch DataLoader objects for the train, dev and test datasets.
  Relies upon functionality in the processor to do the conversion of the data. Will also
  calculate and display some statistics.
   
class_docstr_tok: ["Generates","and","stores","PyTorch","DataLoader","objects","for","the","train",",","dev","and","test","datasets",".","Relies","upon","functionality","in","the","processor","to","do","the","conversion","of","the","data",".","Will","also","calculate","and","display","some","statistics","."]
docstr: :param processor: A dataset specific Processor object which will turn input (file or dict) into a Pytorch Dataset.
  :type processor: Processor
  :param batch_size: The size of batch that should be returned by the DataLoader for the training set.
  :type batch_size: int
  :param eval_batch_size: The size of batch that should be returned by the DataLoaders for the dev and test set.
  :type eval_batch_size: int
  :param distributed: Set to True if you are running in a distributed evn, e.g. using DistributedDataParallel.
                      The DataSilo will init the DataLoader with a DistributedSampler() to distribute batches.
  :type distributed: bool
  :param automatic_loading: Set to False, if you don't want to automatically load data at initialization.
  :type automatic_loading: bool
  :param max_multiprocessing_chunksize: max possible value for chunksize as calculated by `calc_chunksize()`
      in `farm.utils`. For certain cases like lm_finetuning, a smaller value can be set, as the default chunksize
      values are rather large that might cause memory issues.
  :type max_multiprocessing_chunksize: int
  :param max_processes: the maximum number of processes to spawn in the multiprocessing.Pool used in DataSilo.
                        It can be set to 1 to disable the use of multiprocessing or make debugging easier.
  :type max_processes: int
  :param caching: save the processed datasets on disk to save time/compute if the same train data is used to run
                  multiple experiments. Each cache has a checksum based on the train_filename of the Processor
                  and the batch size.
  :type caching: bool
  :param cache_path: root dir for storing the datasets' cache.
  :type cache_path: Path
docstr_tok: [":","param","processor",":","A","dataset","specific","Processor","object","which","will","turn","input","(","file","or","dict",")","into","a","Pytorch","Dataset",".",":","type","processor",":","Processor",":","param","batch_size",":","The","size","of","batch","that","should","be","returned","by","the","DataLoader","for","the","training","set",".",":","type","batch_size",":","int",":","param","eval_batch_size",":","The","size","of","batch","that","should","be","returned","by","the","DataLoaders","for","the","dev","and","test","set",".",":","type","eval_batch_size",":","int",":","param","distributed",":","Set","to","True","if","you","are","running","in","a","distributed","evn",",","e.g",".","using","DistributedDataParallel",".","The","DataSilo","will","init","the","DataLoader","with","a","DistributedSampler","(",")","to","distribute","batches",".",":","type","distributed",":","bool",":","param","automatic_loading",":","Set","to","False",",","if","you","do","n't","want","to","automatically","load","data","at","initialization",".",":","type","automatic_loading",":","bool",":","param","max_multiprocessing_chunksize",":","max","possible","value","for","chunksize","as","calculated","by","`","calc_chunksize","(",")","`","in","`","farm.utils","`",".","For","certain","cases","like","lm_finetuning",",","a","smaller","value","can","be","set",",","as","the","default","chunksize","values","are","rather","large","that","might","cause","memory","issues",".",":","type","max_multiprocessing_chunksize",":","int",":","param","max_processes",":","the","maximum","number","of","processes","to","spawn","in","the","multiprocessing.Pool","used","in","DataSilo",".","It","can","be","set","to","1","to","disable","the","use","of","multiprocessing","or","make","debugging","easier",".",":","type","max_processes",":","int",":","param","caching",":","save","the","processed","datasets","on","disk","to","save","time\/compute","if","the","same","train","data","is","used","to","run","multiple","experiments",".","Each","cache","has","a","checksum","based","on","the","train_filename","of","the","Processor","and","the","batch","size",".",":","type","caching",":","bool",":","param","cache_path",":","root","dir","for","storing","the","datasets","'","cache",".",":","type","cache_path",":","Path"]
code: def __init__(
      self,
      processor,
      batch_size,
      eval_batch_size=None,
      distributed=False,
      automatic_loading=True,
      max_multiprocessing_chunksize=2000,
      max_processes=128,
      caching=False,
      cache_path=Path("cache/data_silo"),
  ):
      """
      :param processor: A dataset specific Processor object which will turn input (file or dict) into a Pytorch Dataset.
      :type processor: Processor
      :param batch_size: The size of batch that should be returned by the DataLoader for the training set.
      :type batch_size: int
      :param eval_batch_size: The size of batch that should be returned by the DataLoaders for the dev and test set.
      :type eval_batch_size: int
      :param distributed: Set to True if you are running in a distributed evn, e.g. using DistributedDataParallel.
                          The DataSilo will init the DataLoader with a DistributedSampler() to distribute batches.
      :type distributed: bool
      :param automatic_loading: Set to False, if you don't want to automatically load data at initialization.
      :type automatic_loading: bool
      :param max_multiprocessing_chunksize: max possible value for chunksize as calculated by `calc_chunksize()`
          in `farm.utils`. For certain cases like lm_finetuning, a smaller value can be set, as the default chunksize
          values are rather large that might cause memory issues.
      :type max_multiprocessing_chunksize: int
      :param max_processes: the maximum number of processes to spawn in the multiprocessing.Pool used in DataSilo.
                            It can be set to 1 to disable the use of multiprocessing or make debugging easier.
      :type max_processes: int
      :param caching: save the processed datasets on disk to save time/compute if the same train data is used to run
                      multiple experiments. Each cache has a checksum based on the train_filename of the Processor
                      and the batch size.
      :type caching: bool
      :param cache_path: root dir for storing the datasets' cache.
      :type cache_path: Path
      """
      self.distributed = distributed
      self.processor = processor
      self.data = {}
      self.batch_size = batch_size
      self.class_weights = None
      self.max_processes = max_processes
      self.max_multiprocessing_chunksize = max_multiprocessing_chunksize
      self.caching = caching
      self.cache_path = cache_path
      self.tensor_names = None
      if eval_batch_size is None:
          self.eval_batch_size = batch_size
      else:
          self.eval_batch_size = eval_batch_size
  
      if len(self.processor.tasks) == 0:
          raise Exception(
              "No task initialized. Try initializing the processor with a metric and a label list. "
              "Alternatively you can add a task using Processor.add_task()"
          )
  
      if type(self.processor.tokenizer) == EmbeddingTokenizer:
          if max_processes != 1:
              logger.warning(
                  "Multiprocessing not efficient for WordEmbedding Tokenizers. Please set max_process \n"
                  "argument in DataSilo to 1."
              )
  
      loaded_from_cache = False
      if self.caching:  # Check if DataSets are present in cache
          checksum = self._get_checksum()
          dataset_path = self.cache_path / checksum
  
          if dataset_path.exists():
              self._load_dataset_from_cache(dataset_path)
              loaded_from_cache = True
  
      if not loaded_from_cache and automatic_loading:
          # In most cases we want to load all data automatically, but in some cases we rather want to do this
          # later or load from dicts instead of file (https://github.com/deepset-ai/FARM/issues/85)
          self._load_data()
  
imports: import copy
  import logging
  import torch.multiprocessing
  from contextlib import ExitStack
  from functools import partial
  import random
  from pathlib import Path
  from itertools import chain, groupby
  import numpy
  from sklearn.utils.class_weight import compute_class_weight
  from torch.utils.data import Dataset, Subset, IterableDataset
  from torch.utils.data.distributed import DistributedSampler
  from torch.utils.data.sampler import RandomSampler, SequentialSampler
  import torch
  from sklearn.model_selection import StratifiedKFold, KFold, ShuffleSplit, StratifiedShuffleSplit
  from tqdm import tqdm
  from farm.data_handler.dataloader import NamedDataLoader
  from farm.data_handler.processor import Processor, BertStyleLMProcessor
  from farm.data_handler.utils import grouper
  from farm.data_handler.dataset import ConcatTensorDataset
  from farm.modeling.tokenization import EmbeddingTokenizer
  from farm.utils import MLFlowLogger
  from farm.utils import log_ascii_workers, calc_chunksize
  from farm.utils import get_dict_checksum
  from farm.visual.ascii.images import TRACTOR_SMALL
label_desc: Use image node_id 1 to create a new DataSilo object  with example: obj = DataSilo(processor, batch_size, eval_batch_size, distributed, automatic_loading, max_multiprocessing_chunksize, max_processes, caching, cache_path)