class_docstr: Generates and stores PyTorch DataLoader objects for the train, dev and test datasets.
  Relies upon functionality in the processor to do the conversion of the data. Will also
  calculate and display some statistics.
   
class_docstr_tok: ["Generates","and","stores","PyTorch","DataLoader","objects","for","the","train",",","dev","and","test","datasets",".","Relies","upon","functionality","in","the","processor","to","do","the","conversion","of","the","data",".","Will","also","calculate","and","display","some","statistics","."]
docstr: Loading the train, dev and test datasets either from files (default) or from supplied dicts.
  The processor is called to handle the full conversion from "raw data" to a Pytorch Dataset.
  The resulting datasets are loaded into DataSilo.data
  
  :param train_dicts: (Optional) dicts containing examples for training.
  :param dev_dicts: (Optional) dicts containing examples for dev.
  :param test_dicts: (Optional) dicts containing examples for test.
  :return: None
docstr_tok: ["Loading","the","train",",","dev","and","test","datasets","either","from","files","(","default",")","or","from","supplied","dicts",".","The","processor","is","called","to","handle","the","full","conversion","from","``","raw","data","''","to","a","Pytorch","Dataset",".","The","resulting","datasets","are","loaded","into","DataSilo.data",":","param","train_dicts",":","(","Optional",")","dicts","containing","examples","for","training",".",":","param","dev_dicts",":","(","Optional",")","dicts","containing","examples","for","dev",".",":","param","test_dicts",":","(","Optional",")","dicts","containing","examples","for","test",".",":","return",":","None"]
code: def _load_data(
      self, train_dicts=None, dev_dicts=None, test_dicts=None
  ):
      """
      Loading the train, dev and test datasets either from files (default) or from supplied dicts.
      The processor is called to handle the full conversion from "raw data" to a Pytorch Dataset.
      The resulting datasets are loaded into DataSilo.data
  
      :param train_dicts: (Optional) dicts containing examples for training.
      :param dev_dicts: (Optional) dicts containing examples for dev.
      :param test_dicts: (Optional) dicts containing examples for test.
      :return: None
      """
  
      logger.info(
          "\nLoading data into the data silo ..."
          "{}".format(TRACTOR_SMALL)
      )
      # train data
      logger.info("LOADING TRAIN DATA")
      logger.info("==================")
      if train_dicts:
          # either from supplied dicts
          logger.info("Loading train set from supplied dicts ")
          self.data["train"], self.tensor_names = self._get_dataset(
              filename=None, dicts=train_dicts
          )
      elif self.processor.train_filename:
          # or from a file (default)
          train_file = (
              self.processor.data_dir / self.processor.train_filename
          )
          logger.info("Loading train set from: {} ".format(train_file))
          self.data["train"], self.tensor_names = self._get_dataset(
              train_file
          )
      else:
          logger.info("No train set is being loaded")
          self.data["train"] = None
  
      # dev data
      logger.info("")
      logger.info("LOADING DEV DATA")
      logger.info("=================")
      if dev_dicts:
          # either from supplied dicts
          logger.info("Loading train set from supplied dicts ")
          self.data["dev"], self.tensor_names = self._get_dataset(
              filename=None, dicts=dev_dicts
          )
      elif self.processor.dev_filename:
          # or from file (default)
          dev_file = (
              self.processor.data_dir / self.processor.dev_filename
          )
          logger.info("Loading dev set from: {}".format(dev_file))
          self.data["dev"], _ = self._get_dataset(dev_file)
      elif self.processor.dev_split > 0.0:
          # or split it apart from train set
          logger.info("Loading dev set as a slice of train set")
          self._create_dev_from_train()
      else:
          logger.info("No dev set is being loaded")
          self.data["dev"] = None
  
      logger.info("")
      logger.info("LOADING TEST DATA")
      logger.info("=================")
      # test data
      if test_dicts:
          # either from supplied dicts
          logger.info("Loading train set from supplied dicts ")
          self.data["test"], self.tensor_names = self._get_dataset(
              filename=None, dicts=test_dicts
          )
      elif self.processor.test_filename:
          # or from file (default)
          test_file = (
              self.processor.data_dir / self.processor.test_filename
          )
          logger.info("Loading test set from: {}".format(test_file))
          if self.tensor_names:
              self.data["test"], _ = self._get_dataset(test_file)
          else:
              self.data["test"], self.tensor_names = self._get_dataset(
                  test_file
              )
      else:
          logger.info("No test set is being loaded")
          self.data["test"] = None
  
      if self.caching:
          self._save_dataset_to_cache()
  
      # derive stats and meta data
      self._calculate_statistics()
      # self.calculate_class_weights()
  
      self._initialize_data_loaders()
  
imports: import copy
  import logging
  import torch.multiprocessing
  from contextlib import ExitStack
  from functools import partial
  import random
  from pathlib import Path
  from itertools import chain, groupby
  import numpy
  from sklearn.utils.class_weight import compute_class_weight
  from torch.utils.data import Dataset, Subset, IterableDataset
  from torch.utils.data.distributed import DistributedSampler
  from torch.utils.data.sampler import RandomSampler, SequentialSampler
  import torch
  from sklearn.model_selection import StratifiedKFold, KFold, ShuffleSplit, StratifiedShuffleSplit
  from tqdm import tqdm
  from farm.data_handler.dataloader import NamedDataLoader
  from farm.data_handler.processor import Processor, BertStyleLMProcessor
  from farm.data_handler.utils import grouper
  from farm.data_handler.dataset import ConcatTensorDataset
  from farm.modeling.tokenization import EmbeddingTokenizer
  from farm.utils import MLFlowLogger
  from farm.utils import log_ascii_workers, calc_chunksize
  from farm.utils import get_dict_checksum
  from farm.visual.ascii.images import TRACTOR_SMALL
label_desc: Use image node_id 4 for calling the DataSilo obj's underlying member method code with example usage: obj._load_data(train_dicts, dev_dicts, test_dicts) without return types