class_docstr: Generates and stores PyTorch DataLoader objects for the train, dev and test datasets.
  Relies upon functionality in the processor to do the conversion of the data. Will also
  calculate and display some statistics.
   
class_docstr_tok: ["Generates","and","stores","PyTorch","DataLoader","objects","for","the","train",",","dev","and","test","datasets",".","Relies","upon","functionality","in","the","processor","to","do","the","conversion","of","the","data",".","Will","also","calculate","and","display","some","statistics","."]
docstr: Split a dev set apart from the train dataset.
  
  The exact way how this is done depends on the dev_split and dev_stratification parameters/attributes
  of the Processor used
docstr_tok: ["Split","a","dev","set","apart","from","the","train","dataset",".","The","exact","way","how","this","is","done","depends","on","the","dev_split","and","dev_stratification","parameters\/attributes","of","the","Processor","used"]
code: def _create_dev_from_train(self):
      """
      Split a dev set apart from the train dataset.
  
      The exact way how this is done depends on the dev_split and dev_stratification parameters/attributes
      of the Processor used
      """
      # currently only the TextClassificationProcessor supports stratified dev sets, but this
      # allows the silo to generically check if a processor supports this.
      if (
          hasattr(self.processor, "dev_stratification")
          and self.processor.dev_stratification is not None
      ):
          if self.processor.dev_stratification is True:
              # to do a stratified split we need to get the labels
              ytensors = [t[3][0] for t in self.data["train"]]
              Y = torch.stack(ytensors)
              from sklearn.model_selection import StratifiedShuffleSplit
  
              splitter = StratifiedShuffleSplit(
                  n_splits=1, test_size=self.processor.dev_split
              )
              train_idxs, dev_idxs = next(
                  splitter.split(
                      list(range(len(self.data["train"]))), Y
                  )
              )
              train_idx_set = set([i for i in train_idxs])  # DEBUG
              dev_idx_set = set([i for i in dev_idxs])  # DEBUG
              logger.info(
                  f"Original train size: {len(self.data['train'])}"
              )
              logger.info(
                  f"Created stratified train/dev split: {len(train_idx_set)}/s{len(dev_idx_set)}"
              )
              assert (
                  len(train_idx_set.intersection(dev_idx_set)) == 0
              )  # DEBUG
              # make this a single dataset concat dataset so it is compatible with what is expected in
              # _calculate_statistics
              new_train = ConcatTensorDataset(
                  [Subset(self.data["train"], train_idxs)]
              )
              self.data["dev"] = Subset(self.data["train"], dev_idxs)
              self.data["train"] = new_train
          elif self.processor.dev_stratification is False:
              from sklearn.model_selection import ShuffleSplit
  
              splitter = ShuffleSplit(
                  n_splits=1, test_size=self.processor.dev_split
              )
              train_idxs, dev_idxs = next(
                  splitter.split(list(range(len(self.data["train"]))))
              )
              train_idx_set = set([i for i in train_idxs])  # DEBUG
              dev_idx_set = set([i for i in dev_idxs])  # DEBUG
              assert (
                  len(train_idx_set.intersection(dev_idx_set)) == 0
              )  # DEBUG
              logger.info(
                  f"Created random train/dev split: {len(train_idx_set)}/{len(dev_idx_set)}"
              )
              new_train = ConcatTensorDataset(
                  [Subset(self.data["train"], train_idxs)]
              )
              self.data["dev"] = Subset(self.data["train"], dev_idxs)
              self.data["train"] = new_train
          elif callable(self.processor.dev_stratification):
              (
                  self.data["train"],
                  self.data["dev"],
              ) = self.processor.dev_stratification(
                  self.data["train"], dev_split=self.processor.dev_split
              )
          else:
              raise RuntimeError(
                  f"Value for parameter dev_stratification not supported: {self.processor.dev_stratification}"
              )
      else:
          n_dev = int(
              self.processor.dev_split * len(self.data["train"])
          )
          n_train = len(self.data["train"]) - n_dev
          train_dataset, dev_dataset = self.random_split_ConcatDataset(
              self.data["train"], lengths=[n_train, n_dev]
          )
          self.data["train"] = train_dataset
          if len(dev_dataset) > 0:
              self.data["dev"] = dev_dataset
          else:
              logger.warning(
                  "No dev set created. Please adjust the dev_split parameter."
              )
          logger.info(
              f"Took {len(dev_dataset)} samples out of train set to create dev set (dev split is roughly {self.processor.dev_split})"
          )
  
imports: import copy
  import logging
  import torch.multiprocessing
  from contextlib import ExitStack
  from functools import partial
  import random
  from pathlib import Path
  from itertools import chain, groupby
  import numpy
  from sklearn.utils.class_weight import compute_class_weight
  from torch.utils.data import Dataset, Subset, IterableDataset
  from torch.utils.data.distributed import DistributedSampler
  from torch.utils.data.sampler import RandomSampler, SequentialSampler
  import torch
  from sklearn.model_selection import StratifiedKFold, KFold, ShuffleSplit, StratifiedShuffleSplit
  from tqdm import tqdm
  from farm.data_handler.dataloader import NamedDataLoader
  from farm.data_handler.processor import Processor, BertStyleLMProcessor
  from farm.data_handler.utils import grouper
  from farm.data_handler.dataset import ConcatTensorDataset
  from farm.modeling.tokenization import EmbeddingTokenizer
  from farm.utils import MLFlowLogger
  from farm.utils import log_ascii_workers, calc_chunksize
  from farm.utils import get_dict_checksum
  from farm.visual.ascii.images import TRACTOR_SMALL
label_desc: Use image node_id 9 for calling the DataSilo obj's underlying member method code with example usage: obj._create_dev_from_train() without return types