class_docstr: Streaming Data Silo loads and preprocesses datasets in parallel to the model training.
  
  The samples are lazily created from the input file and batches are yielded on-the-fly when required during training.
  This is useful if you:
  - work with large datasets that don't fit in memory
  - want to save time (by not preprocessing the entire dataset before starting training)
  
  For optimal training performance and efficient utilization of shiny GPUs, the pipeline always keeps a few
  pre-computed batches ready to avoid any waiting time when a batch is requested during training.
  
  To parallelize the creation of batches, PyTorch DataLoader provide an option to use
  multiple workers that utilize the available CPU cores and ensure enough pre-computed batches.
class_docstr_tok: ["Streaming","Data","Silo","loads","and","preprocesses","datasets","in","parallel","to","the","model","training",".","The","samples","are","lazily","created","from","the","input","file","and","batches","are","yielded","on-the-fly","when","required","during","training",".","This","is","useful","if","you",":","-","work","with","large","datasets","that","do","n't","fit","in","memory","-","want","to","save","time","(","by","not","preprocessing","the","entire","dataset","before","starting","training",")","For","optimal","training","performance","and","efficient","utilization","of","shiny","GPUs",",","the","pipeline","always","keeps","a","few","pre-computed","batches","ready","to","avoid","any","waiting","time","when","a","batch","is","requested","during","training",".","To","parallelize","the","creation","of","batches",",","PyTorch","DataLoader","provide","an","option","to","use","multiple","workers","that","utilize","the","available","CPU","cores","and","ensure","enough","pre-computed","batches","."]
docstr: :param processor: A dataset specific Processor object which will turn input file into a Pytorch Dataset.
  :type processor: Processor
  :param batch_size: The size of batch to use for model training.
  :type batch_size: int
  :param dataloader_workers: number of workers for PyTorch DataLoader to create batches in parallel
  :type dataloader_workers: int
docstr_tok: [":","param","processor",":","A","dataset","specific","Processor","object","which","will","turn","input","file","into","a","Pytorch","Dataset",".",":","type","processor",":","Processor",":","param","batch_size",":","The","size","of","batch","to","use","for","model","training",".",":","type","batch_size",":","int",":","param","dataloader_workers",":","number","of","workers","for","PyTorch","DataLoader","to","create","batches","in","parallel",":","type","dataloader_workers",":","int"]
code: def __init__(
      self,
      processor,
      batch_size,
      distributed=False,
      dataloader_workers=8,
  ):
      """
      :param processor: A dataset specific Processor object which will turn input file into a Pytorch Dataset.
      :type processor: Processor
      :param batch_size: The size of batch to use for model training.
      :type batch_size: int
      :param dataloader_workers: number of workers for PyTorch DataLoader to create batches in parallel
      :type dataloader_workers: int
      """
  
      self.processor = processor
      self.batch_size = batch_size
      self.dataloader_workers = dataloader_workers
      self.distributed = distributed
  
imports: import copy
  import logging
  import torch.multiprocessing
  from contextlib import ExitStack
  from functools import partial
  import random
  from pathlib import Path
  from itertools import chain, groupby
  import numpy
  from sklearn.utils.class_weight import compute_class_weight
  from torch.utils.data import Dataset, Subset, IterableDataset
  from torch.utils.data.distributed import DistributedSampler
  from torch.utils.data.sampler import RandomSampler, SequentialSampler
  import torch
  from sklearn.model_selection import StratifiedKFold, KFold, ShuffleSplit, StratifiedShuffleSplit
  from tqdm import tqdm
  from farm.data_handler.dataloader import NamedDataLoader
  from farm.data_handler.processor import Processor, BertStyleLMProcessor
  from farm.data_handler.utils import grouper
  from farm.data_handler.dataset import ConcatTensorDataset
  from farm.modeling.tokenization import EmbeddingTokenizer
  from farm.utils import MLFlowLogger
  from farm.utils import log_ascii_workers, calc_chunksize
  from farm.utils import get_dict_checksum
  from farm.visual.ascii.images import TRACTOR_SMALL
label_desc: Use image node_id 1 to create a new StreamingDataSilo object  with example: obj = StreamingDataSilo(processor, batch_size, distributed, dataloader_workers)