docstr: :param processor: A dataset specific Processor object which will turn input file into a Pytorch Dataset.
  :type processor: Processor
  :param batch_size: The size of batch that should be returned by the DataLoaders.
  :type batch_size: int
  :param filepath: input filename to load the dataset from
  :type filepath: Path
  :param dataloader_workers: number of workers for PyTorch Dataloader
  :type dataloader_workers: int
docstr_tok: [":","param","processor",":","A","dataset","specific","Processor","object","which","will","turn","input","file","into","a","Pytorch","Dataset",".",":","type","processor",":","Processor",":","param","batch_size",":","The","size","of","batch","that","should","be","returned","by","the","DataLoaders",".",":","type","batch_size",":","int",":","param","filepath",":","input","filename","to","load","the","dataset","from",":","type","filepath",":","Path",":","param","dataloader_workers",":","number","of","workers","for","PyTorch","Dataloader",":","type","dataloader_workers",":","int"]
code: def __init__(
      self,
      processor,
      filepath,
      batch_size,
      dataloader_workers,
      distributed=False,
      n_samples=None,
  ):
      """
      :param processor: A dataset specific Processor object which will turn input file into a Pytorch Dataset.
      :type processor: Processor
      :param batch_size: The size of batch that should be returned by the DataLoaders.
      :type batch_size: int
      :param filepath: input filename to load the dataset from
      :type filepath: Path
      :param dataloader_workers: number of workers for PyTorch Dataloader
      :type dataloader_workers: int
      """
  
      self.batch_size = batch_size
      self.processor = processor
      self.filepath = filepath
      self.dataloader_workers = dataloader_workers
      self.distributed = distributed
  
      # calculate or estimate number of samples so that the data loader can derive number of training steps
      if filepath.is_file():
          files = [filepath]
      else:
          files = [file for file in filepath.iterdir()]
  
      if n_samples:
          self.n_samples = n_samples
      else:
          try:
              self.n_samples = self.processor.estimate_n_samples(
                  files[0]
              ) * len(files)
          except AttributeError:
              AttributeError(
                  f"Could not estimate n_samples for {self.processor.__class__.__name__} in StreamingDataSilo. "
                  f"Make sure that your Processor has `estimate_n_samples()` implemented"
              )
      logger.info(f"Found data for {self.n_samples} samples")
      self.shuffle_files(files)
  
      dicts_from_files = [
          processor.file_to_dicts(file) for file in files
      ]
      self.file_to_dicts_generator = chain(*dicts_from_files)
  
      if self.distributed:
          self.rank = torch.distributed.get_rank()
          self.world_size = torch.distributed.get_world_size()
  
imports: import copy
  import logging
  import torch.multiprocessing
  from contextlib import ExitStack
  from functools import partial
  import random
  from pathlib import Path
  from itertools import chain, groupby
  import numpy
  from sklearn.utils.class_weight import compute_class_weight
  from torch.utils.data import Dataset, Subset, IterableDataset
  from torch.utils.data.distributed import DistributedSampler
  from torch.utils.data.sampler import RandomSampler, SequentialSampler
  import torch
  from sklearn.model_selection import StratifiedKFold, KFold, ShuffleSplit, StratifiedShuffleSplit
  from tqdm import tqdm
  from farm.data_handler.dataloader import NamedDataLoader
  from farm.data_handler.processor import Processor, BertStyleLMProcessor
  from farm.data_handler.utils import grouper
  from farm.data_handler.dataset import ConcatTensorDataset
  from farm.modeling.tokenization import EmbeddingTokenizer
  from farm.utils import MLFlowLogger
  from farm.utils import log_ascii_workers, calc_chunksize
  from farm.utils import get_dict_checksum
  from farm.visual.ascii.images import TRACTOR_SMALL
label_desc: Use image node_id 1 to create a new _StreamingDataSet object from inherited base classes: IterableDataset with example: obj = _StreamingDataSet(processor, filepath, batch_size, dataloader_workers, distributed, n_samples)