class_docstr: Facilitate Featurization of In-memory objects.
  
  When featurizing a dataset, it's often the case that the initial set of
  data (pre-featurization) fits handily within memory. (For example, perhaps
  it fits within a column of a pandas DataFrame.) In this case, it would be
  convenient to directly be able to featurize this column of data. However,
  the process of featurization often generates large arrays which quickly eat
  up available memory. This class provides convenient capabilities to process
  such in-memory data by checkpointing generated features periodically to
  disk.
  
  Example
  -------
  Here's an example with only datapoints and no labels or weights.
  
  >>> import deepchem as dc
  >>> smiles = ["C", "CC", "CCC", "CCCC"]
  >>> featurizer = dc.feat.CircularFingerprint()
  >>> loader = dc.data.InMemoryLoader(tasks=["task1"], featurizer=featurizer)
  >>> dataset = loader.create_dataset(smiles, shard_size=2)
  >>> len(dataset)
  4
  
  Here's an example with both datapoints and labels
  
  >>> import deepchem as dc
  >>> smiles = ["C", "CC", "CCC", "CCCC"]
  >>> labels = [1, 0, 1, 0]
  >>> featurizer = dc.feat.CircularFingerprint()
  >>> loader = dc.data.InMemoryLoader(tasks=["task1"], featurizer=featurizer)
  >>> dataset = loader.create_dataset(zip(smiles, labels), shard_size=2)
  >>> len(dataset)
  4
  
  Here's an example with datapoints, labels, weights and ids all provided.
  
  >>> import deepchem as dc
  >>> smiles = ["C", "CC", "CCC", "CCCC"]
  >>> labels = [1, 0, 1, 0]
  >>> weights = [1.5, 0, 1.5, 0]
  >>> ids = ["C", "CC", "CCC", "CCCC"]
  >>> featurizer = dc.feat.CircularFingerprint()
  >>> loader = dc.data.InMemoryLoader(tasks=["task1"], featurizer=featurizer)
  >>> dataset = loader.create_dataset(zip(smiles, labels, weights, ids), shard_size=2)
  >>> len(dataset)
  4
class_docstr_tok: ["Facilitate","Featurization","of","In-memory","objects",".","When","featurizing","a","dataset",",","it","'s","often","the","case","that","the","initial","set","of","data","(","pre-featurization",")","fits","handily","within","memory",".","(","For","example",",","perhaps","it","fits","within","a","column","of","a","pandas","DataFrame",".",")","In","this","case",",","it","would","be","convenient","to","directly","be","able","to","featurize","this","column","of","data",".","However",",","the","process","of","featurization","often","generates","large","arrays","which","quickly","eat","up","available","memory",".","This","class","provides","convenient","capabilities","to","process","such","in-memory","data","by","checkpointing","generated","features","periodically","to","disk",".","Example","--","--","--","-","Here","'s","an","example","with","only","datapoints","and","no","labels","or","weights",".",">",">",">","import","deepchem","as","dc",">",">",">","smiles","=","[","``","C","''",",","``","CC","''",",","``","CCC","''",",","``","CCCC","''","]",">",">",">","featurizer","=","dc.feat.CircularFingerprint","(",")",">",">",">","loader","=","dc.data.InMemoryLoader","(","tasks=","[","``","task1","''","]",",","featurizer=featurizer",")",">",">",">","dataset","=","loader.create_dataset","(","smiles",",","shard_size=2",")",">",">",">","len","(","dataset",")","4","Here","'s","an","example","with","both","datapoints","and","labels",">",">",">","import","deepchem","as","dc",">",">",">","smiles","=","[","``","C","''",",","``","CC","''",",","``","CCC","''",",","``","CCCC","''","]",">",">",">","labels","=","[","1",",","0",",","1",",","0","]",">",">",">","featurizer","=","dc.feat.CircularFingerprint","(",")",">",">",">","loader","=","dc.data.InMemoryLoader","(","tasks=","[","``","task1","''","]",",","featurizer=featurizer",")",">",">",">","dataset","=","loader.create_dataset","(","zip","(","smiles",",","labels",")",",","shard_size=2",")",">",">",">","len","(","dataset",")","4","Here","'s","an","example","with","datapoints",",","labels",",","weights","and","ids","all","provided",".",">",">",">","import","deepchem","as","dc",">",">",">","smiles","=","[","``","C","''",",","``","CC","''",",","``","CCC","''",",","``","CCCC","''","]",">",">",">","labels","=","[","1",",","0",",","1",",","0","]",">",">",">","weights","=","[","1.5",",","0",",","1.5",",","0","]",">",">",">","ids","=","[","``","C","''",",","``","CC","''",",","``","CCC","''",",","``","CCCC","''","]",">",">",">","featurizer","=","dc.feat.CircularFingerprint","(",")",">",">",">","loader","=","dc.data.InMemoryLoader","(","tasks=","[","``","task1","''","]",",","featurizer=featurizer",")",">",">",">","dataset","=","loader.create_dataset","(","zip","(","smiles",",","labels",",","weights",",","ids",")",",","shard_size=2",")",">",">",">","len","(","dataset",")","4"]
docstr: Featurizes a shard of an input data.
  
  Parameters
  ----------
  shard: List
      List each entry of which must be of the form `(featurization_input,
      label, weight, id)` or `(featurization_input, label, weight)` or
      `(featurization_input, label)` or `featurization_input` for one
      datapoint, where `featurization_input` is any input that is recognized
      by `self.featurizer`.
  global_index: int
      The starting index for this shard in the full set of provided inputs
  
  Returns
  ------
  Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]
      The tuple is `(X, y, w, ids)`. All values are numpy arrays.
docstr_tok: ["Featurizes","a","shard","of","an","input","data",".","Parameters","--","--","--","--","--","shard",":","List","List","each","entry","of","which","must","be","of","the","form","`","(","featurization_input",",","label",",","weight",",","id",")","`","or","`","(","featurization_input",",","label",",","weight",")","`","or","`","(","featurization_input",",","label",")","`","or","`","featurization_input","`","for","one","datapoint",",","where","`","featurization_input","`","is","any","input","that","is","recognized","by","`","self.featurizer","`",".","global_index",":","int","The","starting","index","for","this","shard","in","the","full","set","of","provided","inputs","Returns","--","--","--","Tuple","[","np.ndarray",",","np.ndarray",",","np.ndarray",",","np.ndarray","]","The","tuple","is","`","(","X",",","y",",","w",",","ids",")","`",".","All","values","are","numpy","arrays","."]
code: def _featurize_shard(  # type: ignore[override]
      self, shard: List, global_index: int
  ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
      """Featurizes a shard of an input data.
  
      Parameters
      ----------
      shard: List
          List each entry of which must be of the form `(featurization_input,
          label, weight, id)` or `(featurization_input, label, weight)` or
          `(featurization_input, label)` or `featurization_input` for one
          datapoint, where `featurization_input` is any input that is recognized
          by `self.featurizer`.
      global_index: int
          The starting index for this shard in the full set of provided inputs
  
      Returns
      ------
      Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]
          The tuple is `(X, y, w, ids)`. All values are numpy arrays.
      """
      features = []
      labels = []
      weights = []
      ids = []
      n_tasks = len(self.tasks)
      for i, entry in enumerate(shard):
          if not isinstance(entry, tuple):
              entry = (entry,)
          if len(entry) > 4:
              raise ValueError(
                  "Entry is malformed and must be of length 1-4 containing featurization_input"
                  "and optionally label, weight, and id."
              )
          if len(entry) == 4:
              featurization_input, label, weight, entry_id = entry
          elif len(entry) == 3:
              featurization_input, label, weight = entry
              entry_id = global_index + i
          elif len(entry) == 2:
              featurization_input, label = entry
              weight = np.ones((n_tasks), np.float32)
              entry_id = global_index + i
          elif len(entry) == 1:
              featurization_input = entry
              label = np.zeros((n_tasks), np.float32)
              weight = np.zeros((n_tasks), np.float32)
              entry_id = global_index + i
          feature = self.featurizer(featurization_input)
          features.append(feature)
          weights.append(weight)
          labels.append(label)
          ids.append(entry_id)
      X = np.concatenate(features, axis=0)
      return X, np.array(labels), np.array(weights), np.array(ids)
  
imports: import os
  import tempfile
  import zipfile
  import time
  import logging
  import warnings
  from typing import List, Optional, Tuple, Any, Sequence, Union, Iterator
  import pandas
  import numpy
  from deepchem.utils.typing import OneOrMany
  from deepchem.utils.data_utils import load_image_files, load_csv_files, load_json_files, load_sdf_files, unzip_file
  from deepchem.feat import UserDefinedFeaturizer, Featurizer
  from deepchem.data import Dataset, DiskDataset, NumpyDataset, ImageDataset
  from deepchem.feat.molecule_featurizers import OneHotFeaturizer
  from deepchem.utils.genomics_utils import encode_bio_sequence
label_desc: Use image node_id 3 for calling the InMemoryLoader obj's underlying member method code with example usage: obj._featurize_shard(shard, global_index) and returns: X,  np,  np,  np