code: def create_dataset_images_from_tar(
      tar_paths,
      path_save_dir_tmp_datasets_images,
      num_proc,
      path_save_file_map_url_idx,
      path_save_dir_dataset_images,
  ):
      def process_one_tar(args):
          (tar_path, idx_tar) = args
          with tarfile.open(tar_path) as tar_file:
              tar_members = tar_file.getmembers()
              name_to_url = {}
              name_to_img = {}
              url_to_img = {}
              for tar_member in tar_members:
                  if tar_member.name.endswith(".jpg"):
                      name = tar_member.name.replace(".jpg", "")
                      tar_member_file = tar_file.extractfile(tar_member)
                      img = tar_member_file.read()
                      tar_member_file.close()
                      name_to_img[name] = img
                  elif tar_member.name.endswith(".json"):
                      name = tar_member.name.replace(".json", "")
                      tar_member_file = tar_file.extractfile(tar_member)
                      json_val = json.loads(tar_member_file.read())
                      status = json_val["status"]
                      url = json_val["url"]
                      tar_member_file.close()
                      if (
                          status == "success"
                      ):  # Should always happend with webdataset format, not with parquet
                          name_to_url[name] = url
              for name in name_to_url:
                  url_to_img[name_to_url[name]] = name_to_img[name]
              new_urls_indexed = list(url_to_img.keys())
              new_datasets_images = Dataset.from_dict(
                  {
                      "url": list(url_to_img.keys()),
                      "image": list(url_to_img.values()),
                  }
              )
              # We need to save the new datasets and then reload them, since `from_dict` store the dataset
              # in the RAM and does not use the disk space
              new_datasets_images.save_to_disk(
                  os.path.join(
                      path_save_dir_tmp_datasets_images, str(idx_tar)
                  )
              )
              return new_urls_indexed
  
      logger.info("Starting creating the dataset of all images")
      args_pool = [
          (tar_path, idx_tar)
          for idx_tar, tar_path in enumerate(tar_paths)
      ]
      pool = Pool(num_proc)
      urls_indexed = pool.map(process_one_tar, args_pool)
      urls_indexed = [sub_el for el in urls_indexed for sub_el in el]
  
      map_url_idx = {url: idx for idx, url in enumerate(urls_indexed)}
      with open(path_save_file_map_url_idx, "w") as f:
          json.dump(map_url_idx, f)
      datasets_images = [
          load_from_disk(
              os.path.join(
                  path_save_dir_tmp_datasets_images, str(idx_tar)
              )
          )
          for idx_tar in range(len(tar_paths))
      ]
      dataset_images = concatenate_datasets(datasets_images)
      dataset_images.save_to_disk(path_save_dir_dataset_images)
      logger.info("Finished creating the dataset of all images")
      return dataset_images
  
imports: import glob
  import json
  import logging
  import math
  import os
  import tarfile
  from copy import deepcopy
  import git
  from datasets import Dataset, Image, Sequence, Value, concatenate_datasets, load_from_disk
  from pathos.multiprocessing import ProcessingPool
  from tqdm import tqdm
label_desc: Use image node_id 5 for calling a global function with example usage: create_dataset_images_from_tar(tar_paths, path_save_dir_tmp_datasets_images, num_proc, path_save_file_map_url_idx, path_save_dir_dataset_images) and returns: dataset_images, new_urls_indexed