code: def __init__(
      self,
      cfg,
      trainer,
      tokenizer,
      name,
      data_prefix,
      documents,
      indexed_dataset,
      num_samples,
      seq_length,
      seed,
      drop_last=True,
  ):
      if not HAVE_MEGATRON_CORE:
          raise ImportError(
              "megatron-core was not found. Please see the NeMo README for installation instructions: https://github.com/NVIDIA/NeMo#megatron-gpt."
          )
  
      super().__init__()
      self.name = name
      self.indexed_dataset = indexed_dataset
      self.drop_last = drop_last
      self.seq_length = seq_length
      self.get_attention_mask_from_fusion = cfg.get(
          "get_attention_mask_from_fusion", True
      )
  
      # Checks
      assert np.min(documents) >= 0
      assert np.max(documents) < indexed_dataset.sizes.shape[0]
  
      self.reset_position_ids = cfg.data.get(
          "reset_position_ids", False
      )
      self.reset_attention_mask = cfg.data.get(
          "reset_attention_mask", False
      )
      self.eod_mask_loss = cfg.data.get("eod_mask_loss", False)
      self.create_inputs = any(
          [
              self.reset_position_ids,
              self.reset_attention_mask,
              self.eod_mask_loss,
          ]
      )
      self.cached_inputs = False
      self.eos_id = tokenizer.eos_id
      self.no_seqlen_plus_one_input_tokens = cfg.data.get(
          "no_seqlen_plus_one_input_tokens", False
      )
      self.add_extra_token = 1
      if self.no_seqlen_plus_one_input_tokens:
          self.add_extra_token = 0
      self.shuffle_documents = cfg.data.get("shuffle_documents", True)
      self.exchange_indices_distributed = cfg.data.get(
          "exchange_indices_distributed", False
      )
  
      # save index mappings to a configurable dir
      self.index_mapping_dir = cfg.data.get("index_mapping_dir", None)
  
      # create index_mapping_dir on rank 0
      if (
          torch.distributed.is_available()
          and torch.distributed.is_initialized()
      ):
          if torch.distributed.get_rank() == 0:
              if (
                  self.index_mapping_dir is not None
                  and not os.path.isdir(self.index_mapping_dir)
              ):
                  os.makedirs(self.index_mapping_dir)
          torch.distributed.barrier()
  
      # Build index mappings.
      (
          self.doc_idx,
          self.sample_idx,
          self.shuffle_idx,
      ) = _build_index_mappings(
          self.name,
          data_prefix,
          documents,
          self.indexed_dataset.sizes,
          num_samples,
          seq_length,
          seed,
          index_mapping_dir=self.index_mapping_dir,
          drop_last=drop_last,
          add_extra_token=self.add_extra_token,
          shuffle_documents=self.shuffle_documents,
          exchange_indices_distributed=self.exchange_indices_distributed,
      )
      deallocate_indexed_dataset_memory(self.indexed_dataset)
  
imports: import os
  import time
  import numpy
  import torch
  from omegaconf.dictconfig import DictConfig
  from nemo.collections.nlp.data.language_modeling.megatron.base_dataset_utils import get_datasets_weights_and_num_samples, get_train_valid_test_split_
  from nemo.collections.nlp.data.language_modeling.megatron.blendable_dataset import BlendableDataset
  from nemo.collections.nlp.data.language_modeling.megatron.indexed_dataset import deallocate_indexed_dataset_memory
  from nemo.collections.nlp.data.language_modeling.megatron.indexed_dataset import make_dataset
  from nemo.core import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 1 to create a new GPTDataset object from inherited base classes: Dataset with example: obj = GPTDataset(cfg, trainer, tokenizer, name, data_prefix, documents, indexed_dataset, num_samples, seq_length, seed, drop_last)