code: def _create_ltor_masks_and_position_ids(
      tokens: torch.Tensor,
      eod_token: int,
      reset_position_ids: bool,
      reset_attention_mask: bool,
      eod_mask_loss: bool,
  ):
      """Create `attention_mask`, `loss_mask`, and `position_ids`.
  
      This function is modified :func:`get_ltor_masks_and_position_ids` in nemo/collections/nlp/modules/common/megatron/utils.py:
      `get_ltor_masks_and_position_ids` assumes a microbatch of ``tokens``, i.e. 2D tensor while
      this function assumes ``tokens`` to be 1D tensor.
  
      Args:
          tokens: A 1D tensor that holds the indices of tokens.
          eod_token:
          reset_position_ids:
          reset_attention_mask:
          eod_mask_loss
  
      """
      assert tokens.ndim == 1
      seq_length = tokens.numel()
      # `attention_mask` has the shape of [1, seq_length, seq_length]
      attention_mask = torch.tril(
          torch.ones((seq_length, seq_length))
      ).unsqueeze(0)
      loss_mask = torch.ones(seq_length, dtype=torch.float)
      if eod_mask_loss:
          loss_mask[tokens == eod_token] = 0.0
  
      position_ids = torch.arange(seq_length, dtype=torch.int64)
      if reset_position_ids:
          position_ids = position_ids.clone()
  
      if reset_position_ids or reset_attention_mask:
          # Find indices where EOD token is.
          eod_index = position_ids[tokens[b] == eod_token]
          # Detach indices from positions if going to modify positions.
          if reset_position_ids:
              eod_index = eod_index.clone()
          prev_index = 0
          for j in range(eod_index.numel()):
              i = eod_index[j]
              if reset_attention_mask:
                  attention_mask[0, (i + 1) :, : (i + 1)] = 0
              if reset_position_ids:
                  position_ids[(i + 1) :] -= i + 1 - prev_index
                  prev_index = i + 1
      # Convert attention mask to binary.
      attention_mask = attention_mask < 0.5
      return attention_mask, loss_mask, position_ids
  
imports: import os
  import time
  import numpy
  import torch
  from omegaconf.dictconfig import DictConfig
  from nemo.collections.nlp.data.language_modeling.megatron.base_dataset_utils import get_datasets_weights_and_num_samples, get_train_valid_test_split_
  from nemo.collections.nlp.data.language_modeling.megatron.blendable_dataset import BlendableDataset
  from nemo.collections.nlp.data.language_modeling.megatron.indexed_dataset import deallocate_indexed_dataset_memory
  from nemo.collections.nlp.data.language_modeling.megatron.indexed_dataset import make_dataset
  from nemo.core import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 5 for calling a global function with example usage: _create_ltor_masks_and_position_ids(tokens, eod_token, reset_position_ids, reset_attention_mask, eod_mask_loss) and returns: attention_mask,  loss_mask,  position_ids