docstr: Load a given dataset split (e.g., train, valid, test)
docstr_tok: ["Load","a","given","dataset","split","(","e.g.",",","train",",","valid",",","test",")"]
code: def load_dataset(self, split, epoch=1, combine=False, **kwargs):
      """Load a given dataset split (e.g., train, valid, test)"""
  
      # support sharded datasets
      paths = utils.split_paths(self.cfg.data)
      assert len(paths) > 0
      data_path = paths[(epoch - 1) % len(paths)]
      split_path = os.path.join(data_path, split)
  
      # each element of *data* will be a tensorized line from the original
      # text dataset, similar to ``open(split_path).readlines()``
      data = data_utils.load_indexed_dataset(
          split_path, self.dictionary, combine=combine
      )
      if data is None:
          raise FileNotFoundError(
              "Dataset not found: {} ({})".format(split, split_path)
          )
  
      # this is similar to ``data.view(-1).split(tokens_per_sample)``
      data = TokenBlockDataset(
          data,
          data.sizes,
          block_size=self.cfg.tokens_per_sample,
          pad=None,  # unused
          eos=None,  # unused
          break_mode="none",
      )
  
      self.datasets[split] = TruncatedBPTTDataset(
          data=data,
          bsz_per_shard=self.cfg.batch_size,
          shard_id=self.cfg.data_parallel_rank,
          num_shards=self.cfg.data_parallel_size,
      )
  
imports: import logging
  import os
  from dataclasses import dataclass, field
  from typing import List, Optional, Tuple
  import torch
  from fairseq import utils
  from fairseq.data import Dictionary, TokenBlockDataset, data_utils, iterators
  from fairseq.dataclass import FairseqDataclass
  from fairseq.distributed import utils
  from fairseq.tasks import FairseqTask, register_task
  from omegaconf import II
label_desc: Use image node_id 2 for calling the TruncatedBPTTLMTask obj's underlying member method code with example usage: obj.load_dataset(split, epoch, combine) without return types