class_docstr: Utility class for 1B word benchmark dataset.
  
  The current implementation reads the data from the tokenized text
  files.
class_docstr_tok: ["Utility","class","for","1B","word","benchmark","dataset",".","The","current","implementation","reads","the","data","from","the","tokenized","text","files","."]
docstr: Read one file and convert to ids.
  
  Args:
    shard_name: file path.
  
  Returns:
    list of (id, char_id, global_word_id) tuples.
docstr_tok: ["Read","one","file","and","convert","to","ids",".","Args",":","shard_name",":","file","path",".","Returns",":","list","of","(","id",",","char_id",",","global_word_id",")","tuples","."]
code: def _load_shard(self, shard_name):
      """Read one file and convert to ids.
  
      Args:
        shard_name: file path.
  
      Returns:
        list of (id, char_id, global_word_id) tuples.
      """
      tf.compat.v1.logging.info("Loading data from: %s", shard_name)
      with tf.io.gfile.GFile(shard_name) as f:
          sentences = f.readlines()
      chars_ids = [
          self.vocab.encode_chars(sentence) for sentence in sentences
      ]
      ids = [self.vocab.encode(sentence) for sentence in sentences]
  
      global_word_ids = []
      current_idx = 0
      for word_ids in ids:
          current_size = len(word_ids) - 1  # without <BOS> symbol
          cur_ids = np.arange(current_idx, current_idx + current_size)
          global_word_ids.append(cur_ids)
          current_idx += current_size
  
      tf.compat.v1.logging.info("Loaded %d words.", current_idx)
      tf.compat.v1.logging.info("Finished loading")
      return zip(ids, chars_ids, global_word_ids)
  
imports: import random
  import numpy
  from textattack.shared.utils import LazyLoader
label_desc: Use image node_id 3 for calling the LM1BDataset obj's underlying member method code with example usage: obj._load_shard(shard_name) and returns: zip