class_docstr: Iterable dataset that returns constant length chunks of tokens from stream of text files.
      Args:
          tokenizer (Tokenizer): The processor used for proccessing the data.
          dataset (dataset.Dataset): Dataset with text files.
          infinite (bool): If True the iterator is reset after dataset reaches end else stops.
          seq_length (int): Length of token sequences to return.
          num_of_sequences (int): Number of token sequences to keep in buffer.
          chars_per_token (int): Number of characters per token used to estimate number of tokens in text buffer.
          tokenized (bool): If true we use a pretokenized dataset.
class_docstr_tok: ["Iterable","dataset","that","returns","constant","length","chunks","of","tokens","from","stream","of","text","files",".","Args",":","tokenizer","(","Tokenizer",")",":","The","processor","used","for","proccessing","the","data",".","dataset","(","dataset.Dataset",")",":","Dataset","with","text","files",".","infinite","(","bool",")",":","If","True","the","iterator","is","reset","after","dataset","reaches","end","else","stops",".","seq_length","(","int",")",":","Length","of","token","sequences","to","return",".","num_of_sequences","(","int",")",":","Number","of","token","sequences","to","keep","in","buffer",".","chars_per_token","(","int",")",":","Number","of","characters","per","token","used","to","estimate","number","of","tokens","in","text","buffer",".","tokenized","(","bool",")",":","If","true","we","use","a","pretokenized","dataset","."]
code: def __iter__(self):
      iterator = iter(self.dataset)
      more_examples = True
      while more_examples:
          buffer, buffer_len = [], 0
          while True:
              if buffer_len >= self.max_buffer_size:
                  break
              try:
                  buffer.append(next(iterator)[self.content_field])
                  buffer_len += len(buffer[-1])
              except StopIteration:
                  if self.infinite:
                      iterator = iter(self.dataset)
                      self.epoch += 1
                      logger.info(f"Dataset epoch: {self.epoch}")
                  else:
                      more_examples = False
                      break
          if self.tokenized:
              tokenized_inputs = buffer
          else:
              tokenized_inputs = self.tokenizer(
                  buffer, truncation=False
              )["input_ids"]
          all_token_ids = []
          for tokenized_input in tokenized_inputs:
              all_token_ids.extend(
                  tokenized_input + [self.concat_token_id]
              )
          for i in range(0, len(all_token_ids), self.seq_length):
              input_ids = all_token_ids[i : i + self.seq_length]
              if len(input_ids) == self.seq_length:
                  self.current_size += 1
                  yield torch.tensor(input_ids)
  
imports: import logging
  import os
  import time
  from argparse import Namespace
  from pathlib import Path
  import datasets
  import torch
  from accelerate import Accelerator, DistributedType
  from accelerate.utils import ProjectConfiguration
  from arguments import TrainingArguments
  from datasets import load_dataset
  from huggingface_hub import Repository
  from torch.optim import AdamW
  from torch.utils.data import IterableDataset
  from torch.utils.data.dataloader import DataLoader
  from torch.utils.data.datapipes.iter.combinatorics import ShufflerIterDataPipe
  import transformers
  from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, get_scheduler, set_seed
label_desc: Use image node_id 2 for calling the ConstantLengthDataset obj's underlying member method code with example usage: obj.__iter__() without return types