class_docstr: Iterable dataset that returns constant length chunks of tokens from stream of text files.
      Args:
          tokenizer (Tokenizer): The processor used for proccessing the data.
          dataset (dataset.Dataset): Dataset with text files.
          infinite (bool): If True the iterator is reset after dataset reaches end else stops.
          seq_length (int): Length of token sequences to return.
          num_of_sequences (int): Number of token sequences to keep in buffer.
          chars_per_token (int): Number of characters per token used to estimate number of tokens in text buffer.
          tokenized (bool): If true we use a pretokenized dataset.
class_docstr_tok: ["Iterable","dataset","that","returns","constant","length","chunks","of","tokens","from","stream","of","text","files",".","Args",":","tokenizer","(","Tokenizer",")",":","The","processor","used","for","proccessing","the","data",".","dataset","(","dataset.Dataset",")",":","Dataset","with","text","files",".","infinite","(","bool",")",":","If","True","the","iterator","is","reset","after","dataset","reaches","end","else","stops",".","seq_length","(","int",")",":","Length","of","token","sequences","to","return",".","num_of_sequences","(","int",")",":","Number","of","token","sequences","to","keep","in","buffer",".","chars_per_token","(","int",")",":","Number","of","characters","per","token","used","to","estimate","number","of","tokens","in","text","buffer",".","tokenized","(","bool",")",":","If","true","we","use","a","pretokenized","dataset","."]
code: def __init__(
      self,
      tokenizer,
      dataset,
      infinite=False,
      seq_length=1024,
      num_of_sequences=1024,
      chars_per_token=3.6,
      tokenized=False,
  ):
      self.tokenizer = tokenizer
      self.concat_token_id = tokenizer.bos_token_id
      self.dataset = dataset
      self.seq_length = seq_length
      self.epoch = 0
      self.infinite = infinite
      self.current_size = 0
      self.tokenized = tokenized
  
      if self.tokenized:
          self.max_buffer_size = seq_length * num_of_sequences
          self.content_field = "input_ids"
      else:
          self.max_buffer_size = (
              seq_length * chars_per_token * num_of_sequences
          )
          self.content_field = "content"
  
imports: import logging
  import os
  import time
  from argparse import Namespace
  from pathlib import Path
  import datasets
  import torch
  from accelerate import Accelerator, DistributedType
  from accelerate.utils import ProjectConfiguration
  from arguments import TrainingArguments
  from datasets import load_dataset
  from huggingface_hub import Repository
  from torch.optim import AdamW
  from torch.utils.data import IterableDataset
  from torch.utils.data.dataloader import DataLoader
  from torch.utils.data.datapipes.iter.combinatorics import ShufflerIterDataPipe
  import transformers
  from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, get_scheduler, set_seed
label_desc: Use image node_id 1 to create a new ConstantLengthDataset object from inherited base classes: IterableDataset with example: obj = ConstantLengthDataset(tokenizer, dataset, infinite, seq_length, num_of_sequences, chars_per_token, tokenized)