class_docstr: This class is for testing directly via get_regression_trainer
  
  It mixes in `TrainerIntegrationCommon` which already has a lot of helper validation methods
  which we can re-use here.
  
  Important: this class' setup can only work with a single gpu because it runs within the current
  pytest worker. For multi-gpu tests use TestDeepSpeedWithLauncher.
  
  Note: if any of the tests of this class get run there will be at least one gpu occupied by them
  until this pytest worker exits. This is because the gpu memory allocated by the cuda-kernels
  won't be released until this pytest worker exits.
  
  This may appear as some run-away tests if you watch `nvidia-smi` while other tests that fork new
  processes are run. So there will be one or two "stale" processes reported in `nvidia-smi`. This
  is not a bug.
class_docstr_tok: ["This","class","is","for","testing","directly","via","get_regression_trainer","It","mixes","in","`","TrainerIntegrationCommon","`","which","already","has","a","lot","of","helper","validation","methods","which","we","can","re-use","here",".","Important",":","this","class","'","setup","can","only","work","with","a","single","gpu","because","it","runs","within","the","current","pytest","worker",".","For","multi-gpu","tests","use","TestDeepSpeedWithLauncher",".","Note",":","if","any","of","the","tests","of","this","class","get","run","there","will","be","at","least","one","gpu","occupied","by","them","until","this","pytest","worker","exits",".","This","is","because","the","gpu","memory","allocated","by","the","cuda-kernels","wo","n't","be","released","until","this","pytest","worker","exits",".","This","may","appear","as","some","run-away","tests","if","you","watch","`","nvidia-smi","`","while","other","tests","that","fork","new","processes","are","run",".","So","there","will","be","one","or","two","``","stale","''","processes","reported","in","`","nvidia-smi","`",".","This","is","not","a","bug","."]
code: def test_gradient_accumulation(self, stage, dtype):
      # this test measures that we get identical weights and similar loss with:
      # 1. per_device_train_batch_size=8, gradient_accumulation_steps=1
      # 2. per_device_train_batch_size=4, gradient_accumulation_steps=2
      # since the 2nd should produce the effective batch of 1st, with the same results
      #
      # I can get an identical loss for a small train_len=32, plus the power of the initial
      # dynamic loss scale value set to:
      #   "fp16.initial_scale_power": 1
      # plus having the same WarmupLR's warmup_min_lr == warmup_max_lr in the config file
      # but for some reason going to train_len=64 the weights, weights start to mismatch with this setup.
      # the culprit seems to be `initial_scale_power` - putting it back to its default 32 keeps the weights identical
  
      train_len = 64
      a = b = 0.0
  
      kwargs = dict(
          a=a,
          b=b,
          local_rank=0,
          train_len=train_len,
          deepspeed=self.get_config_dict(stage),
      )
      kwargs[dtype] = True
  
      with mockenv_context(**self.dist_env_1_gpu):
          no_grad_accum_trainer = get_regression_trainer(
              **kwargs,
              per_device_train_batch_size=16,
              gradient_accumulation_steps=1,
          )
          no_grad_accum_result = no_grad_accum_trainer.train()
          no_grad_accum_loss = no_grad_accum_result.training_loss
          no_grad_accum_a = no_grad_accum_trainer.model.a.item()
          no_grad_accum_b = no_grad_accum_trainer.model.b.item()
          # make sure the optimizer kicked in - if it hasn't changed from the original value of a then make train_len bigger
          self.assertNotEqual(no_grad_accum_a, a)
  
      with mockenv_context(**self.dist_env_1_gpu):
          yes_grad_accum_trainer = get_regression_trainer(
              **kwargs,
              per_device_train_batch_size=4,
              gradient_accumulation_steps=4,
          )
          yes_grad_accum_result = yes_grad_accum_trainer.train()
          yes_grad_accum_loss = yes_grad_accum_result.training_loss
          yes_grad_accum_a = yes_grad_accum_trainer.model.a.item()
          yes_grad_accum_b = yes_grad_accum_trainer.model.b.item()
          self.assertNotEqual(yes_grad_accum_a, a)
  
      # training with half the batch size but accumulation steps as 2 should give the same
      # weights, but sometimes get a slight difference still of 1e-6
      self.assertAlmostEqual(
          no_grad_accum_a, yes_grad_accum_a, places=5
      )
      self.assertAlmostEqual(
          no_grad_accum_b, yes_grad_accum_b, places=5
      )
  
      # see the note above how to get identical loss on a small bs
      self.assertAlmostEqual(
          no_grad_accum_loss, yes_grad_accum_loss, places=2
      )
  
imports: import dataclasses
  import io
  import itertools
  import json
  import os
  import unittest
  from copy import deepcopy
  import datasets
  from parameterized import parameterized
  from tests.trainer.test_trainer import TrainerIntegrationCommon
  from transformers import AutoModel, TrainingArguments, is_torch_available, logging
  from transformers.deepspeed import HfDeepSpeedConfig, is_deepspeed_available, unset_hf_deepspeed_config
  from transformers.testing_utils import CaptureLogger, CaptureStd, CaptureStderr, LoggingLevel, TestCasePlus, execute_subprocess_async, get_gpu_count, mockenv_context, require_deepspeed, require_optuna, require_torch_gpu, require_torch_multi_gpu, slow
  from transformers.trainer_utils import get_last_checkpoint, set_seed
  from transformers.utils import WEIGHTS_NAME, is_torch_bf16_gpu_available
label_desc: Use image node_id 10 for calling the TrainerIntegrationDeepSpeed obj's underlying member method code with example usage: obj.test_gradient_accumulation(stage, dtype) without return types