docstr: Calling loss.backward() multiple times should sum up the gradients in .grad
  and accumulate all the individual gradients in .grad-sample
docstr_tok: ["Calling","loss.backward","(",")","multiple","times","should","sum","up","the","gradients","in",".grad","and","accumulate","all","the","individual","gradients","in",".grad-sample"]
code: def test_grad_sample_accumulation(self) -> None:
      """
      Calling loss.backward() multiple times should sum up the gradients in .grad
      and accumulate all the individual gradients in .grad-sample
      """
      gsm_class = get_gsm_class(self.GRAD_SAMPLE_MODE)
      grad_sample_module = gsm_class(self.model)
      data_iter = iter(self.dl)  # 4 batches of size 4 each
      self.model_forward_backward(
          grad_sample_module, data_iter, num_steps=8
      )
      # should accumulate grads in .grad and .grad_sample
  
      for p in self.model.parameters():
          if not p.requires_grad:
              self.assertFalse(hasattr(p, "grad_sample"))
              continue
  
          self.assertTrue(isinstance(p.grad_sample, list))
          self.assertEqual(len(p.grad_sample), 8)
  
          for gs in p.grad_sample:
              self.assertEqual(gs.shape[0], self.BATCH_SIZE)
  
      # the accumulated per-sample gradients
      per_sample_grads = torch.cat(
          [
              torch.cat(p.grad_sample).reshape(self.DATA_SIZE, -1)
              for p in self.model.parameters()
              if p.requires_grad
          ],
          dim=-1,
      )
      # average up all the per-sample gradients
      accumulated_grad = torch.mean(per_sample_grads, dim=0)
  
      # the full data gradient accumulated in .grad
      grad = torch.cat(
          [
              p.grad.reshape(-1)
              for p in self.model.parameters()
              if p.requires_grad
          ]
      ) * (self.BATCH_SIZE / self.DATA_SIZE)
  
      self.optimizer.step()
  
      # the accumulated gradients in .grad without any hooks
      orig_grad = self.effective_batch_grad
  
      self.assertTrue(
          torch.allclose(
              accumulated_grad, orig_grad, atol=10e-5, rtol=10e-3
          )
      )
      self.assertTrue(
          torch.allclose(grad, orig_grad, atol=10e-5, rtol=10e-3)
      )
  
imports: import unittest
  from typing import Iterable, Optional
  import torch
  import torch.nn
  import torch.nn.functional
  from opacus import PrivacyEngine
  from opacus.grad_sample.utils import get_gsm_class
  from opacus.utils.batch_memory_manager import BatchMemoryManager
  from torch.utils.data import DataLoader
  from torchvision import transforms
  from torchvision.datasets import FakeData
label_desc: Use image node_id 5 for calling the GradientAccumulationTest obj's underlying member method code with example usage: obj.test_grad_sample_accumulation() without return types