class_docstr: A manager class to manipulate the tensors in chunks.
  
  Args:
      chunk_configuration (Dict[int, Dict]): the configuration dictionary of this chunk manager.
      init_device (torch.device): optional, the device on which the chunk is initialized. The default is None.
class_docstr_tok: ["A","manager","class","to","manipulate","the","tensors","in","chunks",".","Args",":","chunk_configuration","(","Dict","[","int",",","Dict","]",")",":","the","configuration","dictionary","of","this","chunk","manager",".","init_device","(","torch.device",")",":","optional",",","the","device","on","which","the","chunk","is","initialized",".","The","default","is","None","."]
docstr: Rearrange gradients accumulated in chunk.grad_chunk, and getP prepared for gradient reduction.
docstr_tok: ["Rearrange","gradients","accumulated","in","chunk.grad_chunk",",","and","getP","prepared","for","gradient","reduction","."]
code: def rearrange_accumulated_grad_chunk(self, chunk: Chunk) -> Chunk:
      """Rearrange gradients accumulated in chunk.grad_chunk, and getP prepared for gradient reduction."""
  
      assert chunk.grad_chunk is not None
  
      # Make a backup for gradient accumulated before.
      # Here backup gradients should be multiplied, since it will be divided after gradient reduction.
      if chunk.grad_chunk.is_gathered:
          accumulated_grad = (
              chunk.grad_chunk.cuda_global_chunk.clone()
              .detach()
              .mul_(chunk.pg_size)
          )
          accumulated_grad_gathered = True
      else:
          if chunk.grad_chunk.cuda_shard is not None:
              accumulated_grad = (
                  chunk.grad_chunk.cuda_shard.clone()
                  .detach()
                  .mul_(chunk.pg_size)
              )
          else:
              accumulated_grad = (
                  chunk.grad_chunk.cpu_shard.to(get_current_device())
                  .clone()
                  .detach()
                  .mul_(chunk.pg_size)
              )
          accumulated_grad_gathered = False
  
      # Reset grad_chunk, and chunk.grad_chunk will be accessed.
      grad_chunk = self.init_grad_chunk(chunk)
      grad_chunk.cuda_global_chunk.zero_()
  
      # Add backup gradients to grad_chunk.
      if accumulated_grad_gathered:
          grad_chunk.cuda_global_chunk.add_(accumulated_grad)
      else:
          grad_chunk.cuda_global_chunk[
              grad_chunk.shard_begin : grad_chunk.shard_end
          ].add_(accumulated_grad)
  
      # Release accumulated_grad
      free_storage(accumulated_grad)
  
      return grad_chunk
  
imports: from collections import deque
  from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple
  import torch
  import torch.distributed
  from torch.distributed import ProcessGroup
  from colossalai.utils import free_storage, get_current_device
  from .chunk import Chunk, ChunkFullError, TensorState
label_desc: Use image node_id 23 for calling the ChunkManager obj's underlying member method code with example usage: obj.rearrange_accumulated_grad_chunk(chunk) and returns: grad_chunk