class_docstr: Megatron encoder-decoder base class
class_docstr_tok: ["Megatron","encoder-decoder","base","class"]
docstr: Reduce gradients across data parallel ranks.
  Modified from megatron-lm: https://github.com/NVIDIA/Megatron-LM/blob/d41696840ed0a7edb7e0499eb82a48ae112d9bb3/megatron/model/distributed.py#L188
docstr_tok: ["Reduce","gradients","across","data","parallel","ranks",".","Modified","from","megatron-lm",":","https",":","\/\/github.com\/NVIDIA\/Megatron-LM\/blob\/d41696840ed0a7edb7e0499eb82a48ae112d9bb3\/megatron\/model\/distributed.py","#","L188"]
code: def allreduce_gradients(self):
      """Reduce gradients across data parallel ranks.
      Modified from megatron-lm: https://github.com/NVIDIA/Megatron-LM/blob/d41696840ed0a7edb7e0499eb82a48ae112d9bb3/megatron/model/distributed.py#L188
      """
      # Bucketize and all-reduce
      buckets = {}
      # Pack the buckets.
      for param in self.parameters():
          if param.requires_grad and param.grad is not None:
              tp = param.data.type()
              if tp not in buckets:
                  buckets[tp] = []
              buckets[tp].append(param)
              # param.main_grad = param.grad
  
      # For each bucket, all-reduce and copy all-reduced grads.
      for tp in buckets:
          bucket = buckets[tp]
          grads = [param.grad.data for param in bucket]
          coalesced = torch._utils._flatten_dense_tensors(grads)
          coalesced /= parallel_state.get_data_parallel_world_size()
          torch.distributed.all_reduce(
              coalesced, group=parallel_state.get_data_parallel_group()
          )
          for buf, synced in zip(
              grads,
              torch._utils._unflatten_dense_tensors(coalesced, grads),
          ):
              buf.copy_(synced)
  
imports: import copy
  import functools
  import inspect
  from typing import Any, Dict, List, Optional
  import torch
  from omegaconf import OmegaConf, open_dict
  from omegaconf.dictconfig import DictConfig
  from pytorch_lightning.accelerators import CPUAccelerator
  from pytorch_lightning.trainer.trainer import Trainer
  from nemo.collections.nlp.data.language_modeling.megatron.data_samplers import MegatronPretrainingRandomSampler, MegatronPretrainingSampler
  from nemo.collections.nlp.models.language_modeling.megatron_base_model import MegatronBaseModel
  from nemo.collections.nlp.modules.common.megatron.build_model import build_model
  from nemo.collections.nlp.modules.common.megatron.module import Float16Module
  from nemo.collections.nlp.modules.common.megatron.token_level_encoder_decoder import MegatronTokenLevelEncoderDecoderModule
  from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults, average_losses_across_data_parallel_group, get_params_for_weight_decay_optimization
  from nemo.collections.nlp.modules.common.text_generation_utils import compute_beam_search_len_penalty, get_sampling_token_fn
  from nemo.collections.nlp.parts.utils_funcs import get_last_rank
  from nemo.utils import AppState, logging
label_desc: Use image node_id 15 for calling the MegatronLMEncoderDecoderModel obj's underlying member method code with example usage: obj.allreduce_gradients() without return types