class_docstr: Megatron encoder-decoder base class
class_docstr_tok: ["Megatron","encoder-decoder","base","class"]
code: def __init__(self, cfg: DictConfig, trainer: Trainer):
      super().__init__(cfg, trainer=trainer)
      if cfg.get("pipeline_model_parallel_size", 1) > 1:
          if cfg.get("pipeline_model_parallel_split_rank", 0) <= 0:
              raise ValueError(
                  f"pipeline_model_parallel_split_rank must be > 0 when using pipeline_model_parallel_size > 1"
              )
      if cfg.get("pipeline_model_parallel_size", 1) > 1:
          if not cfg.get("share_token_embeddings", True) or not cfg.get(
              "share_decoder_tokens_head_embeddings", True
          ):
              raise ValueError(
                  "when pipeline_model_parallel_size > 1 we require share_token_embeddings=True and share_decoder_tokens_head_embeddings=True"
              )
  
      # Make sure trainer.accumulate_grad_batches is 1.
      self._validate_trainer()
  
      # TODO: Currently does not support interleaved pipeline parallelism.
      # This means we can only use pipeline parallelism without the interleaved schedule.
      if isinstance(self.trainer.accelerator, CPUAccelerator):
          logging.warning(
              "Using CPUAccelerator, model will be built on CPU."
          )
          self.enc_dec_model = build_model(
              model_provider_func=self.model_provider_func,
              wrap_with_ddp=False,
              on_cpu=True,
              model_type=ModelType.encoder_and_decoder,
          )[0]
      else:
          self.enc_dec_model = build_model(
              model_provider_func=self.model_provider_func,
              wrap_with_ddp=False,
              model_type=ModelType.encoder_and_decoder,
          )[0]
  
      # We don't need to call it explicitly? Since it is a pytorch lightning hook function
      # self.setup_optimizer_param_groups()
  
      self.megatron_amp_O2 = cfg.get("megatron_amp_O2", False)
  
      if self.megatron_amp_O2:
          if not self.with_distributed_adam:
              # Pre-allocate the model on GPU to have master parameters allocated on the same device with matching data type
              self.enc_dec_model.cuda(torch.cuda.current_device())
  
          # Model wrapper to convert both model and inputs to half precision
          self.enc_dec_model = Float16Module(
              config=self.model_parallel_config,
              module=self.enc_dec_model,
              precision=self.cfg.precision,
          )
  
      self.enable_autocast = (
          True
          if (not self.megatron_amp_O2)
          and (self.autocast_dtype in [torch.float16, torch.bfloat16])
          else False
      )
  
      self.enc_dec_model.model_type = ModelType.encoder_and_decoder
  
imports: import copy
  import functools
  import inspect
  from typing import Any, Dict, List, Optional
  import torch
  from omegaconf import OmegaConf, open_dict
  from omegaconf.dictconfig import DictConfig
  from pytorch_lightning.accelerators import CPUAccelerator
  from pytorch_lightning.trainer.trainer import Trainer
  from nemo.collections.nlp.data.language_modeling.megatron.data_samplers import MegatronPretrainingRandomSampler, MegatronPretrainingSampler
  from nemo.collections.nlp.models.language_modeling.megatron_base_model import MegatronBaseModel
  from nemo.collections.nlp.modules.common.megatron.build_model import build_model
  from nemo.collections.nlp.modules.common.megatron.module import Float16Module
  from nemo.collections.nlp.modules.common.megatron.token_level_encoder_decoder import MegatronTokenLevelEncoderDecoderModule
  from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults, average_losses_across_data_parallel_group, get_params_for_weight_decay_optimization
  from nemo.collections.nlp.modules.common.text_generation_utils import compute_beam_search_len_penalty, get_sampling_token_fn
  from nemo.collections.nlp.parts.utils_funcs import get_last_rank
  from nemo.utils import AppState, logging
label_desc: Use image node_id 1 to create a new MegatronLMEncoderDecoderModel object from inherited base classes: MegatronBaseModel with example: obj = MegatronLMEncoderDecoderModel(cfg, trainer)