class_docstr: MegatronGPTAdapterLearningModel is a model that combines a base model (GPTModel) with a adapters.
  This class only supports the canonical Adapter training described in Houlsby et al. (https://arxiv.org/pdf/1902.00751.pdf)
  
  Two adapter's are inserted into each Transformer layer in the base GPT Model.
  
  It is assumed that these set of adapters will then be trained for a specific task.
  Once trained, the adapter weights will be saved and can be re-loaded 
  and infused into the same GPT Model for inference. 
class_docstr_tok: ["MegatronGPTAdapterLearningModel","is","a","model","that","combines","a","base","model","(","GPTModel",")","with","a","adapters",".","This","class","only","supports","the","canonical","Adapter","training","described","in","Houlsby","et","al",".","(","https",":","\/\/arxiv.org\/pdf\/1902.00751.pdf",")","Two","adapter","'s","are","inserted","into","each","Transformer","layer","in","the","base","GPT","Model",".","It","is","assumed","that","these","set","of","adapters","will","then","be","trained","for","a","specific","task",".","Once","trained",",","the","adapter","weights","will","be","saved","and","can","be","re-loaded","and","infused","into","the","same","GPT","Model","for","inference","."]
code: def __init__(self, cfg: DictConfig, trainer: Trainer):
      super().__init__(cfg, trainer)
      assert (
          cfg.adapter_tuning.get("adapter_dim", 0) > 0
      ), "adapter_dim has not been set."
      assert (
          cfg.adapter_tuning.adapter_dim
          % cfg.tensor_model_parallel_size
          == 0
      ), "The adapter dim should be divisible by tensor_model_parallel_size."
      assert cfg.adapter_tuning.type in [
          "linear_adapter",
          "parallel_adapter",
      ], "Adapter type should be 'linear_adapter' or 'parallel_adapter'"
  
      self.adapter_name_keys = [
          AdapterName.PRE_ATTN_ADAPTER,
          AdapterName.POST_ATTN_ADAPTER,
      ]
      for _, layer in self.frozen_model.named_modules():
          if hasattr(layer, "activations_checkpoint_method"):
              layer.activations_checkpoint_method = None  # (@adithyare) adapter learning does not support activations checkpointing atm.
  
      logging.info(
          f"Before adding adapters:\n{self.frozen_model.summarize()}"
      )
  
      if cfg.adapter_tuning.type == "parallel_adapter":
          adapter_cfg = ParallelLinearAdapterConfig(
              in_features=self.frozen_model_cfg.hidden_size,
              out_features=self.frozen_model_cfg.hidden_size,
              dim=cfg.adapter_tuning.adapter_dim,
              norm_position=cfg.adapter_tuning.get(
                  "norm_position", "pre"
              ),
              norm_type=cfg.adapter_tuning.get(
                  "norm_type", "mixedfusedlayernorm"
              ),
              column_init_method=cfg.adapter_tuning.get(
                  "column_init_method", "xavier"
              ),
              row_init_method=cfg.adapter_tuning.get(
                  "row_init_method", "zero"
              ),
              dropout=cfg.adapter_tuning.adapter_dropout,
          )
      else:
          adapter_cfg = LinearAdapterConfig(
              in_features=self.frozen_model_cfg.hidden_size,
              dim=cfg.adapter_tuning.adapter_dim,
              norm_position=cfg.adapter_tuning.get(
                  "norm_position", "pre"
              ),
              dropout=cfg.adapter_tuning.adapter_dropout,
          )
  
      self.frozen_model.freeze()
      for _, module in self.frozen_model.named_modules():
          if isinstance(module, adapter_mixins.AdapterModuleMixin):
              for adapter_key in self.adapter_name_keys:
                  if (
                      model_utils.import_class_by_path(
                          adapter_cfg._target_
                      )
                      in module.get_accepted_adapter_types()
                  ):
                      module.add_adapter(
                          name=adapter_key,
                          cfg=adapter_cfg,
                      )
  
      logging.info(
          f"After adding adapters:\n{self.frozen_model.summarize()}"
      )
  
imports: import itertools
  import os
  import torch
  from omegaconf.dictconfig import DictConfig
  from pytorch_lightning.trainer.trainer import Trainer
  from nemo.collections.common.parts.adapter_modules import LinearAdapterConfig
  from nemo.collections.nlp.models.language_modeling.megatron_gpt_model import MegatronGPTModel
  from nemo.collections.nlp.models.language_modeling.megatron_gpt_prompt_learning_model import MegatronGPTPromptLearningModel
  from nemo.collections.nlp.modules.common import VirtualPromptStyle
  from nemo.collections.nlp.modules.common.megatron.adapters.parallel_adapters import AdapterName, InfusedAdapterConfig, MLPInfusedAdapterConfig, ParallelLinearAdapterConfig
  from nemo.collections.nlp.modules.common.megatron.utils import average_losses_across_data_parallel_group
  from nemo.collections.nlp.parts.nlp_overrides import NLPSaveRestoreConnector
  from nemo.collections.nlp.parts.utils_funcs import get_last_rank
  from nemo.core.classes.mixins import adapter_mixins
  from nemo.utils import logging, model_utils
label_desc: Use image node_id 1 to create a new MegatronGPTAdapterLearningModel object from inherited base classes: MegatronGPTBaseAdapterModel with example: obj = MegatronGPTAdapterLearningModel(cfg, trainer)