class_docstr: NLP Adapter Mixin that can augment any transformer-based model with Adapter module support.
  This mixin class should be used only with a top level ModelPT subclass, that includes either a `model` or an `enc_dec_model` submodule.
  This mixin class adds several utility methods to add, load and save adapters.
  
  An Adapter module is any Pytorch nn.Module that possess a few properties :
  
  - It's input and output dimension are the same, while the hidden dimension need not be the same.
  - The final layer of the Adapter module is zero-initialized, so that the residual connection to the adapter yields the original output.
  
  This mixin class aims to integrate with PEFT, which is one or more adapters modules.
  The two features of PEFT, layer selection and weight tying, are also supported in this mixin class.
class_docstr_tok: ["NLP","Adapter","Mixin","that","can","augment","any","transformer-based","model","with","Adapter","module","support",".","This","mixin","class","should","be","used","only","with","a","top","level","ModelPT","subclass",",","that","includes","either","a","`","model","`","or","an","`","enc_dec_model","`","submodule",".","This","mixin","class","adds","several","utility","methods","to","add",",","load","and","save","adapters",".","An","Adapter","module","is","any","Pytorch","nn.Module","that","possess","a","few","properties",":","-","It","'s","input","and","output","dimension","are","the","same",",","while","the","hidden","dimension","need","not","be","the","same",".","-","The","final","layer","of","the","Adapter","module","is","zero-initialized",",","so","that","the","residual","connection","to","the","adapter","yields","the","original","output",".","This","mixin","class","aims","to","integrate","with","PEFT",",","which","is","one","or","more","adapters","modules",".","The","two","features","of","PEFT",",","layer","selection","and","weight","tying",",","are","also","supported","in","this","mixin","class","."]
code: def _check_and_add_peft_cfg(self, peft_cfg):
      layer_selection = peft_cfg.layer_selection
  
      assert not self.use_mcore_gpt or hasattr(
          peft_cfg, "name_key_to_mcore_mixins"
      ), f"{peft_cfg.__class__.__name__} is not supported in megatron core mode yet."
      name_key_to_mcore_mixins = (
          peft_cfg.name_key_to_mcore_mixins
          if self.use_mcore_gpt
          else None
      )
  
      for (
          adapter_name,
          adapter_cfg,
      ) in peft_cfg.get_config_dict().items():
          # self.mcore_gpt means is GPT and not T5
          if hasattr(self, "mcore_gpt") and not isinstance(
              adapter_cfg, PromptEncoderAdapterConfig
          ):
              if layer_selection is not None:
                  logging.info(
                      f"Layer selection {layer_selection} is enabled for the current model ("
                      f"{self.__class__.__name__} + {adapter_name})"
                  )
              if self.use_mcore_gpt:
                  if self.cfg.megatron_amp_O2:
                      layers = self.model.module.decoder.layers
                  else:
                      layers = self.model.decoder.layers
              else:
                  if self.cfg.megatron_amp_O2:
                      layers = (
                          self.model.module.language_model.encoder.layers
                      )
                  else:
                      layers = self.model.language_model.encoder.layers
              for layer in layers:
                  if layer.layer_number in (
                      layer_selection
                      or list(range(1, self.cfg.num_layers + 1))
                  ):
                      for name, module in layer.named_modules():
                          self._check_and_add_adapter(
                              name,
                              module,
                              adapter_name,
                              adapter_cfg,
                              name_key_to_mcore_mixins,
                          )
          else:
              # Non GPT models, as well as GPT+PTuning do not support layer selection
              if layer_selection is not None:
                  logging.warning(
                      "Layer selection is specified, but it is not supported for either "
                      f"{self.__class__.__name__} or {adapter_name})"
                  )
              for name, module in self.named_modules():
                  self._check_and_add_adapter(
                      name,
                      module,
                      adapter_name,
                      adapter_cfg,
                      name_key_to_mcore_mixins,
                  )
  
imports: import os
  import tempfile
  from typing import List, Optional, Union
  import torch
  from omegaconf import DictConfig, OmegaConf, open_dict
  from nemo.utils.model_utils import inject_model_parallel_rank
  from nemo.collections.nlp.modules.common.megatron.adapters.parallel_adapters import PromptEncoderAdapterConfig
  from nemo.collections.nlp.parts.peft_config import PEFT_CONFIG_MAP, CanonicalAdaptersPEFTConfig, LoraPEFTConfig, PEFTConfig, PtuningPEFTConfig
  from nemo.core.classes.mixins.adapter_mixins import AdapterModuleMixin
  from nemo.core.connectors.save_restore_connector import SaveRestoreConnector
  from nemo.utils import logging, model_utils
label_desc: Use image node_id 5 for calling the NLPAdapterModelMixin obj's underlying member method code with example usage: obj._check_and_add_peft_cfg(peft_cfg) without return types