docstr: Implements adapter modules directly with 3D tensor weight as parameters
  and without using ModuleList orto speed up training throughput.
docstr_tok: ["Implements","adapter","modules","directly","with","3D","tensor","weight","as","parameters","and","without","using","ModuleList","orto","speed","up","training","throughput","."]
code: def __init__(self, adapter_num, input_dim, hidden_dim, act_fn):
      """
      Implements adapter modules directly with 3D tensor weight as parameters
      and without using ModuleList orto speed up training throughput.
      """
      super().__init__()
  
      self.adapter_num = adapter_num
      self.input_dim = input_dim
      self.hidden_dim = hidden_dim
      self.W_a = nn.Parameter(
          torch.empty(adapter_num, hidden_dim, input_dim)
      )
      self.W_b = nn.Parameter(
          torch.empty(adapter_num, input_dim, hidden_dim)
      )
      self.b_a = nn.Parameter(torch.empty(adapter_num, hidden_dim))
      self.b_b = nn.Parameter(torch.empty(adapter_num, input_dim))
  
      self.ln_W = nn.Parameter(torch.empty(adapter_num, input_dim))
      self.ln_b = nn.Parameter(torch.empty(adapter_num, input_dim))
      self.act_fn = nn.Identity()
      if act_fn == "relu":
          self.act_fn = nn.ReLU()
      elif act_fn == "gelu":
          self.act_fn = nn.GELU()
      elif act_fn == "selu":
          self.act_fn = nn.SELU()
      else:
          raise ValueError(f"unsupported {act_fn}")
  
      self.input_dim = input_dim
      self.reset_parameters()
  
imports: import math
  from dataclasses import dataclass, field
  from typing import List, Tuple
  import numpy
  import torch
  import torch.nn
  import torch.nn.functional
  from fairseq import utils
  from fairseq.data.data_utils import compute_mask_indices
  from fairseq.dataclass import ChoiceEnum, FairseqDataclass
  from fairseq.distributed import fsdp_wrap
  from fairseq.models import BaseFairseqModel, register_model
  from fairseq.distributed.fully_sharded_data_parallel import FullyShardedDataParallel
  from fairseq.modules import Fp32GroupNorm, Fp32LayerNorm, GradMultiply, GumbelVectorQuantizer, LayerNorm, MultiheadAttention, RelPositionalEncoding, SamePad, TransposeLast
  from fairseq.modules.checkpoint_activations import checkpoint_wrapper
  from fairseq.modules.conformer_layer import ConformerWav2Vec2EncoderLayer
  from fairseq.modules.transformer_sentence_encoder import init_bert_params
  from fairseq.utils import buffered_arange, index_put, is_xla_tensor
  from .utils import pad_to_multiple
label_desc: Use image node_id 1 to create a new AdapterFast object from inherited base classes: nn with example: obj = AdapterFast(adapter_num, input_dim, hidden_dim, act_fn)