class_docstr: FFN Layer for Switch Transformer and Extra layers
  
  GPTSAN can mix Switch Transformer layers and normal Transformer layers This class is used as Expert in Switch
  Transformer layers and as FFN in regular Transformer layers. RELU is used in the Switch Transformer layer, and
  Swish is used in the normal Transformer layer, so there is a choice of which is used in the argument.
class_docstr_tok: ["FFN","Layer","for","Switch","Transformer","and","Extra","layers","GPTSAN","can","mix","Switch","Transformer","layers","and","normal","Transformer","layers","This","class","is","used","as","Expert","in","Switch","Transformer","layers","and","as","FFN","in","regular","Transformer","layers",".","RELU","is","used","in","the","Switch","Transformer","layer",",","and","Swish","is","used","in","the","normal","Transformer","layer",",","so","there","is","a","choice","of","which","is","used","in","the","argument","."]
code: def __init__(self, config: GPTSanJapaneseConfig, ext_layer=False):
      super().__init__()
      d_inter = config.d_ext if ext_layer else config.d_ff
      self.wi = nn.Linear(config.d_model, d_inter, bias=ext_layer)
      self.wo = nn.Linear(d_inter, config.d_model, bias=ext_layer)
      self.dropout = (
          nn.Identity()
          if ext_layer
          else nn.Dropout(config.dropout_rate)
      )
      self.act = ACT2FN["swish" if ext_layer else "relu"]
  
imports: import copy
  from typing import List, Optional, Tuple, Union
  import torch
  import torch.nn
  from ...activations import ACT2FN
  from ...modeling_outputs import MoECausalLMOutputWithPast, MoEModelOutputWithPastAndCrossAttentions
  from ...modeling_utils import PreTrainedModel
  from ...utils import DUMMY_INPUTS, DUMMY_MASK, add_start_docstrings, add_start_docstrings_to_model_forward, is_torch_fx_proxy, logging
  from .configuration_gptsan_japanese import GPTSanJapaneseConfig
label_desc: Use image node_id 1 to create a new GPTSanJapaneseDenseActDense object from inherited base classes: nn with example: obj = GPTSanJapaneseDenseActDense(config, ext_layer)