class_docstr: FFN Layer for Switch Transformer and Extra layers
  
  GPTSAN can mix Switch Transformer layers and normal Transformer layers This class is used as Expert in Switch
  Transformer layers and as FFN in regular Transformer layers. RELU is used in the Switch Transformer layer, and
  Swish is used in the normal Transformer layer, so there is a choice of which is used in the argument.
class_docstr_tok: ["FFN","Layer","for","Switch","Transformer","and","Extra","layers","GPTSAN","can","mix","Switch","Transformer","layers","and","normal","Transformer","layers","This","class","is","used","as","Expert","in","Switch","Transformer","layers","and","as","FFN","in","regular","Transformer","layers",".","RELU","is","used","in","the","Switch","Transformer","layer",",","and","Swish","is","used","in","the","normal","Transformer","layer",",","so","there","is","a","choice","of","which","is","used","in","the","argument","."]
docstr: Args:
      hidden_states (`torch.Tensor`) :
          [num_groups, tokens_per_group, hidden_dim] inputs to send to experts.
  Returns:
      torch.Tensor[num_groups, tokens_per_group, hidden_dim]
docstr_tok: ["Args",":","hidden_states","(","`","torch.Tensor","`",")",":","[","num_groups",",","tokens_per_group",",","hidden_dim","]","inputs","to","send","to","experts",".","Returns",":","torch.Tensor","[","num_groups",",","tokens_per_group",",","hidden_dim","]"]
code: def forward(self, hidden_states):
      r"""
      Args:
          hidden_states (`torch.Tensor`) :
              [num_groups, tokens_per_group, hidden_dim] inputs to send to experts.
      Returns:
          torch.Tensor[num_groups, tokens_per_group, hidden_dim]
  
      """
      hidden_states = self.wi(hidden_states)
      hidden_states = self.act(hidden_states)
      hidden_states = self.dropout(hidden_states)
      hidden_states = self.wo(hidden_states)
      return hidden_states
  
imports: import copy
  from typing import List, Optional, Tuple, Union
  import torch
  import torch.nn
  from ...activations import ACT2FN
  from ...modeling_outputs import MoECausalLMOutputWithPast, MoEModelOutputWithPastAndCrossAttentions
  from ...modeling_utils import PreTrainedModel
  from ...utils import DUMMY_INPUTS, DUMMY_MASK, add_start_docstrings, add_start_docstrings_to_model_forward, is_torch_fx_proxy, logging
  from .configuration_gptsan_japanese import GPTSanJapaneseConfig
label_desc: Use image node_id 2 for calling the GPTSanJapaneseDenseActDense obj's underlying member method code with example usage: obj.forward(hidden_states) and returns: hidden_states