class_docstr: Switch Transformers Feed Forward layer module. This is a wrapper around the Mixture of Experts module.
  
  Parameters:
      config : ([`GPTSanJapaneseConfig`]): Model configuration class with all the parameters of the model.
          Initializing with a config file does not load the weights associated with the model, only the
          configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
class_docstr_tok: ["Switch","Transformers","Feed","Forward","layer","module",".","This","is","a","wrapper","around","the","Mixture","of","Experts","module",".","Parameters",":","config",":","(","[","`","GPTSanJapaneseConfig","`","]",")",":","Model","configuration","class","with","all","the","parameters","of","the","model",".","Initializing","with","a","config","file","does","not","load","the","weights","associated","with","the","model",",","only","the","configuration",".","Check","out","the","[","`","~PreTrainedModel.from_pretrained","`","]","method","to","load","the","model","weights","."]
code: def __init__(self, config: GPTSanJapaneseConfig):
      super().__init__()
      self.mlp = GPTSanJapaneseSparseMLP(config)
      self.soft_bypass_mlp = nn.Linear(
          config.d_model, config.d_model, bias=False
      )
      self.norm = nn.LayerNorm(
          config.d_model, eps=config.layer_norm_epsilon
      )
  
imports: import copy
  from typing import List, Optional, Tuple, Union
  import torch
  import torch.nn
  from ...activations import ACT2FN
  from ...modeling_outputs import MoECausalLMOutputWithPast, MoEModelOutputWithPastAndCrossAttentions
  from ...modeling_utils import PreTrainedModel
  from ...utils import DUMMY_INPUTS, DUMMY_MASK, add_start_docstrings, add_start_docstrings_to_model_forward, is_torch_fx_proxy, logging
  from .configuration_gptsan_japanese import GPTSanJapaneseConfig
label_desc: Use image node_id 1 to create a new GPTSanJapaneseLayerSparseFF object from inherited base classes: nn with example: obj = GPTSanJapaneseLayerSparseFF(config)