class_docstr: Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`]
  
  Args:
      config: LlamaConfig
class_docstr_tok: ["Transformer","decoder","consisting","of","*","config.num_hidden_layers","*","layers",".","Each","layer","is","a","[","`","LlamaDecoderLayer","`","]","Args",":","config",":","LlamaConfig"]
code: def __init__(self, config: LlamaConfig):
      super().__init__(config)
      self.padding_idx = config.pad_token_id
      self.vocab_size = config.vocab_size
  
      self.embed_tokens = nn.Embedding(
          config.vocab_size, config.hidden_size, self.padding_idx
      )
      self.layers = nn.ModuleList(
          [
              OpenMoeDecoderLayer(
                  config,
                  moe=True
                  if (i + 1) % config.moe_layer_interval == 0
                  else False,
              )
              for i in range(config.num_hidden_layers)
          ]
      )
      self.norm = LlamaRMSNorm(
          config.hidden_size, eps=config.rms_norm_eps
      )
  
      self.gradient_checkpointing = False
      # Initialize weights and apply final processing
      self.post_init()
  
imports: import math
  from typing import List, Optional, Tuple, Union
  import torch
  import torch.nn.functional
  import torch.utils.checkpoint
  from torch import nn
  from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
  from transformers.modeling_utils import PreTrainedModel
  from transformers.models.llama.modeling_llama import LlamaConfig, LlamaRMSNorm
  from transformers.utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings
  from colossalai.kernel.cuda_native.mha.flash_attn_2 import HAS_FLASH_ATTN
  from colossalai.kernel.triton.llama_act_combine_kernel import HAS_TRITON
  from colossalai.moe.layers import SparseMLP
  from colossalai.moe.manager import MOE_MANAGER
  from colossalai.moe.utils import get_activation, set_moe_args
label_desc: Use image node_id 1 to create a new OpenMoeModel object from inherited base classes: OpenMoePreTrainedModel with example: obj = OpenMoeModel(config)