class_docstr: Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`]
  
  Args:
      config: LlamaConfig
class_docstr_tok: ["Transformer","decoder","consisting","of","*","config.num_hidden_layers","*","layers",".","Each","layer","is","a","[","`","LlamaDecoderLayer","`","]","Args",":","config",":","LlamaConfig"]
code: def _prepare_decoder_attention_mask(
      self,
      attention_mask,
      input_shape,
      inputs_embeds,
      past_key_values_length,
  ):
      # create causal mask
      # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
      combined_attention_mask = None
      if input_shape[-1] > 1:
          combined_attention_mask = _make_causal_mask(
              input_shape,
              inputs_embeds.dtype,
              device=inputs_embeds.device,
              past_key_values_length=past_key_values_length,
          )
  
      if attention_mask is not None:
          # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
          expanded_attn_mask = _expand_mask(
              attention_mask,
              inputs_embeds.dtype,
              tgt_len=input_shape[-1],
          ).to(inputs_embeds.device)
          combined_attention_mask = (
              expanded_attn_mask
              if combined_attention_mask is None
              else expanded_attn_mask + combined_attention_mask
          )
  
      return combined_attention_mask
  
imports: import math
  from typing import List, Optional, Tuple, Union
  import torch
  import torch.nn.functional
  import torch.utils.checkpoint
  from torch import nn
  from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
  from transformers.modeling_utils import PreTrainedModel
  from transformers.models.llama.modeling_llama import LlamaConfig, LlamaRMSNorm
  from transformers.utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings
  from colossalai.kernel.cuda_native.mha.flash_attn_2 import HAS_FLASH_ATTN
  from colossalai.kernel.triton.llama_act_combine_kernel import HAS_TRITON
  from colossalai.moe.layers import SparseMLP
  from colossalai.moe.manager import MOE_MANAGER
  from colossalai.moe.utils import get_activation, set_moe_args
label_desc: Use image node_id 4 for calling the OpenMoeModel obj's underlying member method code with example usage: obj._prepare_decoder_attention_mask(attention_mask, input_shape, inputs_embeds, past_key_values_length) and returns: combined_attention_mask