class_docstr: Multi-headed attention from 'Attention Is All You Need' paper
class_docstr_tok: ["Multi-headed","attention","from","'Attention","Is","All","You","Need","'","paper"]
code: def __init__(self, config: LlamaConfig):
      super().__init__()
      self.config = config
      self.hidden_size = config.hidden_size
      self.num_heads = config.num_attention_heads
      self.head_dim = self.hidden_size // self.num_heads
      self.max_position_embeddings = config.max_position_embeddings
  
      if (self.head_dim * self.num_heads) != self.hidden_size:
          raise ValueError(
              f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
              f" and `num_heads`: {self.num_heads})."
          )
      self.q_proj = nn.Linear(
          self.hidden_size, self.num_heads * self.head_dim, bias=False
      )
      self.k_proj = nn.Linear(
          self.hidden_size, self.num_heads * self.head_dim, bias=False
      )
      self.v_proj = nn.Linear(
          self.hidden_size, self.num_heads * self.head_dim, bias=False
      )
      self.o_proj = nn.Linear(
          self.num_heads * self.head_dim, self.hidden_size, bias=False
      )
      self.rotary_emb = LlamaRotaryEmbedding(
          self.head_dim,
          max_position_embeddings=self.max_position_embeddings,
      )
  
imports: import math
  from typing import List, Optional, Tuple, Union
  import torch
  import torch.utils.checkpoint
  from torch import nn
  from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
  from transformers.activations import ACT2FN
  from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast, SequenceClassifierOutputWithPast
  from transformers.modeling_utils import PreTrainedModel
  from transformers.utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings
  from transformers.models.llama.configuration_llama import LlamaConfig
label_desc: Use image node_id 1 to create a new LlamaAttention object from inherited base classes: nn with example: obj = LlamaAttention(config)