class_docstr: Multi-headed attention from 'Attention Is All You Need' paper
class_docstr_tok: ["Multi-headed","attention","from","'Attention","Is","All","You","Need","'","paper"]
code: def __init__(
      self,
      embed_dim: int,
      num_heads: int,
      dropout: float = 0.0,
      is_decoder: bool = False,
      bias: bool = True,
  ):
      super().__init__()
      self.embed_dim = embed_dim
      self.num_heads = num_heads
      self.dropout = dropout
      self.head_dim = embed_dim // num_heads
  
      if (self.head_dim * num_heads) != self.embed_dim:
          raise ValueError(
              f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
              f" and `num_heads`: {num_heads})."
          )
      self.scaling = self.head_dim**-0.5
      self.is_decoder = is_decoder
  
      self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
      self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
      self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
      self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
  
imports: import math
  import random
  from typing import List, Optional, Tuple, Union
  import torch
  from torch import nn
  from torch.nn import CrossEntropyLoss
  from ...activations import ACT2FN
  from ...deepspeed import is_deepspeed_zero3_enabled
  from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPastAndCrossAttentions, Seq2SeqLMOutput, Seq2SeqModelOutput
  from ...modeling_utils import PreTrainedModel
  from ...utils import add_code_sample_docstrings, add_end_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings
  from .configuration_m2m_100 import M2M100Config
label_desc: Use image node_id 1 to create a new M2M100Attention object from inherited base classes: nn with example: obj = M2M100Attention(embed_dim, num_heads, dropout, is_decoder, bias)