class_docstr: Multi-headed attention.
  
  See "Attention Is All You Need" for more details.
class_docstr_tok: ["Multi-headed","attention",".","See","``","Attention","Is","All","You","Need","''","for","more","details","."]
code: def __init__(
      self,
      embed_dim,
      num_heads,
      kdim=None,
      vdim=None,
      dropout=0.0,
      bias=True,
      add_bias_kv=False,
      add_zero_attn=False,
      self_attention=False,
      encoder_decoder_attention=False,
      dictionary=None,
      q_noise=0.0,
      qn_block_size=8,
      # TODO: pass in config rather than string.
      # config defined in xformers.components.attention.AttentionConfig
      xformers_att_config: Optional[str] = None,
      xformers_blocksparse_layout: Optional[
          torch.Tensor
      ] = None,  # This should be part of the config
      xformers_blocksparse_blocksize: Optional[
          int
      ] = 16,  # This should be part of the config
  ):
      super().__init__(dictionary)
  
      xformers_att_config = utils.eval_str_dict(xformers_att_config)
      self.use_xformers = xformers_att_config is not None
      if self.use_xformers and not _xformers_available:
          raise ImportError("\n\n  Please install xFormers.")
      self.embed_dim = embed_dim
      self.kdim = kdim if kdim is not None else embed_dim
      self.vdim = vdim if vdim is not None else embed_dim
      self.qkv_same_dim = (
          self.kdim == embed_dim and self.vdim == embed_dim
      )
  
      self.num_heads = num_heads
      self.dropout_module = FairseqDropout(
          dropout, module_name=self.__class__.__name__
      )
  
      self.head_dim = embed_dim // num_heads
      assert (
          self.head_dim * num_heads == self.embed_dim
      ), "embed_dim must be divisible by num_heads"
      self.scaling = self.head_dim**-0.5
  
      self.self_attention = self_attention
      self.encoder_decoder_attention = encoder_decoder_attention
  
      assert not self.self_attention or self.qkv_same_dim, (
          "Self-attention requires query, key and "
          "value to be of the same size"
      )
  
      self.k_proj = quant_noise(
          nn.Linear(self.kdim, embed_dim, bias=bias),
          q_noise,
          qn_block_size,
      )
      self.v_proj = quant_noise(
          nn.Linear(self.vdim, embed_dim, bias=bias),
          q_noise,
          qn_block_size,
      )
      self.q_proj = quant_noise(
          nn.Linear(embed_dim, embed_dim, bias=bias),
          q_noise,
          qn_block_size,
      )
  
      self.out_proj = quant_noise(
          nn.Linear(embed_dim, embed_dim, bias=bias),
          q_noise,
          qn_block_size,
      )
  
      if add_bias_kv:
          self.bias_k = Parameter(torch.Tensor(1, 1, embed_dim))
          self.bias_v = Parameter(torch.Tensor(1, 1, embed_dim))
      else:
          self.bias_k = self.bias_v = None
  
      self.add_zero_attn = add_zero_attn
      self.beam_size = 1
      self.reset_parameters()
  
      if self.use_xformers:
          xformers_att_config["dropout"] = xformers_att_config.get(
              "dropout", dropout
          )
          xformers_att_config["num_heads"] = xformers_att_config.get(
              "num_heads", num_heads
          )
  
          if xformers_blocksparse_layout is not None:
              # Could be part of a single config passed only once
              xformers_att_config[
                  "block_size"
              ] = xformers_blocksparse_blocksize
              xformers_att_config[
                  "layout"
              ] = xformers_blocksparse_layout
              xformers_att_config["name"] = "blocksparse"
  
          self.attention = build_attention(xformers_att_config)
  
      self.onnx_trace = False
      self.skip_embed_dim_check = False
      self.init_incremental_state()
  
imports: import math
  from typing import Dict, List, Optional, Tuple
  import torch
  import torch.nn.functional
  from torch import Tensor, nn
  from torch.nn import Parameter
  from fairseq import utils
  from fairseq.modules.fairseq_dropout import FairseqDropout
  from fairseq.modules.quant_noise import quant_noise
  from fairseq.models.fairseq_incremental_decoder import FairseqIncrementalDecoder
label_desc: Use image node_id 1 to create a new MultiheadAttention object from inherited base classes: FairseqIncrementalDecoder with example: obj = MultiheadAttention(embed_dim, num_heads, kdim, vdim, dropout, bias, add_bias_kv, add_zero_attn, _attention, encoder_decoder_attention, dictionary, q_noise, qn_block_size, xformers_att_config, xformers_blocksparse_layout, xformers_blocksparse_blocksize)