class_docstr: Multi-headed attention.
  
  See "Attention Is All You Need" for more details.
class_docstr_tok: ["Multi-headed","attention",".","See","``","Attention","Is","All","You","Need","''","for","more","details","."]
code: def _xformers_attn_forward(
      self,
      query,
      key: Optional[Tensor],
      value: Optional[Tensor],
      key_padding_mask: Optional[Tensor] = None,
      need_weights: bool = True,
      attn_mask: Optional[Tensor] = None,
  ) -> Tuple[Tensor, Optional[Tensor]]:
      tgt_len, bsz, embed_dim = query.size()
  
      if key_padding_mask is not None:
          assert key_padding_mask.size(0) == bsz
          assert key_padding_mask.size(1) == tgt_len
  
      if self.self_attention:
          key = query
          value = query
      elif self.encoder_decoder_attention:
          value = key
  
      q = self.q_proj(query)
      k = self.k_proj(key)
      v = self.v_proj(value)
  
      if self.bias_k is not None:
          assert self.bias_v is not None
          k, v, attn_mask, key_padding_mask = self._add_bias(
              k, v, attn_mask, key_padding_mask, bsz
          )
  
      def fold_heads(x):
          return (
              x.contiguous()
              .view(-1, bsz * self.num_heads, self.head_dim)
              .transpose(0, 1)
          )
  
      def split_heads(x):
          return (
              x.contiguous()
              .view(-1, bsz, self.num_heads, self.head_dim)
              .transpose(0, 1)
              .transpose(1, 2)
          )
  
      massage = (
          split_heads
          if self.attention.requires_head_dimension
          else fold_heads
      )
      q = massage(q)
      if k is not None:
          k = massage(k)
      if v is not None:
          v = massage(v)
  
      if self.add_zero_attn:
          k, v, key_padding_mask, attn_mask = self._append_zero_attn(
              k=k,
              v=v,
              key_padding_mask=key_padding_mask,
              attn_mask=attn_mask,
          )
  
      kwargs = {}
  
      if (
          attn_mask is not None
          and self.attention.supports_attention_mask
      ):
          attn_mask = _mask_for_xformers(attn_mask, to_dtype=q.dtype)
          kwargs["att_mask"] = attn_mask
  
      if key_padding_mask is not None:
          key_padding_mask = _mask_for_xformers(
              key_padding_mask, to_dtype=torch.bool
          )
          if not self.attention.requires_separate_masks:
              attn_mask = maybe_merge_masks(
                  attn_mask,
                  key_padding_mask,
                  batch_size=bsz,
                  src_len=k.size(-2),
                  tgt_len=q.size(-2),
                  num_heads=self.num_heads,
              )
              key_padding_mask = None
              kwargs["att_mask"] = attn_mask
          if self.attention.supports_key_padding_mask:
              kwargs["key_padding_mask"] = key_padding_mask
  
      y = self.attention(q, k, v, **kwargs)
  
      y = (
          y.view(bsz, self.num_heads, tgt_len, self.head_dim)
          .transpose(1, 2)
          .flatten(start_dim=2, end_dim=3)
          .transpose(0, 1)
      )
      assert list(y.size()) == [tgt_len, bsz, embed_dim]
  
      # Dropout not needed because already applied in attention.
      # It is applied to the attention weights before matmul with v.
      y = self.out_proj(y)
  
      # TODO: support returning attention weights if needed.
      return y, None
  
imports: import math
  from typing import Dict, List, Optional, Tuple
  import torch
  import torch.nn.functional
  from torch import Tensor, nn
  from torch.nn import Parameter
  from fairseq import utils
  from fairseq.modules.fairseq_dropout import FairseqDropout
  from fairseq.modules.quant_noise import quant_noise
  from fairseq.models.fairseq_incremental_decoder import FairseqIncrementalDecoder
label_desc: Use image node_id 10 for calling the MultiheadAttention obj's underlying member method code with example usage: obj._xformers_attn_forward(query, key, value, key_padding_mask, need_weights, attn_mask) and returns: y,  None, x, x