class_docstr: Multi-headed attention block.
  
  Args:
      num_heads (int): number of attention heads
      dim_model (int): input/output dimensionality
      dropout (float): the dropout probability
      mask (torch.Tensor): the mask tensor
      device: (Optional[torch.device]).
  
  Example::
  
      self.attention = MultiHeadedAttention(
          num_heads=attn_heads, dim_model=hidden, dropout=dropout, device=device
      )
      self.attention.forward(query, key, value, mask=mask)
class_docstr_tok: ["Multi-headed","attention","block",".","Args",":","num_heads","(","int",")",":","number","of","attention","heads","dim_model","(","int",")",":","input\/output","dimensionality","dropout","(","float",")",":","the","dropout","probability","mask","(","torch.Tensor",")",":","the","mask","tensor","device",":","(","Optional","[","torch.device","]",")",".","Example",":",":","self.attention","=","MultiHeadedAttention","(","num_heads=attn_heads",",","dim_model=hidden",",","dropout=dropout",",","device=device",")","self.attention.forward","(","query",",","key",",","value",",","mask=mask",")"]
code: def __init__(
      self,
      num_heads: int,
      dim_model: int,
      dropout: float = 0.1,
      device: Optional[torch.device] = None,
  ) -> None:
      super().__init__()
      assert dim_model % num_heads == 0
      # We assume d_v always equals d_k
      self.d_k: int = dim_model // num_heads
      self.num_heads = num_heads
      self.linear_layers = nn.ModuleList(
          [
              nn.Linear(dim_model, dim_model, device=device)
              for _ in range(3)
          ]
      )
      self.output_linear = nn.Linear(
          dim_model, dim_model, device=device
      )
      self.attention = Attention()
      self.dropout = nn.Dropout(p=dropout)
  
imports: import copy
  import math
  from typing import Callable, Optional, Tuple
  import torch
  import torch.nn
  from torchrec.modules.embedding_configs import EmbeddingConfig
  from torchrec.modules.embedding_modules import EmbeddingCollection
  from torchrec.sparse.jagged_tensor import KeyedJaggedTensor
label_desc: Use image node_id 1 to create a new MultiHeadedAttention object from inherited base classes: nn with example: obj = MultiHeadedAttention(num_heads, dim_model, dropout, device)