class_docstr: RNN decoder with multihead attention. Attention is calculated using encoder
  output and output of decoder's first RNN layerself. Attention is applied
  after first RNN layer and concatenated to input of subsequent layers.
class_docstr_tok: ["RNN","decoder","with","multihead","attention",".","Attention","is","calculated","using","encoder","output","and","output","of","decoder","'s","first","RNN","layerself",".","Attention","is","applied","after","first","RNN","layer","and","concatenated","to","input","of","subsequent","layers","."]
code: def __init__(
      self,
      src_dict,
      dst_dict,
      embed_tokens,
      vocab_reduction_params=None,
      encoder_hidden_dim=512,
      embed_dim=512,
      hidden_dim=512,
      out_embed_dim=512,
      cell_type="lstm",
      num_layers=1,
      dropout_in=0.1,
      dropout_out=0.1,
      attention_type="dot",
      attention_heads=8,
      first_layer_attention=False,
      residual_level=None,
      averaging_encoder=False,
      project_output=True,
      out_embed_norm=None,
      tie_embeddings=False,
      pretrained_embed=None,
      projection_pretrained_embed=None,
      att_weighted_src_embeds=False,
      src_embed_dim=512,
      att_weighted_activation_type="tanh",
      predictor=None,
      fp16: bool = False,
  ):
      super().__init__(
          src_dict,
          dst_dict,
          vocab_reduction_params,
          out_embed_dim,
          project_output=project_output,
          pretrained_embed=projection_pretrained_embed,
          out_embed_norm=out_embed_norm,
          att_weighted_src_embeds=att_weighted_src_embeds,
          src_embed_dim=src_embed_dim,
          att_weighted_activation_type=att_weighted_activation_type,
          predictor=predictor,
          fp16=fp16,
      )
      encoder_hidden_dim = max(1, encoder_hidden_dim)
      self.encoder_hidden_dim = encoder_hidden_dim
      self.embed_dim = embed_dim
      self.hidden_dim = hidden_dim
      self.out_embed_dim = out_embed_dim
      self.dropout_in = dropout_in
      self.dropout_out = dropout_out
      self.attention_type = attention_type
      self.residual_level = residual_level
      self.tie_embeddings = tie_embeddings
      self.attention_heads = attention_heads
      self.first_layer_attention = first_layer_attention
      self.padding_idx = dst_dict.pad()
      self.embed_tokens = embed_tokens
  
      if self.tie_embeddings:
          assert self.embed_dim == self.out_embed_dim, (
              "Input embeddings and output projections must have the same "
              "dimension for the weights to be tied"
          )
          self.embed_tokens.weight = self.output_projection_w
  
      self.hidden_dim = hidden_dim
      self.averaging_encoder = averaging_encoder
  
      if cell_type == "lstm":
          cell_class = rnn_cell.LSTMCell
      elif cell_type == "milstm":
          cell_class = rnn_cell.MILSTMCell
      elif cell_type == "layer_norm_lstm":
          cell_class = rnn_cell.LayerNormLSTMCell
  
      if hidden_dim != encoder_hidden_dim:
          hidden_init_fc_list = []
          cell_init_fc_list = []
          for _ in range(num_layers):
              hidden_init_fc_list.append(
                  Linear(encoder_hidden_dim, hidden_dim)
              )
              cell_init_fc_list.append(
                  Linear(encoder_hidden_dim, hidden_dim)
              )
          self.hidden_init_fc_list = nn.ModuleList(hidden_init_fc_list)
          self.cell_init_fc_list = nn.ModuleList(cell_init_fc_list)
  
      self.attention = attention.build_attention(
          attention_type=attention_type,
          decoder_hidden_state_dim=hidden_dim,
          context_dim=encoder_hidden_dim,
          nheads=attention_heads,  # specific to multihead_attention
      )
  
      if self.attention.context_dim:
          self.initial_attn_context = nn.Parameter(
              torch.Tensor(self.attention.context_dim).zero_()
          )
      self.combined_output_and_context_dim = (
          self.attention.context_dim + hidden_dim
      )
  
      layers = []
      for layer in range(num_layers):
          if layer == 0:
              cell_input_dim = embed_dim
          else:
              cell_input_dim = hidden_dim
  
          # attention applied to first layer always.
          if self.first_layer_attention or layer == 0:
              cell_input_dim += self.attention.context_dim
          layers.append(
              cell_class(
                  input_dim=cell_input_dim, hidden_dim=hidden_dim
              )
          )
      self.layers = nn.ModuleList(layers)
  
      if self.combined_output_and_context_dim != out_embed_dim:
          self.additional_fc = Linear(
              self.combined_output_and_context_dim, out_embed_dim
          )
  
imports: import numpy
  import torch
  import torch.nn
  import torch.nn.functional
  import torch.onnx.operators
  from fairseq import options, utils
  from fairseq.models import FairseqEncoder, FairseqEncoderDecoderModel, register_model, register_model_architecture
  from fairseq.modules import AdaptiveSoftmax
  from pytorch_translate import attention, rnn_cell, utils, vocab_reduction
  from pytorch_translate.common_layers import ContextEmbedding, DecoderWithOutputProjection, Embedding, Linear, RNNLayer, VariableTracker
  from pytorch_translate.data import data
  from pytorch_translate.multi_model import MultiDecoder, MultiEncoder
  from pytorch_translate.multilingual import MultilingualDecoder, MultilingualEncoder
  from pytorch_translate.ngram import NGramDecoder
  from pytorch_translate.semi_supervised import SemiSupervisedModel
  from pytorch_translate.utils import maybe_cat, maybe_cuda, torch_find
  from torch.nn.utils.rnn import pack_padded_sequence, PackedSequence, pad_packed_sequence
label_desc: Use image node_id 1 to create a new RNNDecoder object from inherited base classes: DecoderWithOutputProjection with example: obj = RNNDecoder(src_dict, dst_dict, embed_tokens, vocab_reduction_params, encoder_hidden_dim, embed_dim, hidden_dim, out_embed_dim, cell_type, num_layers, dropout_in, dropout_out, attention_type, attention_heads, first_layer_attention, residual_level, averaging_encoder, project_output, out_embed_norm, tie_embeddings, pretrained_embed, projection_pretrained_embed, att_weighted_src_embeds, src_embed_dim, att_weighted_activation_type, predictor, fp16)