code: def __init__(
      self,
      n_head,
      d_model,
      d_head,
      dropout,
      dropatt=0,
      tgt_len=None,
      ext_len=None,
      mem_len=None,
      pre_lnorm=False,
      moe=False,
      moe_num_expert=64,
      moe_top_k=2,
  ):
      super(RelMultiHeadAttn, self).__init__()
  
      self.n_head = n_head
      self.d_model = d_model
      self.d_head = d_head
      self.dropout = dropout
  
      self.qkv_net = nn.Linear(d_model, 3 * n_head * d_head, bias=False)
  
      self.drop = nn.Dropout(dropout)
      self.dropatt = nn.Dropout(dropatt)
      self.o_net = nn.Linear(n_head * d_head, d_model, bias=False)
  
      self.layer_norm = nn.LayerNorm(d_model)
  
      self.scale = 1 / (d_head**0.5)
  
      self.pre_lnorm = pre_lnorm
  
imports: import sys
  import math
  import functools
  import numpy
  import torch
  import torch.nn
  import torch.nn.functional
  from proj_adaptive_softmax import ProjectedAdaptiveLogSoftmax, Projection
  from log_uniform_sampler import LogUniformSampler, sample_logits
  from fmoe import FMoETransformerMLP
label_desc: Use image node_id 1 to create a new RelMultiHeadAttn object from inherited base classes: nn with example: obj = RelMultiHeadAttn(n_head, d_model, d_head, dropout, dropatt, tgt_len, ext_len, mem_len, pre_lnorm, moe, moe_num_expert, moe_top_k)