code: def __init__(
      self,
      n_token,
      n_layer,
      n_head,
      d_model,
      d_head,
      d_inner,
      dropout,
      dropatt,
      tie_weight=True,
      d_embed=None,
      div_val=1,
      tie_projs=[False],
      pre_lnorm=False,
      tgt_len=None,
      ext_len=None,
      mem_len=None,
      cutoffs=[],
      adapt_inp=False,
      same_length=False,
      attn_type=0,
      clamp_len=-1,
      sample_softmax=-1,
      moe=False,
      moe_num_expert=64,
      moe_top_k=2,
  ):
      super(MemTransformerLM, self).__init__()
      self.n_token = n_token
  
      d_embed = d_model if d_embed is None else d_embed
      self.d_embed = d_embed
      self.d_model = d_model
      self.n_head = n_head
      self.d_head = d_head
  
      self.word_emb = AdaptiveEmbedding(
          n_token, d_embed, d_model, cutoffs, div_val=div_val
      )
  
      self.drop = nn.Dropout(dropout)
  
      self.n_layer = n_layer
  
      self.tgt_len = tgt_len
      self.mem_len = mem_len
      self.ext_len = ext_len
      self.max_klen = tgt_len + ext_len + mem_len
  
      self.attn_type = attn_type
  
      self.layers = nn.ModuleList()
      if attn_type == 0:  # the default attention
          for i in range(n_layer):
              self.layers.append(
                  RelPartialLearnableDecoderLayer(
                      n_head,
                      d_model,
                      d_head,
                      d_inner,
                      dropout,
                      tgt_len=tgt_len,
                      ext_len=ext_len,
                      mem_len=mem_len,
                      dropatt=dropatt,
                      pre_lnorm=pre_lnorm,
                      moe=moe,
                      moe_num_expert=moe_num_expert,
                      moe_top_k=moe_top_k,
                  )
              )
      elif attn_type == 1:  # learnable embeddings
          for i in range(n_layer):
              self.layers.append(
                  RelLearnableDecoderLayer(
                      n_head,
                      d_model,
                      d_head,
                      d_inner,
                      dropout,
                      tgt_len=tgt_len,
                      ext_len=ext_len,
                      mem_len=mem_len,
                      dropatt=dropatt,
                      pre_lnorm=pre_lnorm,
                      moe=moe,
                      moe_num_expert=moe_num_expert,
                      moe_top_k=moe_top_k,
                  )
              )
      elif attn_type in [2, 3]:  # absolute embeddings
          for i in range(n_layer):
              self.layers.append(
                  DecoderLayer(
                      n_head,
                      d_model,
                      d_head,
                      d_inner,
                      dropout,
                      dropatt=dropatt,
                      pre_lnorm=pre_lnorm,
                      moe=moe,
                      moe_num_expert=moe_num_expert,
                      moe_top_k=moe_top_k,
                  )
              )
  
      self.sample_softmax = sample_softmax
      # use sampled softmax
      if sample_softmax > 0:
          self.out_layer = nn.Linear(d_model, n_token)
          if tie_weight:
              self.out_layer.weight = self.word_emb.weight
          self.tie_weight = tie_weight
          self.sampler = LogUniformSampler(n_token, sample_softmax)
  
      # use adaptive softmax (including standard softmax)
      else:
          self.crit = ProjectedAdaptiveLogSoftmax(
              n_token, d_embed, d_model, cutoffs, div_val=div_val
          )
  
          if tie_weight:
              for i in range(len(self.crit.out_layers)):
                  self.crit.out_layers[
                      i
                  ].weight = self.word_emb.emb_layers[i].weight
  
          if tie_projs:
              for i, tie_proj in enumerate(tie_projs):
                  if tie_proj and div_val == 1 and d_model != d_embed:
                      self.crit.out_projs[
                          i
                      ].weight = self.word_emb.emb_projs[0].weight
                  elif tie_proj and div_val != 1:
                      self.crit.out_projs[
                          i
                      ].weight = self.word_emb.emb_projs[i].weight
  
      self.same_length = same_length
      self.clamp_len = clamp_len
  
      self._create_params()
  
imports: import sys
  import math
  import functools
  import numpy
  import torch
  import torch.nn
  import torch.nn.functional
  from proj_adaptive_softmax import ProjectedAdaptiveLogSoftmax, Projection
  from log_uniform_sampler import LogUniformSampler, sample_logits
  from fmoe import FMoETransformerMLP
label_desc: Use image node_id 1 to create a new MemTransformerLM object from inherited base classes: nn with example: obj = MemTransformerLM(n_token, n_layer, n_head, d_model, d_head, d_inner, dropout, dropatt, tie_weight, d_embed, div_val, tie_projs, pre_lnorm, tgt_len, ext_len, mem_len, cutoffs, adapt_inp, same_length, attn_type, clamp_len, sample_softmax, moe, moe_num_expert, moe_top_k)