docstr: Incrementally generate the output given the decoder input.
  
  Parameters
  ----------
  data
      Shape (batch_size, C_in)
  states
      The previous states, contains
  
      1. layout = 'NT':
          - prev_multi_key
              Shape (batch_size, prev_seq_length, num_heads, C_key)
          - prev_multi_value
              Shape (batch_size, prev_seq_length, num_heads, C_value)
  
      2. layout = 'TN'
          - prev_multi_key
              Shape (prev_seq_length, batch_size, num_heads, C_key)
          - prev_multi_value
              Shape (prev_seq_length, batch_size, num_heads, C_value)
  
  mem
      The memory
  
      1. layout = 'NT'
          Shape (batch_size, mem_length, C_mem)
      2. layout = 'TN'
          Shape (mem_length, batch_size, C_mem)
  
  mem_valid_length
      Valid length of the memory
      Shape (batch_size,)
  mem_attn_mask
      The attention mask between data and the memory
      Has shape (batch_size, 1, mem_length)
  
  Returns
  -------
  out
      Shape (batch_size, C_out)
  updated_states
      - new_key
          Shape (batch_size, prev_seq_length + 1, num_heads, C_key)
      - new_value
          Shape (batch_size, prev_seq_length + 1, num_heads, C_value)
docstr_tok: ["Incrementally","generate","the","output","given","the","decoder","input",".","Parameters","--","--","--","--","--","data","Shape","(","batch_size",",","C_in",")","states","The","previous","states",",","contains","1.","layout","=","'NT","'",":","-","prev_multi_key","Shape","(","batch_size",",","prev_seq_length",",","num_heads",",","C_key",")","-","prev_multi_value","Shape","(","batch_size",",","prev_seq_length",",","num_heads",",","C_value",")","2.","layout","=","'TN'","-","prev_multi_key","Shape","(","prev_seq_length",",","batch_size",",","num_heads",",","C_key",")","-","prev_multi_value","Shape","(","prev_seq_length",",","batch_size",",","num_heads",",","C_value",")","mem","The","memory","1.","layout","=","'NT'","Shape","(","batch_size",",","mem_length",",","C_mem",")","2.","layout","=","'TN'","Shape","(","mem_length",",","batch_size",",","C_mem",")","mem_valid_length","Valid","length","of","the","memory","Shape","(","batch_size",",",")","mem_attn_mask","The","attention","mask","between","data","and","the","memory","Has","shape","(","batch_size",",","1",",","mem_length",")","Returns","--","--","--","-","out","Shape","(","batch_size",",","C_out",")","updated_states","-","new_key","Shape","(","batch_size",",","prev_seq_length","+","1",",","num_heads",",","C_key",")","-","new_value","Shape","(","batch_size",",","prev_seq_length","+","1",",","num_heads",",","C_value",")"]
code: def incremental_decode(
      self, data, states, mem, mem_valid_length, mem_attn_mask=None
  ):
      """Incrementally generate the output given the decoder input.
  
      Parameters
      ----------
      data
          Shape (batch_size, C_in)
      states
          The previous states, contains
  
          1. layout = 'NT':
              - prev_multi_key
                  Shape (batch_size, prev_seq_length, num_heads, C_key)
              - prev_multi_value
                  Shape (batch_size, prev_seq_length, num_heads, C_value)
  
          2. layout = 'TN'
              - prev_multi_key
                  Shape (prev_seq_length, batch_size, num_heads, C_key)
              - prev_multi_value
                  Shape (prev_seq_length, batch_size, num_heads, C_value)
  
      mem
          The memory
  
          1. layout = 'NT'
              Shape (batch_size, mem_length, C_mem)
          2. layout = 'TN'
              Shape (mem_length, batch_size, C_mem)
  
      mem_valid_length
          Valid length of the memory
          Shape (batch_size,)
      mem_attn_mask
          The attention mask between data and the memory
          Has shape (batch_size, 1, mem_length)
  
      Returns
      -------
      out
          Shape (batch_size, C_out)
      updated_states
          - new_key
              Shape (batch_size, prev_seq_length + 1, num_heads, C_key)
          - new_value
              Shape (batch_size, prev_seq_length + 1, num_heads, C_value)
  
      """
      if self._pre_norm:
          data = self.ln_in(data)
      if self.layout == "NT":
          time_axis = 1
      else:
          time_axis = 0
      data = np.expand_dims(data, axis=time_axis)
      # Shape (B, prev_L, #Head, C_K), (B, prev_L, #Head, C_V)
      #  or (prev_L, B, #Head, C_K), (prev_L, B, #Head, C_V)
      prev_key, prev_value = states
      if mem_attn_mask is None:
          mem_attn_mask = gen_mem_attn_mask(
              mem,
              mem_valid_length,
              data,
              None,
              dtype=self._dtype,
              layout=self.layout,
          )
      # 1. Get the causal self-attention value, we need to attend to both the current data
      # and the previous stored key/values
      # Shape (B, 1, 3 * num_heads * C_key)
      #  or (1, B, 3 * num_heads * C_key)
      step_qkv = self.attn_in_qkv(data)
      step_query, step_key, step_value = np.split(step_qkv, 3, axis=-1)
      step_query = npx.reshape(
          step_query, (-2, -2, self._num_heads, -1)
      )
      step_key = npx.reshape(step_key, (-2, -2, self._num_heads, -1))
      step_value = npx.reshape(
          step_value, (-2, -2, self._num_heads, -1)
      )
      new_key = np.concatenate([prev_key, step_key], axis=time_axis)
      new_value = np.concatenate(
          [prev_value, step_value], axis=time_axis
      )
      out, [_, attn_weight] = self.self_attention(
          step_query, new_key, new_value, None
      )
      out = self.proj_in(out)
      out = self.dropout_layer(out)
      out = out + data
      if not self._pre_norm:
          out = self.ln_in(out)
      # 2. Attend to the contextual memory
      data = out
      if self._pre_norm:
          data = self.ln_inter(data)
      out, _ = self.inter_attention(
          npx.reshape(
              self.attn_inter_q(data), (-2, -2, self._num_heads, -1)
          ),
          npx.reshape(
              self.attn_inter_k(mem), (-2, -2, self._num_heads, -1)
          ),
          npx.reshape(
              self.attn_inter_v(mem), (-2, -2, self._num_heads, -1)
          ),
          mem_attn_mask,
      )
      out = self.proj_inter(out)
      out = self.dropout_layer(out)
      out = out + data
      if not self._pre_norm:
          out = self.ln_inter(out)
      # 3. Encode the output via an FFN layer
      out = self.ffn(out)
      out = npx.reshape(out, (-5, -1))
      return out, (new_key, new_value)
  
imports: import numpy
  import mxnet
  from mxnet import np, npx
  from mxnet import use_np
  from mxnet.gluon import nn, HybridBlock
  from typing import Optional, Tuple, List
  from ..utils.registry import Registry
  from ..attention_cell import MultiHeadAttentionCell, gen_self_attn_mask, gen_mem_attn_mask
  from ..layers import PositionalEmbedding, PositionwiseFFN, InitializerType
  from ..utils.config import CfgNode
  from ..sequence_sampler import BaseStepDecoder
label_desc: Use image node_id 6 for calling the TransformerDecoderLayer obj's underlying member method code with example usage: obj.incremental_decode(data, states, mem, mem_valid_length, mem_attn_mask) and returns: out,  new_key,  new_value