class_docstr: Transformer Encoder Layer
class_docstr_tok: ["Transformer","Encoder","Layer"]
docstr: Parameters
  ----------
  units
  hidden_size
  num_heads
  attention_dropout_prob
  hidden_dropout_prob
  activation_dropout_prob
  layer_norm_eps
  pre_norm
      Whether to attach the normalization layer before attention layer
      If pre_norm:
          norm(data) -> attn -> res(+data) -> ffn
      Else:
          data -> attn -> norm(res(+data)) -> ffn
  
  use_qkv_bias
      Whether to use bias for self attention
  weight_initializer
  bias_initializer
  activation
  dtype
  layout
docstr_tok: ["Parameters","--","--","--","--","--","units","hidden_size","num_heads","attention_dropout_prob","hidden_dropout_prob","activation_dropout_prob","layer_norm_eps","pre_norm","Whether","to","attach","the","normalization","layer","before","attention","layer","If","pre_norm",":","norm","(","data",")","-",">","attn","-",">","res","(","+data",")","-",">","ffn","Else",":","data","-",">","attn","-",">","norm","(","res","(","+data",")",")","-",">","ffn","use_qkv_bias","Whether","to","use","bias","for","self","attention","weight_initializer","bias_initializer","activation","dtype","layout"]
code: def __init__(
      self,
      units: int = 512,
      hidden_size: int = 2048,
      num_heads: int = 8,
      attention_dropout_prob: float = 0.1,
      hidden_dropout_prob: float = 0.1,
      activation_dropout_prob: float = 0.0,
      layer_norm_eps: float = 1e-12,
      pre_norm: bool = False,
      use_qkv_bias: bool = True,
      weight_initializer: Optional[InitializerType] = None,
      bias_initializer: Optional[InitializerType] = "zeros",
      activation: str = "relu",
      dtype="float32",
      layout="NT",
  ):
      """
  
      Parameters
      ----------
      units
      hidden_size
      num_heads
      attention_dropout_prob
      hidden_dropout_prob
      activation_dropout_prob
      layer_norm_eps
      pre_norm
          Whether to attach the normalization layer before attention layer
          If pre_norm:
              norm(data) -> attn -> res(+data) -> ffn
          Else:
              data -> attn -> norm(res(+data)) -> ffn
  
      use_qkv_bias
          Whether to use bias for self attention
      weight_initializer
      bias_initializer
      activation
      dtype
      layout
      """
      super().__init__()
      self._units = units
      self._hidden_size = hidden_size
      self._num_heads = num_heads
      self._attention_dropout_prob = attention_dropout_prob
      self._hidden_dropout_prob = hidden_dropout_prob
      self._activation_dropout_prob = activation_dropout_prob
      self._pre_norm = pre_norm
      self._dtype = dtype
      self._layout = layout
      assert layout in ["TN", "NT"], (
          "Invalid layout received = {}. "
          'Only "TN" and "NT" are accepted!'.format(layout)
      )
      assert (
          self._units % self._num_heads == 0
      ), "units must be divisive by the number of heads"
      self.dropout_layer = nn.Dropout(hidden_dropout_prob)
      self.attn_qkv = nn.Dense(
          3 * units,
          flatten=False,
          use_bias=use_qkv_bias,
          in_units=units,
          weight_initializer=weight_initializer,
          bias_initializer=bias_initializer,
          dtype=self._dtype,
      )
      self.attention_proj = nn.Dense(
          units=units,
          flatten=False,
          in_units=units,
          use_bias=True,
          weight_initializer=weight_initializer,
          bias_initializer=bias_initializer,
          dtype=self._dtype,
      )
      attention_layout = "NTK" if self._layout == "NT" else "TNK"
      self.attention_cell = MultiHeadAttentionCell(
          query_units=self._units,
          num_heads=self._num_heads,
          attention_dropout=self._attention_dropout_prob,
          scaled=True,
          dtype=self._dtype,
          layout=attention_layout,
      )
      self.layer_norm = nn.LayerNorm(
          epsilon=layer_norm_eps, in_channels=units
      )
      self.ffn = PositionwiseFFN(
          units=units,
          hidden_size=hidden_size,
          dropout=hidden_dropout_prob,
          activation_dropout=activation_dropout_prob,
          weight_initializer=weight_initializer,
          bias_initializer=bias_initializer,
          layer_norm_eps=layer_norm_eps,
          activation=activation,
          pre_norm=pre_norm,
          dtype=self._dtype,
      )
  
imports: import numpy
  import mxnet
  from mxnet import np, npx
  from mxnet import use_np
  from mxnet.gluon import nn, HybridBlock
  from typing import Optional, Tuple, List
  from ..utils.registry import Registry
  from ..attention_cell import MultiHeadAttentionCell, gen_self_attn_mask, gen_mem_attn_mask
  from ..layers import PositionalEmbedding, PositionwiseFFN, InitializerType
  from ..utils.config import CfgNode
  from ..sequence_sampler import BaseStepDecoder
label_desc: Use image node_id 1 to create a new TransformerEncoderLayer object from inherited base classes: HybridBlock with example: obj = TransformerEncoderLayer(units, hidden_size, num_heads, attention_dropout_prob, hidden_dropout_prob, activation_dropout_prob, layer_norm_eps, pre_norm, use_qkv_bias, weight_initializer, bias_initializer, activation, dtype, layout)