class_docstr: Implements a Transformer Encoder Layer used in BERT/XLM style pre-trained
  models.
class_docstr_tok: ["Implements","a","Transformer","Encoder","Layer","used","in","BERT\/XLM","style","pre-trained","models","."]
code: def __init__(
      self,
      embedding_dim: float = 768,
      ffn_embedding_dim: float = 3072,
      num_attention_heads: int = 8,
      dropout: float = 0.1,
      attention_dropout: float = 0.1,
      activation_dropout: float = 0.1,
      activation_fn: str = "relu",
      layer_norm_first: bool = False,
  ) -> None:
      super().__init__()
      # Initialize parameters
      self.embedding_dim = embedding_dim
      self.dropout = dropout
      self.activation_dropout = activation_dropout
  
      # Initialize blocks
      self.activation_fn = utils.get_activation_fn(activation_fn)
      self.self_attn = MultiheadAttention(
          self.embedding_dim,
          num_attention_heads,
          dropout=attention_dropout,
          self_attention=True,
      )
  
      self.dropout1 = nn.Dropout(dropout)
      self.dropout2 = nn.Dropout(self.activation_dropout)
      self.dropout3 = nn.Dropout(dropout)
  
      self.layer_norm_first = layer_norm_first
  
      # layer norm associated with the self attention layer
      self.self_attn_layer_norm = LayerNorm(self.embedding_dim)
      self.fc1 = nn.Linear(self.embedding_dim, ffn_embedding_dim)
      self.fc2 = nn.Linear(ffn_embedding_dim, self.embedding_dim)
  
      # layer norm associated with the position wise feed-forward NN
      self.final_layer_norm = LayerNorm(self.embedding_dim)
  
imports: import math
  from dataclasses import dataclass, field
  from typing import List, Tuple
  import numpy
  import torch
  import torch.nn
  import torch.nn.functional
  from fairseq import utils
  from fairseq.data.data_utils import compute_mask_indices
  from fairseq.dataclass import ChoiceEnum, FairseqDataclass
  from fairseq.distributed import fsdp_wrap
  from fairseq.models import BaseFairseqModel, register_model
  from fairseq.distributed.fully_sharded_data_parallel import FullyShardedDataParallel
  from fairseq.modules import Fp32GroupNorm, Fp32LayerNorm, GradMultiply, GumbelVectorQuantizer, LayerNorm, MultiheadAttention, RelPositionalEncoding, SamePad, TransposeLast
  from fairseq.modules.checkpoint_activations import checkpoint_wrapper
  from fairseq.modules.conformer_layer import ConformerWav2Vec2EncoderLayer
  from fairseq.modules.transformer_sentence_encoder import init_bert_params
  from fairseq.utils import buffered_arange, index_put, is_xla_tensor
  from .utils import pad_to_multiple
label_desc: Use image node_id 1 to create a new TransformerSentenceEncoderLayer object from inherited base classes: nn with example: obj = TransformerSentenceEncoderLayer(embedding_dim, ffn_embedding_dim, num_attention_heads, dropout, attention_dropout, activation_dropout, activation_fn, layer_norm_first)