class_docstr: kerple (Attention with Linear Biases) relative position embedding for auto-regressive decoder
  and joint encoder (symmetric for forward and backward distance).
  Based on https://arxiv.org/bas/2108.12409
class_docstr_tok: ["kerple","(","Attention","with","Linear","Biases",")","relative","position","embedding","for","auto-regressive","decoder","and","joint","encoder","(","symmetric","for","forward","and","backward","distance",")",".","Based","on","https",":","\/\/arxiv.org\/bas\/2108.12409"]
docstr: Args:
      bidirectional: Whether to use bidirectional relative position embedding
      num_attention_heads: Number of attention heads
      layer_type: Layer type. Can be one of [LayerType.encoder or LayerType.decoder]. Willdetermine the bias construction
      num_attention_heads_kerple: Number of attention heads for which kerple bias will be used
      max_seq_len: Maximum sequence length for precomputed relative positions. Larger sizes will result in more memory usage by computing kerple mask on-the-fly.
docstr_tok: ["Args",":","bidirectional",":","Whether","to","use","bidirectional","relative","position","embedding","num_attention_heads",":","Number","of","attention","heads","layer_type",":","Layer","type",".","Can","be","one","of","[","LayerType.encoder","or","LayerType.decoder","]",".","Willdetermine","the","bias","construction","num_attention_heads_kerple",":","Number","of","attention","heads","for","which","kerple","bias","will","be","used","max_seq_len",":","Maximum","sequence","length","for","precomputed","relative","positions",".","Larger","sizes","will","result","in","more","memory","usage","by","computing","kerple","mask","on-the-fly","."]
code: def __init__(
      self,
      bidirectional,
      num_attention_heads,
      layer_type,
      num_attention_heads_kerple=None,
      max_seq_len=512,
  ):
      """
      Args:
          bidirectional: Whether to use bidirectional relative position embedding
          num_attention_heads: Number of attention heads
          layer_type: Layer type. Can be one of [LayerType.encoder or LayerType.decoder]. Willdetermine the bias construction
          num_attention_heads_kerple: Number of attention heads for which kerple bias will be used
          max_seq_len: Maximum sequence length for precomputed relative positions. Larger sizes will result in more memory usage by computing kerple mask on-the-fly.
      """
      super().__init__()
  
      if (num_attention_heads_kerple is None) or (
          num_attention_heads_kerple <= 0
      ):
          num_attention_heads_kerple = num_attention_heads
  
      if num_attention_heads_kerple > num_attention_heads:
          raise ValueError(
              f"num_attention_heads_kerple ({num_attention_heads_kerple}) cannot be larger than num_attention_heads ({num_attention_heads})"
          )
  
      self.bidirectional = bidirectional
      self.num_attention_heads = num_attention_heads
      # LayerType.encoder or LayerType.decoder. Is only needed to determine the group for the all_reduce
      self.layer_type = layer_type
      # define the size of pre-computed relative position slopes.
      # define the number of attention heads for which kerple mask will be pre-computed (the rest are disabled).
      self.num_attention_heads_kerple = num_attention_heads_kerple
      # Larger sizes will result in more memory usage by computing kerple mask on-the-fly.
      self.max_seq_len = max_seq_len
  
      # initialize the slopes
      self.kerple_b = torch.nn.Parameter(
          build_slopes(num_attention_heads, num_attention_heads_kerple)
      )
      self.kerple_a = torch.zeros_like(self.kerple_b)
      self.kerple_p = torch.ones_like(self.kerple_b)
  
      # cache the relative position bias. shape (num_attention_heads, max_seq_len, max_seq_len)
      self.relative_position = build_relative_position(
          max_seq_len, max_seq_len, num_attention_heads
      )
  
imports: import math
  import torch
  from nemo.collections.nlp.modules.common.megatron.alibi_relative_position_embedding import build_relative_position, build_slopes
label_desc: Use image node_id 1 to create a new KERPLERelativePositionEmbedding object from inherited base classes: torch.nn with example: obj = KERPLERelativePositionEmbedding(bidirectional, num_attention_heads, layer_type, num_attention_heads_kerple, max_seq_len)