class_docstr: ## [RoPE embeddings](../rope/index.html)
  
  *We use rotary position embeddings in self-attention layers.
  We assume the positional information gets embedded in embeddings
  and therefore not use them in causal attention.
  [Non-causal self-attention needs explicit positional information
   because it cannot infer it](https://arxiv.org/abs/3999902edc8511eba3db37f65e372566).*
class_docstr_tok: ["#","#","[","RoPE","embeddings","]","(","..","\/rope\/index.html",")","*","We","use","rotary","position","embeddings","in","self-attention","layers",".","We","assume","the","positional","information","gets","embedded","in","embeddings","and","therefore","not","use","them","in","causal","attention",".","[","Non-causal","self-attention","needs","explicit","positional","information","because","it","can","not","infer","it","]","(","https",":","\/\/arxiv.org\/abs\/3999902edc8511eba3db37f65e372566",")",".","*"]
docstr: * `d` is the number of features $d$
  * `base` is the constant used for calculating $\Theta$
docstr_tok: ["*","`","d","`","is","the","number","of","features","$","d","$","*","`","base","`","is","the","constant","used","for","calculating","$","\\Theta","$"]
code: def __init__(self, d: int, base: int = 10_000):
      """
      * `d` is the number of features $d$
      * `base` is the constant used for calculating $\Theta$
      """
      super().__init__()
      # $\Theta = {\theta_i = 10000^{\frac{2(i-1)}{d}}, i \in [1, 2, ..., \frac{d}{2}]}$
      self.theta = nn.Parameter(
          1.0 / (base ** (torch.arange(0, d, 2).float() / d)),
          requires_grad=False,
      )
  
imports: import math
  from typing import Set
  import torch
  from torch import nn
  from labml.logger import inspect
label_desc: Use image node_id 1 to create a new RotaryPositionalEmbeddings object from inherited base classes: nn with example: obj = RotaryPositionalEmbeddings(d, base)