class_docstr: ## Adam Optimizer with Warmup
  
  This class extends from AMSGrad optimizer defined in [`amsgrad.py`](amsgrad.html).
class_docstr_tok: ["#","#","Adam","Optimizer","with","Warmup","This","class","extends","from","AMSGrad","optimizer","defined","in","[","`","amsgrad.py","`","]","(","amsgrad.html",")","."]
docstr: ### Get learning-rate
  
  $$lpha \min igg(1, rac{t}{w}igg)$$
  where $w$ is the number of warmup steps.
docstr_tok: ["#","#","#","Get","learning-rate","$","$","\u0007lpha","\\min","\bigg","(","1",",","rac","{","t","}","{","w","}","\bigg",")","$","$","where","$","w","$","is","the","number","of","warmup","steps","."]
code: def get_lr(self, state: Dict[str, any], group: Dict[str, any]):
      """
      ### Get learning-rate
  
      $$\alpha \min \bigg(1, \frac{t}{w}\bigg)$$
      where $w$ is the number of warmup steps.
      """
      # If we are in warmup stage
      if group["warmup"] > state["step"]:
          # A linearly increasing learning rate from $0$ to $\alpha$
          return 1e-8 + state["step"] * group["lr"] / group["warmup"]
      else:
          # Constant learning rate $\alpha$
          return group["lr"]
  
imports: from typing import Dict
  from labml_nn.optimizers import WeightDecay
  from labml_nn.optimizers.amsgrad import AMSGrad
label_desc: Use image node_id 2 for calling the AdamWarmup obj's underlying member method code with example usage: obj.get_lr(state, group) and returns: unknown, group