class_docstr: Increase the learning rate of each parameter group from min lr to max lr
  over warmup_num_steps steps, and then decay at linear rate over the remaining training steps.
  
  Args:
      optimizer (Optimizer): Wrapped optimizer.
      total_num_steps (int): total number of training steps
      warmup_min_lr (float or list): minimum learning rate. Default: 0
      warmup_max_lr (float or list): maximum learning rate. Default: 0.001
      warmup_num_steps (int): number of steps to warm up from min_lr to max_lr. Default: 1000
      warmup_type {‘log’, ‘linear’}: increasing function from min_lr to max_lr during warmup. Default: log
      last_batch_iteration (int): The index of the last batch. Default: -1.
  Example:
      >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
      >>> scheduler = WarmupDecayLR(optimizer, 1000000)
      >>> data_loader = torch.utils.data.DataLoader(...)
      >>> for epoch in range(10):
      >>>     for batch in data_loader:
      >>>         train_batch(...)
      >>>         scheduler.step()
class_docstr_tok: ["Increase","the","learning","rate","of","each","parameter","group","from","min","lr","to","max","lr","over","warmup_num_steps","steps",",","and","then","decay","at","linear","rate","over","the","remaining","training","steps",".","Args",":","optimizer","(","Optimizer",")",":","Wrapped","optimizer",".","total_num_steps","(","int",")",":","total","number","of","training","steps","warmup_min_lr","(","float","or","list",")",":","minimum","learning","rate",".","Default",":","0","warmup_max_lr","(","float","or","list",")",":","maximum","learning","rate",".","Default",":","0.001","warmup_num_steps","(","int",")",":","number","of","steps","to","warm","up","from","min_lr","to","max_lr",".","Default",":","1000","warmup_type","{","\u2018","log","\u2019",",","\u2018","linear","\u2019","}",":","increasing","function","from","min_lr","to","max_lr","during","warmup",".","Default",":","log","last_batch_iteration","(","int",")",":","The","index","of","the","last","batch",".","Default",":","-1",".","Example",":",">",">",">","optimizer","=","torch.optim.SGD","(","model.parameters","(",")",",","lr=0.1",",","momentum=0.9",")",">",">",">","scheduler","=","WarmupDecayLR","(","optimizer",",","1000000",")",">",">",">","data_loader","=","torch.utils.data.DataLoader","(","...",")",">",">",">","for","epoch","in","range","(","10",")",":",">",">",">","for","batch","in","data_loader",":",">",">",">","train_batch","(","...",")",">",">",">","scheduler.step","(",")"]
code: def __init__(
      self,
      optimizer: Optimizer,
      total_num_steps: int,
      warmup_min_lr: float = 0.0,
      warmup_max_lr: float = 0.001,
      warmup_num_steps: int = 1000,
      warmup_type: str = WARMUP_LOG_RATE,
      last_batch_iteration: int = -1,
  ):
      self.total_num_steps = total_num_steps
      super(WarmupDecayLR, self).__init__(
          optimizer,
          warmup_min_lr,
          warmup_max_lr,
          warmup_num_steps,
          warmup_type,
          last_batch_iteration,
      )
      if self.total_num_steps < self.warmup_num_steps:
          logger.warning(
              "total_num_steps {} is less than warmup_num_steps {}".format(
                  total_num_steps, warmup_num_steps
              )
          )
  
imports: import argparse
  from torch.optim import Optimizer
  import math
  from deepspeed.utils import logger
label_desc: Use image node_id 1 to create a new WarmupDecayLR object from inherited base classes: WarmupLR with example: obj = WarmupDecayLR(optimizer, total_num_steps, warmup_min_lr, warmup_max_lr, warmup_num_steps, warmup_type, last_batch_iteration)