class_docstr: Increase the learning rate of each parameter group from min lr ratio to max lr ratio
  over warmup_num_steps steps, and then decay at cosine rate over the remaining training steps to min cosine ratio.
  
  Args:
      optimizer (Optimizer): Wrapped optimizer.
      total_num_steps (int): total number of training steps
      warmup_min_ratio (float or list): warmup start learning rate ratio. Default: 0
      warmup_num_steps (int): number of steps to warm up from warmup_min_ratio to 1.0. Default: 1000
      warmup_type {‘log’, ‘linear’}: increasing function from min_lr to max_lr during warmup. Default: log
      cos_min_ratio (float): cosine end learning rate ratio. Default: 0.0001
      last_batch_iteration (int): The index of the last batch. Default: -1.
  Example:
      >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
      >>> scheduler = WarmupCosineLR(optimizer, 1000000)
      >>> data_loader = torch.utils.data.DataLoader(...)
      >>> for epoch in range(10):
      >>>     for batch in data_loader:
      >>>         train_batch(...)
      >>>         scheduler.step()
class_docstr_tok: ["Increase","the","learning","rate","of","each","parameter","group","from","min","lr","ratio","to","max","lr","ratio","over","warmup_num_steps","steps",",","and","then","decay","at","cosine","rate","over","the","remaining","training","steps","to","min","cosine","ratio",".","Args",":","optimizer","(","Optimizer",")",":","Wrapped","optimizer",".","total_num_steps","(","int",")",":","total","number","of","training","steps","warmup_min_ratio","(","float","or","list",")",":","warmup","start","learning","rate","ratio",".","Default",":","0","warmup_num_steps","(","int",")",":","number","of","steps","to","warm","up","from","warmup_min_ratio","to","1.0",".","Default",":","1000","warmup_type","{","\u2018","log","\u2019",",","\u2018","linear","\u2019","}",":","increasing","function","from","min_lr","to","max_lr","during","warmup",".","Default",":","log","cos_min_ratio","(","float",")",":","cosine","end","learning","rate","ratio",".","Default",":","0.0001","last_batch_iteration","(","int",")",":","The","index","of","the","last","batch",".","Default",":","-1",".","Example",":",">",">",">","optimizer","=","torch.optim.SGD","(","model.parameters","(",")",",","lr=0.1",",","momentum=0.9",")",">",">",">","scheduler","=","WarmupCosineLR","(","optimizer",",","1000000",")",">",">",">","data_loader","=","torch.utils.data.DataLoader","(","...",")",">",">",">","for","epoch","in","range","(","10",")",":",">",">",">","for","batch","in","data_loader",":",">",">",">","train_batch","(","...",")",">",">",">","scheduler.step","(",")"]
code: def get_lr_ratio(self):
      if self.last_batch_iteration < 0:
          logger.warning(
              "Attempting to get learning rate from scheduler before it has started"
          )
          return [0.0]
  
      if self.last_batch_iteration < self.warmup_num_steps:
          if self.warmup_type == WARMUP_LOG_RATE:
              ratio = self.inverse_log_warm_up * math.log(
                  self.last_batch_iteration + 1
              )
          elif self.warmup_type == WARMUP_LINEAR_RATE:
              ratio = self.last_batch_iteration / self.warmup_num_steps
          ratio_delta = 1.0 - self.warmup_min_ratio
          ratio = self.warmup_min_ratio + ratio * ratio_delta
          return ratio
  
      real_last_step = (
          self.last_batch_iteration - self.warmup_num_steps + 1
      )
      real_total_steps = self.total_num_steps - self.warmup_num_steps
      ratio_delta = 1.0 - self.cos_min_ratio
      ratio = (
          1 + math.cos(math.pi * real_last_step / real_total_steps)
      ) / 2
      ratio = max(0.0, self.cos_min_ratio + ratio_delta * ratio)
      return ratio
  
imports: import argparse
  from torch.optim import Optimizer
  import math
  from deepspeed.utils import logger
label_desc: Use image node_id 2 for calling the WarmupCosineLR obj's underlying member method code with example usage: obj.get_lr_ratio() and returns: ratio, list, ratio