class_docstr: FP16_Optimizer is designed to wrap an existing PyTorch optimizer,
  and enable an fp16 model to be trained using a main copy of fp32 weights.
  
  Args:
      optimizer (torch.optim.optimizer):  Existing optimizer containing initialized fp16 parameters.  Internally, FP16_Optimizer replaces the passed optimizer's fp16 parameters with new fp32 parameters copied from the original ones.  FP16_Optimizer also stores references to the original fp16 parameters, and updates these fp16 parameters from the main fp32 copy after each step.
      static_loss_scale (float, optional, default=1.0):  Loss scale used internally to scale fp16 gradients computed by the model.  Scaled gradients will be copied to fp32, then downscaled before being applied to the fp32 main params, so static_loss_scale should not affect learning rate.
      dynamic_loss_scale (bool, optional, default=False):  Use dynamic loss scaling.  If True, this will override any static_loss_scale option.
class_docstr_tok: ["FP16_Optimizer","is","designed","to","wrap","an","existing","PyTorch","optimizer",",","and","enable","an","fp16","model","to","be","trained","using","a","main","copy","of","fp32","weights",".","Args",":","optimizer","(","torch.optim.optimizer",")",":","Existing","optimizer","containing","initialized","fp16","parameters",".","Internally",",","FP16_Optimizer","replaces","the","passed","optimizer","'s","fp16","parameters","with","new","fp32","parameters","copied","from","the","original","ones",".","FP16_Optimizer","also","stores","references","to","the","original","fp16","parameters",",","and","updates","these","fp16","parameters","from","the","main","fp32","copy","after","each","step",".","static_loss_scale","(","float",",","optional",",","default=1.0",")",":","Loss","scale","used","internally","to","scale","fp16","gradients","computed","by","the","model",".","Scaled","gradients","will","be","copied","to","fp32",",","then","downscaled","before","being","applied","to","the","fp32","main","params",",","so","static_loss_scale","should","not","affect","learning","rate",".","dynamic_loss_scale","(","bool",",","optional",",","default=False",")",":","Use","dynamic","loss","scaling",".","If","True",",","this","will","override","any","static_loss_scale","option","."]
code: def __init__(
      self, optimizer, static_loss_scale=1.0, dynamic_loss_scale=False
  ):
      if not torch.cuda.is_available:
          raise SystemError("Cannot use fp16 without CUDA")
  
      self.fp16_param_groups = []
      self.fp32_param_groups = []
      self.fp32_flattened_groups = []
      for i, param_group in enumerate(optimizer.param_groups):
          print("FP16_Optimizer processing param group {}:".format(i))
          fp16_params_this_group = []
          fp32_params_this_group = []
          for param in param_group["params"]:
              if param.requires_grad:
                  if param.type() == "torch.cuda.HalfTensor":
                      print(
                          "FP16_Optimizer received torch.cuda.HalfTensor with {}".format(
                              param.size()
                          )
                      )
                      fp16_params_this_group.append(param)
                  elif param.type() == "torch.cuda.FloatTensor":
                      print(
                          "FP16_Optimizer received torch.cuda.FloatTensor with {}".format(
                              param.size()
                          )
                      )
                      fp32_params_this_group.append(param)
                  else:
                      raise TypeError(
                          "Wrapped parameters must be either "
                          "torch.cuda.FloatTensor or torch.cuda.HalfTensor. "
                          "Received {}".format(param.type())
                      )
  
          fp32_flattened_this_group = None
          if len(fp16_params_this_group) > 0:
              fp32_flattened_this_group = _flatten_dense_tensors(
                  [
                      param.detach().data.clone().float()
                      for param in fp16_params_this_group
                  ]
              )
  
              fp32_flattened_this_group = Variable(
                  fp32_flattened_this_group, requires_grad=True
              )
  
              fp32_flattened_this_group.grad = (
                  fp32_flattened_this_group.new(
                      *fp32_flattened_this_group.size()
                  )
              )
  
          # python's lovely list concatenation via +
          if fp32_flattened_this_group is not None:
              param_group["params"] = [
                  fp32_flattened_this_group
              ] + fp32_params_this_group
          else:
              param_group["params"] = fp32_params_this_group
  
          self.fp16_param_groups.append(fp16_params_this_group)
          self.fp32_param_groups.append(fp32_params_this_group)
          self.fp32_flattened_groups.append(fp32_flattened_this_group)
  
      # print("self.fp32_flattened_groups = ", self.fp32_flattened_groups)
      # print("self.fp16_param_groups = ", self.fp16_param_groups)
  
      self.optimizer = optimizer.__class__(optimizer.param_groups)
  
      # self.optimizer.load_state_dict(optimizer.state_dict())
  
      self.param_groups = self.optimizer.param_groups
  
      if dynamic_loss_scale:
          self.dynamic_loss_scale = True
          self.loss_scaler = DynamicLossScaler()
      else:
          self.dynamic_loss_scale = False
          self.loss_scaler = LossScaler(static_loss_scale)
  
      self.overflow = False
      self.first_closure_call_this_step = True
  
imports: import torch
  from torch import nn
  from torch.autograd import Variable
  from torch.nn.parameter import Parameter
  from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors
  from common.loss_scaler import DynamicLossScaler, LossScaler
label_desc: Use image node_id 1 to create a new FP16_Optimizer object from inherited base classes: object with example: obj = FP16_Optimizer(optimizer, static_loss_scale, dynamic_loss_scale)