class_docstr: Implements Adagrad algorithm.
  
  Currently GPU-only.  Requires Apex to be installed via
  ``pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./``.
  
  This version of fused Adagrad implements 2 fusions.
    * Fusion of the Adagrad update's elementwise operations
    * A multi-tensor apply launch that batches the elementwise updates applied to all the model's parameters into one or a few kernel launches.
  
  :class:`apex.optimizers.FusedAdagrad`'s usage is identical to any ordinary Pytorch optimizer::
      opt = apex.optimizers.FusedAdagrad(model.parameters(), lr = ....)
      ...
      opt.step()
  
  :class:`apex.optimizers.FusedAdagrad` may be used with or without Amp.  If you wish to use :class:`FusedAdagrad` with Amp,
  you may choose any ``opt_level``::
      opt = apex.optimizers.FusedAdagrad(model.parameters(), lr = ....)
      model, opt = amp.initialize(model, opt, opt_level="O0" or "O1 or "O2")
      ...
      opt.step()
  In general, ``opt_level="O1"`` is recommended.
  
  It has been proposed in `Adaptive Subgradient Methods for Online Learning
  and Stochastic Optimization`_.
  Arguments:
      params (iterable): iterable of parameters to optimize or dicts defining
          parameter groups
      lr (float, optional): learning rate (default: 1e-2)
      weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
      eps (float, optional): term added to the denominator to improve
          numerical stability (default: 1e-10)
      adagrad_w_mode (boolean, optional): Apply L2 regularization or weight decay
          True for decoupled weight decay (also known as AdamW) (default: False)
  
  .. _Adaptive Subgradient Methods for Online Learning and Stochastic
      Optimization: http://jmlr.org/papers/v12/duchi11a.html
class_docstr_tok: ["Implements","Adagrad","algorithm",".","Currently","GPU-only",".","Requires","Apex","to","be","installed","via","``","pip","install","-v","--","no-cache-dir","--","global-option=","''","--","cpp_ext","''","--","global-option=","''","--","cuda_ext","''",".\/","``",".","This","version","of","fused","Adagrad","implements","2","fusions",".","*","Fusion","of","the","Adagrad","update","'s","elementwise","operations","*","A","multi-tensor","apply","launch","that","batches","the","elementwise","updates","applied","to","all","the","model","'s","parameters","into","one","or","a","few","kernel","launches",".",":","class",":","`","apex.optimizers.FusedAdagrad","`","'s","usage","is","identical","to","any","ordinary","Pytorch","optimizer",":",":","opt","=","apex.optimizers.FusedAdagrad","(","model.parameters","(",")",",","lr","=","....",")","...","opt.step","(",")",":","class",":","`","apex.optimizers.FusedAdagrad","`","may","be","used","with","or","without","Amp",".","If","you","wish","to","use",":","class",":","`","FusedAdagrad","`","with","Amp",",","you","may","choose","any","``","opt_level","``",":",":","opt","=","apex.optimizers.FusedAdagrad","(","model.parameters","(",")",",","lr","=","....",")","model",",","opt","=","amp.initialize","(","model",",","opt",",","opt_level=","''","O0","''","or","``","O1","or","``","O2","''",")","...","opt.step","(",")","In","general",",","``","opt_level=","''","O1","''","``","is","recommended",".","It","has","been","proposed","in","`","Adaptive","Subgradient","Methods","for","Online","Learning","and","Stochastic","Optimization","`","_",".","Arguments",":","params","(","iterable",")",":","iterable","of","parameters","to","optimize","or","dicts","defining","parameter","groups","lr","(","float",",","optional",")",":","learning","rate","(","default",":","1e-2",")","weight_decay","(","float",",","optional",")",":","weight","decay","(","L2","penalty",")","(","default",":","0",")","eps","(","float",",","optional",")",":","term","added","to","the","denominator","to","improve","numerical","stability","(","default",":","1e-10",")","adagrad_w_mode","(","boolean",",","optional",")",":","Apply","L2","regularization","or","weight","decay","True","for","decoupled","weight","decay","(","also","known","as","AdamW",")","(","default",":","False",")","..","_Adaptive","Subgradient","Methods","for","Online","Learning","and","Stochastic","Optimization",":","http",":","\/\/jmlr.org\/papers\/v12\/duchi11a.html"]
code: def __init__(
      self,
      params,
      lr=1e-2,
      eps=1e-10,
      weight_decay=0.0,
      set_grad_none=True,
      adagrad_w_mode=False,
  ):
      defaults = dict(lr=lr, eps=eps, weight_decay=weight_decay)
      super(FusedAdagrad, self).__init__(params, defaults)
      self.adagrad_w_mode = 1 if adagrad_w_mode else 0
      self.set_grad_none = set_grad_none
  
      if multi_tensor_applier.available:
          import amp_C
  
          # Skip buffer
          self._dummy_overflow_buf = torch.cuda.IntTensor([0])
          self.multi_tensor_adagrad = amp_C.multi_tensor_adagrad
      else:
          raise RuntimeError(
              "apex.optimizers.FusedAdagrad requires cuda extensions"
          )
  
imports: import torch
  from apex.multi_tensor_apply import multi_tensor_applier
label_desc: Use image node_id 1 to create a new FusedAdagrad object from inherited base classes: torch.optim with example: obj = FusedAdagrad(params, lr, eps, weight_decay, set_grad_none, adagrad_w_mode)