class_docstr: Adagrad optimization algorithm.
  
  Adagrad is an algorithm that adapts the learning rate individually for each parameter
  by dividing by the :math:`L_2`-norm of all previous gradients. Given the parameters
  :math:`\theta`, gradient :math:`\nabla J`, accumulating norm :math:`G`, and smoothing
  factor :math:`\epsilon`, we use the update equations:
  
  .. math::
  
      G' = G + (\nabla J)^2
  
  .. math::
  
      \theta' = \theta - \frac{\alpha}{\sqrt{G' + \epsilon}} \nabla J
  
  where the smoothing factor :math:`\epsilon` prevents from dividing by zero.
  By adjusting the learning rate individually for each parameter, Adagrad adapts
  to the geometry of the error surface. Differently scaled weights have appropriately scaled
  update steps.
  
  Example usage:
  
  .. code-block:: python
  
      from neon.optimizers import Adagrad
  
      # use Adagrad with a learning rate of 0.01
      optimizer = Adagrad(learning_rate=0.01, epsilon=1e-6)
class_docstr_tok: ["Adagrad","optimization","algorithm",".","Adagrad","is","an","algorithm","that","adapts","the","learning","rate","individually","for","each","parameter","by","dividing","by","the",":","math",":","`","L_2","`","-norm","of","all","previous","gradients",".","Given","the","parameters",":","math",":","`","\\theta","`",",","gradient",":","math",":","`","\\nabla","J","`",",","accumulating","norm",":","math",":","`","G","`",",","and","smoothing","factor",":","math",":","`","\\epsilon","`",",","we","use","the","update","equations",":","..","math",":",":","G","'","=","G","+","(","\\nabla","J",")","^2","..","math",":",":","\\theta","'","=","\\theta","-","\\frac","{","\\alpha","}","{","\\sqrt","{","G","'","+","\\epsilon","}","}","\\nabla","J","where","the","smoothing","factor",":","math",":","`","\\epsilon","`","prevents","from","dividing","by","zero",".","By","adjusting","the","learning","rate","individually","for","each","parameter",",","Adagrad","adapts","to","the","geometry","of","the","error","surface",".","Differently","scaled","weights","have","appropriately","scaled","update","steps",".","Example","usage",":","..","code-block",":",":","python","from","neon.optimizers","import","Adagrad","#","use","Adagrad","with","a","learning","rate","of","0.01","optimizer","=","Adagrad","(","learning_rate=0.01",",","epsilon=1e-6",")"]
docstr: Apply the learning rule to all the layers and update the states.
  
  Arguments:
      layer_list (list): a list of Layer objects to optimize.
      epoch (int): the current epoch, needed for the Schedule object.
docstr_tok: ["Apply","the","learning","rule","to","all","the","layers","and","update","the","states",".","Arguments",":","layer_list","(","list",")",":","a","list","of","Layer","objects","to","optimize",".","epoch","(","int",")",":","the","current","epoch",",","needed","for","the","Schedule","object","."]
code: def optimize(self, layer_list, epoch):
      """
      Apply the learning rule to all the layers and update the states.
  
      Arguments:
          layer_list (list): a list of Layer objects to optimize.
          epoch (int): the current epoch, needed for the Schedule object.
      """
      lrate, epsilon = (self.learning_rate, self.epsilon)
      param_list = get_param_list(layer_list)
  
      scale_factor = self.clip_gradient_norm(
          param_list, self.gradient_clip_norm
      )
  
      for (param, grad), states in param_list:
          param.rounding = self.stochastic_round
          if len(states) == 0:
              states.append(self.be.zeros_like(grad))
  
          grad = grad / self.be.bsz
          grad = self.clip_value(grad, self.gradient_clip_value)
  
          # update state
          state = states[0]
          state[:] = state + self.be.square(grad)
          param[:] = self.clip_value(
              param
              - (scale_factor * grad * lrate)
              / (self.be.sqrt(state + epsilon)),
              self.param_clip_value,
          )
  
imports: from __future__ import division
  from neon import NervanaObject
  from neon.util.persist import load_class
  import logging
  import numpy
label_desc: Use image node_id 2 for calling the Adagrad obj's underlying member method code with example usage: obj.optimize(layer_list, epoch) without return types