class_docstr: Adam optimizer.
  
  The Adam optimizer combines features from RMSprop and Adagrad. We
  accumulate both the first and second moments of the gradient with decay
  rates :math:`\beta_1` and :math:`\beta_2` corresponding to window sizes of
  :math:`1/\beta_1` and :math:`1/\beta_2`, respectively.
  
  .. math::
      m' &= \beta_1 m + (1-\beta_1) \nabla J
  
  .. math::
      v' &= \beta_2 v + (1-\beta_2) (\nabla J)^2
  
  We update the parameters by the ratio of the two moments:
  
  .. math::
      \theta = \theta - \alpha \frac{\hat{m}'}{\sqrt{\hat{v}'}+\epsilon}
  
  where we compute the bias-corrected moments :math:`\hat{m}'` and :math:`\hat{v}'` via
  
  .. math::
      \hat{m}' &= m'/(1-\beta_1^t)
  
  .. math::
      \hat{v}' &= v'/(1-\beta_1^t)
  
  Example usage:
  
  .. code-block:: python
  
      from neon.optimizers import Adam
  
      # use Adam
      optimizer = Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999)
class_docstr_tok: ["Adam","optimizer",".","The","Adam","optimizer","combines","features","from","RMSprop","and","Adagrad",".","We","accumulate","both","the","first","and","second","moments","of","the","gradient","with","decay","rates",":","math",":","`","\\beta_1","`","and",":","math",":","`","\\beta_2","`","corresponding","to","window","sizes","of",":","math",":","`","1\/\\beta_1","`","and",":","math",":","`","1\/\\beta_2","`",",","respectively",".","..","math",":",":","m","'","&","=","\\beta_1","m","+","(","1-\\beta_1",")","\\nabla","J","..","math",":",":","v","'","&","=","\\beta_2","v","+","(","1-\\beta_2",")","(","\\nabla","J",")","^2","We","update","the","parameters","by","the","ratio","of","the","two","moments",":","..","math",":",":","\\theta","=","\\theta","-","\\alpha","\\frac","{","\\hat","{","m","}","'","}","{","\\sqrt","{","\\hat","{","v","}","'","}","+\\epsilon","}","where","we","compute","the","bias-corrected","moments",":","math",":","`","\\hat","{","m","}","'","`","and",":","math",":","`","\\hat","{","v","}","'","`","via","..","math",":",":","\\hat","{","m","}","'","&","=","m'\/","(","1-\\beta_1^t",")","..","math",":",":","\\hat","{","v","}","'","&","=","v'\/","(","1-\\beta_1^t",")","Example","usage",":","..","code-block",":",":","python","from","neon.optimizers","import","Adam","#","use","Adam","optimizer","=","Adam","(","learning_rate=0.001",",","beta_1=0.9",",","beta_2=0.999",")"]
docstr: Apply the learning rule to all the layers and update the states.
  
  Arguments:
      param_list (list): a list of tuples of the form ((param, grad), state),
          corresponding to parameters, grads, and states of layers to be updated
      epoch (int): the current epoch, needed for the Schedule object.
docstr_tok: ["Apply","the","learning","rule","to","all","the","layers","and","update","the","states",".","Arguments",":","param_list","(","list",")",":","a","list","of","tuples","of","the","form","(","(","param",",","grad",")",",","state",")",",","corresponding","to","parameters",",","grads",",","and","states","of","layers","to","be","updated","epoch","(","int",")",":","the","current","epoch",",","needed","for","the","Schedule","object","."]
code: def optimize(self, layer_list, epoch):
      """
      Apply the learning rule to all the layers and update the states.
  
      Arguments:
          param_list (list): a list of tuples of the form ((param, grad), state),
              corresponding to parameters, grads, and states of layers to be updated
          epoch (int): the current epoch, needed for the Schedule object.
      """
      self.t = self.t + 1
      l = (
          self.learning_rate
          * self.be.sqrt(1 - self.beta_2**self.t)
          / (1 - self.beta_1**self.t)
      )
  
      param_list = get_param_list(layer_list)
  
      scale_factor = self.clip_gradient_norm(
          param_list, self.gradient_clip_norm
      )
  
      for (param, grad), states in param_list:
          param.rounding = self.stochastic_round
          if len(states) == 0:
              # running_1st_mom, running_2nd_mom
              states.extend(
                  [self.be.zeros_like(grad) for i in range(2)]
              )
  
          grad = grad / self.be.bsz
          grad = self.clip_value(grad, self.gradient_clip_value)
  
          m, v = states
          m[:] = m * self.beta_1 + (1.0 - self.beta_1) * grad
          v[:] = v * self.beta_2 + (1.0 - self.beta_2) * grad * grad
  
          param[:] = self.clip_value(
              param
              - (scale_factor * l * m)
              / (self.be.sqrt(v) + self.epsilon),
              self.param_clip_value,
          )
  
imports: from __future__ import division
  from neon import NervanaObject
  from neon.util.persist import load_class
  import logging
  import numpy
label_desc: Use image node_id 2 for calling the Adam obj's underlying member method code with example usage: obj.optimize(layer_list, epoch) without return types