class_docstr: The ACER (Actor-Critic with Experience Replay) model class, https://arxiv.org/abs/1611.01224
  
  :param policy: (ActorCriticPolicy or str) The policy model to use (MlpPolicy, CnnPolicy, CnnLstmPolicy, ...)
  :param env: (Gym environment or str) The environment to learn from (if registered in Gym, can be str)
  :param gamma: (float) The discount value
  :param n_steps: (int) The number of steps to run for each environment per update
      (i.e. batch size is n_steps * n_env where n_env is number of environment copies running in parallel)
  :param num_procs: (int) The number of threads for TensorFlow operations
  
      .. deprecated:: 2.9.0
          Use `n_cpu_tf_sess` instead.
  
  :param q_coef: (float) The weight for the loss on the Q value
  :param ent_coef: (float) The weight for the entropy loss
  :param max_grad_norm: (float) The clipping value for the maximum gradient
  :param learning_rate: (float) The initial learning rate for the RMS prop optimizer
  :param lr_schedule: (str) The type of scheduler for the learning rate update ('linear', 'constant',
                            'double_linear_con', 'middle_drop' or 'double_middle_drop')
  :param rprop_epsilon: (float) RMSProp epsilon (stabilizes square root computation in denominator of RMSProp update)
      (default: 1e-5)
  :param rprop_alpha: (float) RMSProp decay parameter (default: 0.99)
  :param buffer_size: (int) The buffer size in number of steps
  :param replay_ratio: (float) The number of replay learning per on policy learning on average,
                       using a poisson distribution
  :param replay_start: (int) The minimum number of steps in the buffer, before learning replay
  :param correction_term: (float) Importance weight clipping factor (default: 10)
  :param trust_region: (bool) Whether or not algorithms estimates the gradient KL divergence
      between the old and updated policy and uses it to determine step size  (default: True)
  :param alpha: (float) The decay rate for the Exponential moving average of the parameters
  :param delta: (float) max KL divergence between the old policy and updated policy (default: 1)
  :param verbose: (int) the verbosity level: 0 none, 1 training information, 2 tensorflow debug
  :param tensorboard_log: (str) the log location for tensorboard (if None, no logging)
  :param _init_setup_model: (bool) Whether or not to build the network at the creation of the instance
  :param policy_kwargs: (dict) additional arguments to be passed to the policy on creation
  :param full_tensorboard_log: (bool) enable additional logging when using tensorboard
      WARNING: this logging can take a lot of space quickly
  :param seed: (int) Seed for the pseudo-random generators (python, numpy, tensorflow).
      If None (default), use random seed. Note that if you want completely deterministic
      results, you must set `n_cpu_tf_sess` to 1.
  :param n_cpu_tf_sess: (int) The number of threads for TensorFlow operations
      If None, the number of cpu of the current machine will be used.
class_docstr_tok: ["The","ACER","(","Actor-Critic","with","Experience","Replay",")","model","class",",","https",":","\/\/arxiv.org\/abs\/1611.01224",":","param","policy",":","(","ActorCriticPolicy","or","str",")","The","policy","model","to","use","(","MlpPolicy",",","CnnPolicy",",","CnnLstmPolicy",",","...",")",":","param","env",":","(","Gym","environment","or","str",")","The","environment","to","learn","from","(","if","registered","in","Gym",",","can","be","str",")",":","param","gamma",":","(","float",")","The","discount","value",":","param","n_steps",":","(","int",")","The","number","of","steps","to","run","for","each","environment","per","update","(","i.e",".","batch","size","is","n_steps","*","n_env","where","n_env","is","number","of","environment","copies","running","in","parallel",")",":","param","num_procs",":","(","int",")","The","number","of","threads","for","TensorFlow","operations","..","deprecated",":",":","2.9.0","Use","`","n_cpu_tf_sess","`","instead",".",":","param","q_coef",":","(","float",")","The","weight","for","the","loss","on","the","Q","value",":","param","ent_coef",":","(","float",")","The","weight","for","the","entropy","loss",":","param","max_grad_norm",":","(","float",")","The","clipping","value","for","the","maximum","gradient",":","param","learning_rate",":","(","float",")","The","initial","learning","rate","for","the","RMS","prop","optimizer",":","param","lr_schedule",":","(","str",")","The","type","of","scheduler","for","the","learning","rate","update","(","'linear","'",",","'constant","'",",","'double_linear_con","'",",","'middle_drop","'","or","'double_middle_drop","'",")",":","param","rprop_epsilon",":","(","float",")","RMSProp","epsilon","(","stabilizes","square","root","computation","in","denominator","of","RMSProp","update",")","(","default",":","1e-5",")",":","param","rprop_alpha",":","(","float",")","RMSProp","decay","parameter","(","default",":","0.99",")",":","param","buffer_size",":","(","int",")","The","buffer","size","in","number","of","steps",":","param","replay_ratio",":","(","float",")","The","number","of","replay","learning","per","on","policy","learning","on","average",",","using","a","poisson","distribution",":","param","replay_start",":","(","int",")","The","minimum","number","of","steps","in","the","buffer",",","before","learning","replay",":","param","correction_term",":","(","float",")","Importance","weight","clipping","factor","(","default",":","10",")",":","param","trust_region",":","(","bool",")","Whether","or","not","algorithms","estimates","the","gradient","KL","divergence","between","the","old","and","updated","policy","and","uses","it","to","determine","step","size","(","default",":","True",")",":","param","alpha",":","(","float",")","The","decay","rate","for","the","Exponential","moving","average","of","the","parameters",":","param","delta",":","(","float",")","max","KL","divergence","between","the","old","policy","and","updated","policy","(","default",":","1",")",":","param","verbose",":","(","int",")","the","verbosity","level",":","0","none",",","1","training","information",",","2","tensorflow","debug",":","param","tensorboard_log",":","(","str",")","the","log","location","for","tensorboard","(","if","None",",","no","logging",")",":","param","_init_setup_model",":","(","bool",")","Whether","or","not","to","build","the","network","at","the","creation","of","the","instance",":","param","policy_kwargs",":","(","dict",")","additional","arguments","to","be","passed","to","the","policy","on","creation",":","param","full_tensorboard_log",":","(","bool",")","enable","additional","logging","when","using","tensorboard","WARNING",":","this","logging","can","take","a","lot","of","space","quickly",":","param","seed",":","(","int",")","Seed","for","the","pseudo-random","generators","(","python",",","numpy",",","tensorflow",")",".","If","None","(","default",")",",","use","random","seed",".","Note","that","if","you","want","completely","deterministic","results",",","you","must","set","`","n_cpu_tf_sess","`","to","1.",":","param","n_cpu_tf_sess",":","(","int",")","The","number","of","threads","for","TensorFlow","operations","If","None",",","the","number","of","cpu","of","the","current","machine","will","be","used","."]
code: def __init__(
      self,
      policy,
      env,
      gamma=0.99,
      n_steps=20,
      num_procs=None,
      q_coef=0.5,
      ent_coef=0.01,
      max_grad_norm=10,
      learning_rate=7e-4,
      lr_schedule="linear",
      rprop_alpha=0.99,
      rprop_epsilon=1e-5,
      buffer_size=5000,
      replay_ratio=4,
      replay_start=1000,
      correction_term=10.0,
      trust_region=True,
      alpha=0.99,
      delta=1,
      verbose=0,
      tensorboard_log=None,
      _init_setup_model=True,
      policy_kwargs=None,
      full_tensorboard_log=False,
      seed=None,
      n_cpu_tf_sess=1,
  ):
      if num_procs is not None:
          warnings.warn(
              "num_procs will be removed in a future version (v3.x.x) "
              "use n_cpu_tf_sess instead",
              DeprecationWarning,
          )
          n_cpu_tf_sess = num_procs
  
      self.n_steps = n_steps
      self.replay_ratio = replay_ratio
      self.buffer_size = buffer_size
      self.replay_start = replay_start
      self.gamma = gamma
      self.alpha = alpha
      self.correction_term = correction_term
      self.q_coef = q_coef
      self.ent_coef = ent_coef
      self.trust_region = trust_region
      self.delta = delta
      self.max_grad_norm = max_grad_norm
      self.rprop_alpha = rprop_alpha
      self.rprop_epsilon = rprop_epsilon
      self.learning_rate = learning_rate
      self.lr_schedule = lr_schedule
      self.tensorboard_log = tensorboard_log
      self.full_tensorboard_log = full_tensorboard_log
  
      self.action_ph = None
      self.done_ph = None
      self.reward_ph = None
      self.mu_ph = None
      self.learning_rate_ph = None
      self.polyak_model = None
      self.learning_rate_schedule = None
      self.run_ops = None
      self.names_ops = None
      self.train_model = None
      self.step_model = None
      self.proba_step = None
      self.n_act = None
      self.n_batch = None
      self.summary = None
  
      super(ACER, self).__init__(
          policy=policy,
          env=env,
          verbose=verbose,
          requires_vec_env=True,
          _init_setup_model=_init_setup_model,
          policy_kwargs=policy_kwargs,
          seed=seed,
          n_cpu_tf_sess=n_cpu_tf_sess,
      )
  
      if _init_setup_model:
          self.setup_model()
  
imports: import time
  import warnings
  import numpy
  import tensorflow
  from gym.spaces import Discrete, Box
  from collections import deque
  from stable_baselines import logger
  from stable_baselines.common.schedules import Scheduler
  from stable_baselines.common.tf_util import batch_to_seq, seq_to_batch, check_shape, avg_norm, gradient_add, q_explained_variance, total_episode_reward_logger
  from stable_baselines.acer.buffer import Buffer
  from stable_baselines.common import ActorCriticRLModel, tf_util, SetVerbosity, TensorboardWriter
  from stable_baselines.common.runners import AbstractEnvRunner
  from stable_baselines.common.policies import ActorCriticPolicy, RecurrentActorCriticPolicy
label_desc: Use image node_id 1 to create a new ACER object from inherited base classes: ActorCriticRLModel with example: obj = ACER(policy, env, gamma, n_steps, num_procs, q_coef, ent_coef, max_grad_norm, learning_rate, lr_schedule, rprop_alpha, rprop_epsilon, buffer_size, replay_ratio, replay_start, correction_term, trust_region, alpha, delta, verbose, tensorboard_log, _init_setup_model, policy_kwargs, full_tensorboard_log, seed, n_cpu_tf_sess)