class_docstr: The ACER (Actor-Critic with Experience Replay) model class, https://arxiv.org/abs/1611.01224
  
  :param policy: (ActorCriticPolicy or str) The policy model to use (MlpPolicy, CnnPolicy, CnnLstmPolicy, ...)
  :param env: (Gym environment or str) The environment to learn from (if registered in Gym, can be str)
  :param gamma: (float) The discount value
  :param n_steps: (int) The number of steps to run for each environment per update
      (i.e. batch size is n_steps * n_env where n_env is number of environment copies running in parallel)
  :param num_procs: (int) The number of threads for TensorFlow operations
  
      .. deprecated:: 2.9.0
          Use `n_cpu_tf_sess` instead.
  
  :param q_coef: (float) The weight for the loss on the Q value
  :param ent_coef: (float) The weight for the entropy loss
  :param max_grad_norm: (float) The clipping value for the maximum gradient
  :param learning_rate: (float) The initial learning rate for the RMS prop optimizer
  :param lr_schedule: (str) The type of scheduler for the learning rate update ('linear', 'constant',
                            'double_linear_con', 'middle_drop' or 'double_middle_drop')
  :param rprop_epsilon: (float) RMSProp epsilon (stabilizes square root computation in denominator of RMSProp update)
      (default: 1e-5)
  :param rprop_alpha: (float) RMSProp decay parameter (default: 0.99)
  :param buffer_size: (int) The buffer size in number of steps
  :param replay_ratio: (float) The number of replay learning per on policy learning on average,
                       using a poisson distribution
  :param replay_start: (int) The minimum number of steps in the buffer, before learning replay
  :param correction_term: (float) Importance weight clipping factor (default: 10)
  :param trust_region: (bool) Whether or not algorithms estimates the gradient KL divergence
      between the old and updated policy and uses it to determine step size  (default: True)
  :param alpha: (float) The decay rate for the Exponential moving average of the parameters
  :param delta: (float) max KL divergence between the old policy and updated policy (default: 1)
  :param verbose: (int) the verbosity level: 0 none, 1 training information, 2 tensorflow debug
  :param tensorboard_log: (str) the log location for tensorboard (if None, no logging)
  :param _init_setup_model: (bool) Whether or not to build the network at the creation of the instance
  :param policy_kwargs: (dict) additional arguments to be passed to the policy on creation
  :param full_tensorboard_log: (bool) enable additional logging when using tensorboard
      WARNING: this logging can take a lot of space quickly
  :param seed: (int) Seed for the pseudo-random generators (python, numpy, tensorflow).
      If None (default), use random seed. Note that if you want completely deterministic
      results, you must set `n_cpu_tf_sess` to 1.
  :param n_cpu_tf_sess: (int) The number of threads for TensorFlow operations
      If None, the number of cpu of the current machine will be used.
class_docstr_tok: ["The","ACER","(","Actor-Critic","with","Experience","Replay",")","model","class",",","https",":","\/\/arxiv.org\/abs\/1611.01224",":","param","policy",":","(","ActorCriticPolicy","or","str",")","The","policy","model","to","use","(","MlpPolicy",",","CnnPolicy",",","CnnLstmPolicy",",","...",")",":","param","env",":","(","Gym","environment","or","str",")","The","environment","to","learn","from","(","if","registered","in","Gym",",","can","be","str",")",":","param","gamma",":","(","float",")","The","discount","value",":","param","n_steps",":","(","int",")","The","number","of","steps","to","run","for","each","environment","per","update","(","i.e",".","batch","size","is","n_steps","*","n_env","where","n_env","is","number","of","environment","copies","running","in","parallel",")",":","param","num_procs",":","(","int",")","The","number","of","threads","for","TensorFlow","operations","..","deprecated",":",":","2.9.0","Use","`","n_cpu_tf_sess","`","instead",".",":","param","q_coef",":","(","float",")","The","weight","for","the","loss","on","the","Q","value",":","param","ent_coef",":","(","float",")","The","weight","for","the","entropy","loss",":","param","max_grad_norm",":","(","float",")","The","clipping","value","for","the","maximum","gradient",":","param","learning_rate",":","(","float",")","The","initial","learning","rate","for","the","RMS","prop","optimizer",":","param","lr_schedule",":","(","str",")","The","type","of","scheduler","for","the","learning","rate","update","(","'linear","'",",","'constant","'",",","'double_linear_con","'",",","'middle_drop","'","or","'double_middle_drop","'",")",":","param","rprop_epsilon",":","(","float",")","RMSProp","epsilon","(","stabilizes","square","root","computation","in","denominator","of","RMSProp","update",")","(","default",":","1e-5",")",":","param","rprop_alpha",":","(","float",")","RMSProp","decay","parameter","(","default",":","0.99",")",":","param","buffer_size",":","(","int",")","The","buffer","size","in","number","of","steps",":","param","replay_ratio",":","(","float",")","The","number","of","replay","learning","per","on","policy","learning","on","average",",","using","a","poisson","distribution",":","param","replay_start",":","(","int",")","The","minimum","number","of","steps","in","the","buffer",",","before","learning","replay",":","param","correction_term",":","(","float",")","Importance","weight","clipping","factor","(","default",":","10",")",":","param","trust_region",":","(","bool",")","Whether","or","not","algorithms","estimates","the","gradient","KL","divergence","between","the","old","and","updated","policy","and","uses","it","to","determine","step","size","(","default",":","True",")",":","param","alpha",":","(","float",")","The","decay","rate","for","the","Exponential","moving","average","of","the","parameters",":","param","delta",":","(","float",")","max","KL","divergence","between","the","old","policy","and","updated","policy","(","default",":","1",")",":","param","verbose",":","(","int",")","the","verbosity","level",":","0","none",",","1","training","information",",","2","tensorflow","debug",":","param","tensorboard_log",":","(","str",")","the","log","location","for","tensorboard","(","if","None",",","no","logging",")",":","param","_init_setup_model",":","(","bool",")","Whether","or","not","to","build","the","network","at","the","creation","of","the","instance",":","param","policy_kwargs",":","(","dict",")","additional","arguments","to","be","passed","to","the","policy","on","creation",":","param","full_tensorboard_log",":","(","bool",")","enable","additional","logging","when","using","tensorboard","WARNING",":","this","logging","can","take","a","lot","of","space","quickly",":","param","seed",":","(","int",")","Seed","for","the","pseudo-random","generators","(","python",",","numpy",",","tensorflow",")",".","If","None","(","default",")",",","use","random","seed",".","Note","that","if","you","want","completely","deterministic","results",",","you","must","set","`","n_cpu_tf_sess","`","to","1.",":","param","n_cpu_tf_sess",":","(","int",")","The","number","of","threads","for","TensorFlow","operations","If","None",",","the","number","of","cpu","of","the","current","machine","will","be","used","."]
code: def setup_model(self):
      with SetVerbosity(self.verbose):
          assert issubclass(self.policy, ActorCriticPolicy), (
              "Error: the input policy for the ACER model must be "
              "an instance of common.policies.ActorCriticPolicy."
          )
  
          if isinstance(self.action_space, Discrete):
              self.n_act = self.action_space.n
              continuous = False
          elif isinstance(self.action_space, Box):
              # self.n_act = self.action_space.shape[-1]
              # continuous = True
              raise NotImplementedError(
                  "WIP: Acer does not support Continuous actions yet."
              )
          else:
              raise ValueError(
                  "Error: ACER does not work with {} actions space.".format(
                      self.action_space
                  )
              )
  
          self.n_batch = self.n_envs * self.n_steps
  
          self.graph = tf.Graph()
          with self.graph.as_default():
              self.sess = tf_util.make_session(
                  num_cpu=self.n_cpu_tf_sess, graph=self.graph
              )
              self.set_random_seed(self.seed)
              n_batch_step = None
              if issubclass(self.policy, RecurrentActorCriticPolicy):
                  n_batch_step = self.n_envs
              n_batch_train = self.n_envs * (self.n_steps + 1)
  
              step_model = self.policy(
                  self.sess,
                  self.observation_space,
                  self.action_space,
                  self.n_envs,
                  1,
                  n_batch_step,
                  reuse=False,
                  **self.policy_kwargs,
              )
  
              self.params = tf_util.get_trainable_vars("model")
  
              with tf.variable_scope(
                  "train_model",
                  reuse=True,
                  custom_getter=tf_util.outer_scope_getter(
                      "train_model"
                  ),
              ):
                  train_model = self.policy(
                      self.sess,
                      self.observation_space,
                      self.action_space,
                      self.n_envs,
                      self.n_steps + 1,
                      n_batch_train,
                      reuse=True,
                      **self.policy_kwargs,
                  )
  
              with tf.variable_scope("moving_average"):
                  # create averaged model
                  ema = tf.train.ExponentialMovingAverage(self.alpha)
                  ema_apply_op = ema.apply(self.params)
  
                  def custom_getter(getter, name, *args, **kwargs):
                      name = name.replace("polyak_model/", "")
                      val = ema.average(getter(name, *args, **kwargs))
                      return val
  
              with tf.variable_scope(
                  "polyak_model",
                  reuse=True,
                  custom_getter=custom_getter,
              ):
                  self.polyak_model = polyak_model = self.policy(
                      self.sess,
                      self.observation_space,
                      self.action_space,
                      self.n_envs,
                      self.n_steps + 1,
                      self.n_envs * (self.n_steps + 1),
                      reuse=True,
                      **self.policy_kwargs,
                  )
  
              with tf.variable_scope("loss", reuse=False):
                  self.done_ph = tf.placeholder(
                      tf.float32, [self.n_batch]
                  )  # dones
                  self.reward_ph = tf.placeholder(
                      tf.float32, [self.n_batch]
                  )  # rewards, not returns
                  self.mu_ph = tf.placeholder(
                      tf.float32, [self.n_batch, self.n_act]
                  )  # mu's
                  self.action_ph = (
                      train_model.pdtype.sample_placeholder(
                          [self.n_batch]
                      )
                  )
                  self.learning_rate_ph = tf.placeholder(tf.float32, [])
                  eps = 1e-6
  
                  # Notation: (var) = batch variable, (var)s = sequence variable,
                  # (var)_i = variable index by action at step i
                  # shape is [n_envs * (n_steps + 1)]
                  if continuous:
                      value = train_model.value_flat
                  else:
                      value = tf.reduce_sum(
                          train_model.policy_proba
                          * train_model.q_value,
                          axis=-1,
                      )
  
                  rho, rho_i_ = None, None
                  if continuous:
                      action_ = strip(
                          train_model.proba_distribution.sample(),
                          self.n_envs,
                          self.n_steps,
                      )
                      distribution_f = tf.contrib.distributions.MultivariateNormalDiag(
                          loc=strip(
                              train_model.proba_distribution.mean,
                              self.n_envs,
                              self.n_steps,
                          ),
                          scale_diag=strip(
                              train_model.proba_distribution.logstd,
                              self.n_envs,
                              self.n_steps,
                          ),
                      )
                      f_polyak = tf.contrib.distributions.MultivariateNormalDiag(
                          loc=strip(
                              polyak_model.proba_distribution.mean,
                              self.n_envs,
                              self.n_steps,
                          ),
                          scale_diag=strip(
                              polyak_model.proba_distribution.logstd,
                              self.n_envs,
                              self.n_steps,
                          ),
                      )
  
                      f_i = distribution_f.prob(self.action_ph)
                      f_i_ = distribution_f.prob(action_)
                      f_polyak_i = f_polyak.prob(self.action_ph)
                      phi_i = strip(
                          train_model.proba_distribution.mean,
                          self.n_envs,
                          self.n_steps,
                      )
  
                      q_value = strip(
                          train_model.value_fn,
                          self.n_envs,
                          self.n_steps,
                      )
                      q_i = q_value[:, 0]
  
                      rho_i = tf.reshape(f_i, [-1, 1]) / (
                          self.mu_ph + eps
                      )
                      rho_i_ = tf.reshape(f_i_, [-1, 1]) / (
                          self.mu_ph + eps
                      )
  
                      qret = q_retrace(
                          self.reward_ph,
                          self.done_ph,
                          q_i,
                          value,
                          tf.pow(rho_i, 1 / self.n_act),
                          self.n_envs,
                          self.n_steps,
                          self.gamma,
                      )
                  else:
                      # strip off last step
                      # f is a distribution, chosen to be Gaussian distributions
                      # with fixed diagonal covariance and mean \phi(x)
                      # in the paper
                      distribution_f, f_polyak, q_value = map(
                          lambda variables: strip(
                              variables, self.n_envs, self.n_steps
                          ),
                          [
                              train_model.policy_proba,
                              polyak_model.policy_proba,
                              train_model.q_value,
                          ],
                      )
  
                      # Get pi and q values for actions taken
                      f_i = get_by_index(distribution_f, self.action_ph)
                      f_i_ = distribution_f
                      phi_i = distribution_f
                      f_polyak_i = f_polyak
  
                      q_i = get_by_index(q_value, self.action_ph)
  
                      # Compute ratios for importance truncation
                      rho = distribution_f / (self.mu_ph + eps)
                      rho_i = get_by_index(rho, self.action_ph)
  
                      # Calculate Q_retrace targets
                      qret = q_retrace(
                          self.reward_ph,
                          self.done_ph,
                          q_i,
                          value,
                          rho_i,
                          self.n_envs,
                          self.n_steps,
                          self.gamma,
                      )
  
                  # Calculate losses
                  # Entropy
                  entropy = tf.reduce_sum(
                      train_model.proba_distribution.entropy()
                  )
  
                  # Policy Gradient loss, with truncated importance sampling & bias correction
                  value = strip(value, self.n_envs, self.n_steps, True)
                  # check_shape([qret, value, rho_i, f_i], [[self.n_envs * self.n_steps]] * 4)
                  # check_shape([rho, distribution_f, q_value], [[self.n_envs * self.n_steps, self.n_act]] * 2)
  
                  # Truncated importance sampling
                  adv = qret - value
                  log_f = tf.log(f_i + eps)
                  # [n_envs * n_steps]
                  gain_f = log_f * tf.stop_gradient(
                      adv * tf.minimum(self.correction_term, rho_i)
                  )
                  loss_f = -tf.reduce_mean(gain_f)
  
                  # Bias correction for the truncation
                  adv_bc = q_value - tf.reshape(
                      value, [self.n_envs * self.n_steps, 1]
                  )  # [n_envs * n_steps, n_act]
  
                  # check_shape([adv_bc, log_f_bc], [[self.n_envs * self.n_steps, self.n_act]] * 2)
                  if continuous:
                      gain_bc = tf.stop_gradient(
                          adv_bc
                          * tf.nn.relu(
                              1.0
                              - (self.correction_term / (rho_i_ + eps))
                          )
                          * f_i_
                      )
                  else:
                      log_f_bc = tf.log(f_i_ + eps)  # / (f_old + eps)
                      gain_bc = tf.reduce_sum(
                          log_f_bc
                          * tf.stop_gradient(
                              adv_bc
                              * tf.nn.relu(
                                  1.0
                                  - (self.correction_term / (rho + eps))
                              )
                              * f_i_
                          ),
                          axis=1,
                      )
                  # IMP: This is sum, as expectation wrt f
                  loss_bc = -tf.reduce_mean(gain_bc)
  
                  loss_policy = loss_f + loss_bc
  
                  # Value/Q function loss, and explained variance
                  check_shape(
                      [qret, q_i], [[self.n_envs * self.n_steps]] * 2
                  )
                  explained_variance = q_explained_variance(
                      tf.reshape(q_i, [self.n_envs, self.n_steps]),
                      tf.reshape(qret, [self.n_envs, self.n_steps]),
                  )
                  loss_q = tf.reduce_mean(
                      tf.square(tf.stop_gradient(qret) - q_i) * 0.5
                  )
  
                  # Net loss
                  check_shape([loss_policy, loss_q, entropy], [[]] * 3)
                  loss = (
                      loss_policy
                      + self.q_coef * loss_q
                      - self.ent_coef * entropy
                  )
  
                  tf.summary.scalar("entropy_loss", entropy)
                  tf.summary.scalar("policy_gradient_loss", loss_policy)
                  tf.summary.scalar("value_function_loss", loss_q)
                  tf.summary.scalar("loss", loss)
  
                  norm_grads_q, norm_grads_policy, avg_norm_grads_f = (
                      None,
                      None,
                      None,
                  )
                  (
                      avg_norm_k,
                      avg_norm_g,
                      avg_norm_k_dot_g,
                      avg_norm_adj,
                  ) = (None, None, None, None)
                  if self.trust_region:
                      # [n_envs * n_steps, n_act]
                      grad = tf.gradients(
                          -(loss_policy - self.ent_coef * entropy)
                          * self.n_steps
                          * self.n_envs,
                          phi_i,
                      )
                      # [n_envs * n_steps, n_act] # Directly computed gradient of KL divergence wrt f
                      kl_grad = -f_polyak_i / (f_i_ + eps)
                      k_dot_g = tf.reduce_sum(kl_grad * grad, axis=-1)
                      adj = tf.maximum(
                          0.0,
                          (
                              tf.reduce_sum(kl_grad * grad, axis=-1)
                              - self.delta
                          )
                          / (
                              tf.reduce_sum(tf.square(kl_grad), axis=-1)
                              + eps
                          ),
                      )  # [n_envs * n_steps]
  
                      # Calculate stats (before doing adjustment) for logging.
                      avg_norm_k = avg_norm(kl_grad)
                      avg_norm_g = avg_norm(grad)
                      avg_norm_k_dot_g = tf.reduce_mean(tf.abs(k_dot_g))
                      avg_norm_adj = tf.reduce_mean(tf.abs(adj))
  
                      grad = (
                          grad
                          - tf.reshape(
                              adj, [self.n_envs * self.n_steps, 1]
                          )
                          * kl_grad
                      )
                      # These are turst region adjusted gradients wrt f ie statistics of policy pi
                      grads_f = -grad / (self.n_envs * self.n_steps)
                      grads_policy = tf.gradients(
                          f_i_, self.params, grads_f
                      )
                      grads_q = tf.gradients(
                          loss_q * self.q_coef, self.params
                      )
                      grads = [
                          gradient_add(
                              g1, g2, param, verbose=self.verbose
                          )
                          for (g1, g2, param) in zip(
                              grads_policy, grads_q, self.params
                          )
                      ]
  
                      avg_norm_grads_f = avg_norm(grads_f) * (
                          self.n_steps * self.n_envs
                      )
                      norm_grads_q = tf.global_norm(grads_q)
                      norm_grads_policy = tf.global_norm(grads_policy)
                  else:
                      grads = tf.gradients(loss, self.params)
  
                  norm_grads = None
                  if self.max_grad_norm is not None:
                      grads, norm_grads = tf.clip_by_global_norm(
                          grads, self.max_grad_norm
                      )
                  grads = list(zip(grads, self.params))
  
              with tf.variable_scope("input_info", reuse=False):
                  tf.summary.scalar(
                      "rewards", tf.reduce_mean(self.reward_ph)
                  )
                  tf.summary.scalar(
                      "learning_rate",
                      tf.reduce_mean(self.learning_rate),
                  )
                  tf.summary.scalar("advantage", tf.reduce_mean(adv))
                  tf.summary.scalar(
                      "action_probability", tf.reduce_mean(self.mu_ph)
                  )
  
                  if self.full_tensorboard_log:
                      tf.summary.histogram("rewards", self.reward_ph)
                      tf.summary.histogram(
                          "learning_rate", self.learning_rate
                      )
                      tf.summary.histogram("advantage", adv)
                      tf.summary.histogram(
                          "action_probability", self.mu_ph
                      )
                      if tf_util.is_image(self.observation_space):
                          tf.summary.image(
                              "observation", train_model.obs_ph
                          )
                      else:
                          tf.summary.histogram(
                              "observation", train_model.obs_ph
                          )
  
              trainer = tf.train.RMSPropOptimizer(
                  learning_rate=self.learning_rate_ph,
                  decay=self.rprop_alpha,
                  epsilon=self.rprop_epsilon,
              )
              _opt_op = trainer.apply_gradients(grads)
  
              # so when you call _train, you first do the gradient step, then you apply ema
              with tf.control_dependencies([_opt_op]):
                  _train = tf.group(ema_apply_op)
  
              # Ops/Summaries to run, and their names for logging
              assert norm_grads is not None
              run_ops = [
                  _train,
                  loss,
                  loss_q,
                  entropy,
                  loss_policy,
                  loss_f,
                  loss_bc,
                  explained_variance,
                  norm_grads,
              ]
              names_ops = [
                  "loss",
                  "loss_q",
                  "entropy",
                  "loss_policy",
                  "loss_f",
                  "loss_bc",
                  "explained_variance",
                  "norm_grads",
              ]
              if self.trust_region:
                  self.run_ops = run_ops + [
                      norm_grads_q,
                      norm_grads_policy,
                      avg_norm_grads_f,
                      avg_norm_k,
                      avg_norm_g,
                      avg_norm_k_dot_g,
                      avg_norm_adj,
                  ]
                  self.names_ops = names_ops + [
                      "norm_grads_q",
                      "norm_grads_policy",
                      "avg_norm_grads_f",
                      "avg_norm_k",
                      "avg_norm_g",
                      "avg_norm_k_dot_g",
                      "avg_norm_adj",
                  ]
  
              self.train_model = train_model
              self.step_model = step_model
              self.step = step_model.step
              self.proba_step = step_model.proba_step
              self.initial_state = step_model.initial_state
  
              tf.global_variables_initializer().run(session=self.sess)
  
              self.summary = tf.summary.merge_all()
  
imports: import time
  import warnings
  import numpy
  import tensorflow
  from gym.spaces import Discrete, Box
  from collections import deque
  from stable_baselines import logger
  from stable_baselines.common.schedules import Scheduler
  from stable_baselines.common.tf_util import batch_to_seq, seq_to_batch, check_shape, avg_norm, gradient_add, q_explained_variance, total_episode_reward_logger
  from stable_baselines.acer.buffer import Buffer
  from stable_baselines.common import ActorCriticRLModel, tf_util, SetVerbosity, TensorboardWriter
  from stable_baselines.common.runners import AbstractEnvRunner
  from stable_baselines.common.policies import ActorCriticPolicy, RecurrentActorCriticPolicy
label_desc: Use image node_id 5 for calling the ACER obj's underlying member method code with example usage: obj.setup_model() and returns: val