class_docstr: The ACER (Actor-Critic with Experience Replay) model class, https://arxiv.org/abs/1611.01224
  
  :param policy: (ActorCriticPolicy or str) The policy model to use (MlpPolicy, CnnPolicy, CnnLstmPolicy, ...)
  :param env: (Gym environment or str) The environment to learn from (if registered in Gym, can be str)
  :param gamma: (float) The discount value
  :param n_steps: (int) The number of steps to run for each environment per update
      (i.e. batch size is n_steps * n_env where n_env is number of environment copies running in parallel)
  :param num_procs: (int) The number of threads for TensorFlow operations
  
      .. deprecated:: 2.9.0
          Use `n_cpu_tf_sess` instead.
  
  :param q_coef: (float) The weight for the loss on the Q value
  :param ent_coef: (float) The weight for the entropy loss
  :param max_grad_norm: (float) The clipping value for the maximum gradient
  :param learning_rate: (float) The initial learning rate for the RMS prop optimizer
  :param lr_schedule: (str) The type of scheduler for the learning rate update ('linear', 'constant',
                            'double_linear_con', 'middle_drop' or 'double_middle_drop')
  :param rprop_epsilon: (float) RMSProp epsilon (stabilizes square root computation in denominator of RMSProp update)
      (default: 1e-5)
  :param rprop_alpha: (float) RMSProp decay parameter (default: 0.99)
  :param buffer_size: (int) The buffer size in number of steps
  :param replay_ratio: (float) The number of replay learning per on policy learning on average,
                       using a poisson distribution
  :param replay_start: (int) The minimum number of steps in the buffer, before learning replay
  :param correction_term: (float) Importance weight clipping factor (default: 10)
  :param trust_region: (bool) Whether or not algorithms estimates the gradient KL divergence
      between the old and updated policy and uses it to determine step size  (default: True)
  :param alpha: (float) The decay rate for the Exponential moving average of the parameters
  :param delta: (float) max KL divergence between the old policy and updated policy (default: 1)
  :param verbose: (int) the verbosity level: 0 none, 1 training information, 2 tensorflow debug
  :param tensorboard_log: (str) the log location for tensorboard (if None, no logging)
  :param _init_setup_model: (bool) Whether or not to build the network at the creation of the instance
  :param policy_kwargs: (dict) additional arguments to be passed to the policy on creation
  :param full_tensorboard_log: (bool) enable additional logging when using tensorboard
      WARNING: this logging can take a lot of space quickly
  :param seed: (int) Seed for the pseudo-random generators (python, numpy, tensorflow).
      If None (default), use random seed. Note that if you want completely deterministic
      results, you must set `n_cpu_tf_sess` to 1.
  :param n_cpu_tf_sess: (int) The number of threads for TensorFlow operations
      If None, the number of cpu of the current machine will be used.
class_docstr_tok: ["The","ACER","(","Actor-Critic","with","Experience","Replay",")","model","class",",","https",":","\/\/arxiv.org\/abs\/1611.01224",":","param","policy",":","(","ActorCriticPolicy","or","str",")","The","policy","model","to","use","(","MlpPolicy",",","CnnPolicy",",","CnnLstmPolicy",",","...",")",":","param","env",":","(","Gym","environment","or","str",")","The","environment","to","learn","from","(","if","registered","in","Gym",",","can","be","str",")",":","param","gamma",":","(","float",")","The","discount","value",":","param","n_steps",":","(","int",")","The","number","of","steps","to","run","for","each","environment","per","update","(","i.e",".","batch","size","is","n_steps","*","n_env","where","n_env","is","number","of","environment","copies","running","in","parallel",")",":","param","num_procs",":","(","int",")","The","number","of","threads","for","TensorFlow","operations","..","deprecated",":",":","2.9.0","Use","`","n_cpu_tf_sess","`","instead",".",":","param","q_coef",":","(","float",")","The","weight","for","the","loss","on","the","Q","value",":","param","ent_coef",":","(","float",")","The","weight","for","the","entropy","loss",":","param","max_grad_norm",":","(","float",")","The","clipping","value","for","the","maximum","gradient",":","param","learning_rate",":","(","float",")","The","initial","learning","rate","for","the","RMS","prop","optimizer",":","param","lr_schedule",":","(","str",")","The","type","of","scheduler","for","the","learning","rate","update","(","'linear","'",",","'constant","'",",","'double_linear_con","'",",","'middle_drop","'","or","'double_middle_drop","'",")",":","param","rprop_epsilon",":","(","float",")","RMSProp","epsilon","(","stabilizes","square","root","computation","in","denominator","of","RMSProp","update",")","(","default",":","1e-5",")",":","param","rprop_alpha",":","(","float",")","RMSProp","decay","parameter","(","default",":","0.99",")",":","param","buffer_size",":","(","int",")","The","buffer","size","in","number","of","steps",":","param","replay_ratio",":","(","float",")","The","number","of","replay","learning","per","on","policy","learning","on","average",",","using","a","poisson","distribution",":","param","replay_start",":","(","int",")","The","minimum","number","of","steps","in","the","buffer",",","before","learning","replay",":","param","correction_term",":","(","float",")","Importance","weight","clipping","factor","(","default",":","10",")",":","param","trust_region",":","(","bool",")","Whether","or","not","algorithms","estimates","the","gradient","KL","divergence","between","the","old","and","updated","policy","and","uses","it","to","determine","step","size","(","default",":","True",")",":","param","alpha",":","(","float",")","The","decay","rate","for","the","Exponential","moving","average","of","the","parameters",":","param","delta",":","(","float",")","max","KL","divergence","between","the","old","policy","and","updated","policy","(","default",":","1",")",":","param","verbose",":","(","int",")","the","verbosity","level",":","0","none",",","1","training","information",",","2","tensorflow","debug",":","param","tensorboard_log",":","(","str",")","the","log","location","for","tensorboard","(","if","None",",","no","logging",")",":","param","_init_setup_model",":","(","bool",")","Whether","or","not","to","build","the","network","at","the","creation","of","the","instance",":","param","policy_kwargs",":","(","dict",")","additional","arguments","to","be","passed","to","the","policy","on","creation",":","param","full_tensorboard_log",":","(","bool",")","enable","additional","logging","when","using","tensorboard","WARNING",":","this","logging","can","take","a","lot","of","space","quickly",":","param","seed",":","(","int",")","Seed","for","the","pseudo-random","generators","(","python",",","numpy",",","tensorflow",")",".","If","None","(","default",")",",","use","random","seed",".","Note","that","if","you","want","completely","deterministic","results",",","you","must","set","`","n_cpu_tf_sess","`","to","1.",":","param","n_cpu_tf_sess",":","(","int",")","The","number","of","threads","for","TensorFlow","operations","If","None",",","the","number","of","cpu","of","the","current","machine","will","be","used","."]
code: def learn(
      self,
      total_timesteps,
      callback=None,
      log_interval=100,
      tb_log_name="ACER",
      reset_num_timesteps=True,
  ):
      new_tb_log = self._init_num_timesteps(reset_num_timesteps)
      callback = self._init_callback(callback)
  
      with SetVerbosity(self.verbose), TensorboardWriter(
          self.graph, self.tensorboard_log, tb_log_name, new_tb_log
      ) as writer:
          self._setup_learn()
  
          self.learning_rate_schedule = Scheduler(
              initial_value=self.learning_rate,
              n_values=total_timesteps,
              schedule=self.lr_schedule,
          )
  
          episode_stats = EpisodeStats(self.n_steps, self.n_envs)
  
          if self.replay_ratio > 0:
              buffer = Buffer(
                  env=self.env,
                  n_steps=self.n_steps,
                  size=self.buffer_size,
              )
          else:
              buffer = None
  
          t_start = time.time()
          callback.on_training_start(locals(), globals())
  
          # n_batch samples, 1 on_policy call and multiple off-policy calls
          for steps in range(0, total_timesteps, self.n_batch):
              callback.on_rollout_start()
  
              (
                  enc_obs,
                  obs,
                  actions,
                  rewards,
                  mus,
                  dones,
                  masks,
              ) = self.runner.run(callback)
              callback.update_locals(locals())
              callback.on_rollout_end()
  
              # Early stopping due to the callback
              if not self.runner.continue_training:
                  break
  
              episode_stats.feed(rewards, dones)
  
              if buffer is not None:
                  buffer.put(
                      enc_obs, actions, rewards, mus, dones, masks
                  )
  
              if writer is not None:
                  total_episode_reward_logger(
                      self.episode_reward,
                      rewards.reshape((self.n_envs, self.n_steps)),
                      dones.reshape((self.n_envs, self.n_steps)),
                      writer,
                      self.num_timesteps,
                  )
  
              # reshape stuff correctly
              obs = obs.reshape(self.runner.batch_ob_shape)
              actions = actions.reshape([self.n_batch])
              rewards = rewards.reshape([self.n_batch])
              mus = mus.reshape([self.n_batch, self.n_act])
              dones = dones.reshape([self.n_batch])
              masks = masks.reshape([self.runner.batch_ob_shape[0]])
  
              names_ops, values_ops = self._train_step(
                  obs,
                  actions,
                  rewards,
                  dones,
                  mus,
                  self.initial_state,
                  masks,
                  self.num_timesteps,
                  writer,
              )
  
              if self.verbose >= 1 and (
                  int(steps / self.n_batch) % log_interval == 0
              ):
                  logger.record_tabular(
                      "total_timesteps", self.num_timesteps
                  )
                  logger.record_tabular(
                      "fps", int(steps / (time.time() - t_start))
                  )
                  # IMP: In EpisodicLife env, during training, we get done=True at each loss of life,
                  # not just at the terminal state. Thus, this is mean until end of life, not end of episode.
                  # For true episode rewards, see the monitor files in the log folder.
                  logger.record_tabular(
                      "mean_episode_length", episode_stats.mean_length()
                  )
                  logger.record_tabular(
                      "mean_episode_reward", episode_stats.mean_reward()
                  )
                  for name, val in zip(names_ops, values_ops):
                      logger.record_tabular(name, float(val))
                  logger.dump_tabular()
  
              if (
                  self.replay_ratio > 0
                  and buffer is not None
                  and buffer.has_atleast(self.replay_start)
              ):
                  samples_number = np.random.poisson(self.replay_ratio)
                  for _ in range(samples_number):
                      # get obs, actions, rewards, mus, dones from buffer.
                      (
                          obs,
                          actions,
                          rewards,
                          mus,
                          dones,
                          masks,
                      ) = buffer.get()
  
                      # reshape stuff correctly
                      obs = obs.reshape(self.runner.batch_ob_shape)
                      actions = actions.reshape([self.n_batch])
                      rewards = rewards.reshape([self.n_batch])
                      mus = mus.reshape([self.n_batch, self.n_act])
                      dones = dones.reshape([self.n_batch])
                      masks = masks.reshape(
                          [self.runner.batch_ob_shape[0]]
                      )
  
                      self._train_step(
                          obs,
                          actions,
                          rewards,
                          dones,
                          mus,
                          self.initial_state,
                          masks,
                          self.num_timesteps,
                      )
  
      callback.on_training_end()
  
      return self
  
imports: import time
  import warnings
  import numpy
  import tensorflow
  from gym.spaces import Discrete, Box
  from collections import deque
  from stable_baselines import logger
  from stable_baselines.common.schedules import Scheduler
  from stable_baselines.common.tf_util import batch_to_seq, seq_to_batch, check_shape, avg_norm, gradient_add, q_explained_variance, total_episode_reward_logger
  from stable_baselines.acer.buffer import Buffer
  from stable_baselines.common import ActorCriticRLModel, tf_util, SetVerbosity, TensorboardWriter
  from stable_baselines.common.runners import AbstractEnvRunner
  from stable_baselines.common.policies import ActorCriticPolicy, RecurrentActorCriticPolicy
label_desc: Use image node_id 7 for calling the ACER obj's underlying member method code with example usage: obj.learn(total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps) and returns: self