class_docstr: Wrapped DqnAgent that supports recycled training.
class_docstr_tok: ["Wrapped","DqnAgent","that","supports","recycled","training","."]
docstr: Returns a train op to update the agent's networks.
  
  This method trains with the provided batched experience.
  
  Args:
    experience: A time-stacked trajectory object.
    weights: Optional scalar or elementwise (per-batch-entry) importance
      weights.
  
  Returns:
    A train_op.
  
  Raises:
    ValueError: If optimizers are None and no default value was provided to
      the constructor.
docstr_tok: ["Returns","a","train","op","to","update","the","agent","'s","networks",".","This","method","trains","with","the","provided","batched","experience",".","Args",":","experience",":","A","time-stacked","trajectory","object",".","weights",":","Optional","scalar","or","elementwise","(","per-batch-entry",")","importance","weights",".","Returns",":","A","train_op",".","Raises",":","ValueError",":","If","optimizers","are","None","and","no","default","value","was","provided","to","the","constructor","."]
code: def _train(self, experience, weights):
      """Returns a train op to update the agent's networks.
  
      This method trains with the provided batched experience.
  
      Args:
        experience: A time-stacked trajectory object.
        weights: Optional scalar or elementwise (per-batch-entry) importance
          weights.
  
      Returns:
        A train_op.
  
      Raises:
        ValueError: If optimizers are None and no default value was provided to
          the constructor.
      """
      tf.summary.experimental.set_step(self.train_step_counter)
      transition = self._as_transition(experience)
      time_steps, policy_steps, next_time_steps = transition
      actions = policy_steps.action
  
      trainable_critic_variables = list(
          object_identity.ObjectIdentitySet(
              self._critic_network_1.trainable_variables
              + self._critic_network_2.trainable_variables
          )
      )
  
      with tf.GradientTape(
          watch_accessed_variables=False, persistent=True
      ) as tape:
          assert trainable_critic_variables, (
              "No trainable critic variables to " "optimize."
          )
          tape.watch(trainable_critic_variables)
          critic_loss = self._critic_loss_weight * self.critic_loss(
              time_steps,
              actions,
              next_time_steps,
              td_errors_loss_fn=self._td_errors_loss_fn,
              gamma=self._gamma,
              reward_scale_factor=self._reward_scale_factor,
              weights=weights,
              training=True,
          )
  
          self.critic1_act = get_intermedieates(self._critic_network_1)
          self.critic2_act = get_intermedieates(self._critic_network_2)
          self.critic1_act_grads = (
              tape.gradient(critic_loss, self.critic1_act)
              if self.neuron_score_algo == "activation_grad"
              else None
          )
          self.critic2_act_grads = (
              tape.gradient(critic_loss, self.critic2_act)
              if self.neuron_score_algo == "activation_grad"
              else None
          )
  
      tf.debugging.check_numerics(
          critic_loss, "Critic loss is inf or nan."
      )
      critic_grads = tape.gradient(
          critic_loss, trainable_critic_variables
      )
      self._apply_gradients(
          critic_grads,
          trainable_critic_variables,
          self._critic_optimizer,
      )
  
      trainable_actor_variables = (
          self._actor_network.trainable_variables
      )
      with tf.GradientTape(
          watch_accessed_variables=False, persistent=True
      ) as tape:
          assert trainable_actor_variables, (
              "No trainable actor variables to " "optimize."
          )
          tape.watch(trainable_actor_variables)
          actor_loss = self._actor_loss_weight * self.actor_loss(
              time_steps, weights=weights, training=True
          )
  
          self.actor_act = get_intermedieates(self._actor_network)
          self.actor_act_grads = (
              tape.gradient(actor_loss, self.actor_act)
              if self.neuron_score_algo == "activation_grad"
              else None
          )
  
      tf.debugging.check_numerics(
          actor_loss, "Actor loss is inf or nan."
      )
      actor_grads = tape.gradient(actor_loss, trainable_actor_variables)
      self._apply_gradients(
          actor_grads, trainable_actor_variables, self._actor_optimizer
      )
  
      is_deadneurons_log_step = self.is_dead_neurons_log_iter()
      tf.cond(
          is_deadneurons_log_step,
          self.log_deadneurons_models,
          lambda: None,
      )
  
      if self.reset_mode:
          is_reset_step = self.is_reset_iter()
          tf.cond(is_reset_step, self.reset_models, lambda: None)
  
      is_logging = self.train_step_counter % self.log_interval == 0
      log_w_mean_c1 = functools.partial(
          self.log_weights_mean, "critic_1", self._critic_network_1
      )
      tf.cond(is_logging, log_w_mean_c1, lambda: None)
      log_w_mean_c2 = functools.partial(
          self.log_weights_mean, "critic_2", self._critic_network_2
      )
      tf.cond(is_logging, log_w_mean_c2, lambda: None)
      log_w_mean_actor = functools.partial(
          self.log_weights_mean, "actor", self._actor_network
      )
      tf.cond(is_logging, log_w_mean_actor, lambda: None)
  
      alpha_variable = [self._log_alpha]
      with tf.GradientTape(watch_accessed_variables=False) as tape:
          assert alpha_variable, "No alpha variable to optimize."
          tape.watch(alpha_variable)
          alpha_loss = self._alpha_loss_weight * self.alpha_loss(
              time_steps, weights=weights, training=True
          )
      tf.debugging.check_numerics(
          alpha_loss, "Alpha loss is inf or nan."
      )
      alpha_grads = tape.gradient(alpha_loss, alpha_variable)
      self._apply_gradients(
          alpha_grads, alpha_variable, self._alpha_optimizer
      )
  
      with tf.name_scope("Losses"):
          tf.compat.v2.summary.scalar(
              name="critic_loss",
              data=critic_loss,
              step=self.train_step_counter,
          )
          tf.compat.v2.summary.scalar(
              name="actor_loss",
              data=actor_loss,
              step=self.train_step_counter,
          )
          tf.compat.v2.summary.scalar(
              name="alpha_loss",
              data=alpha_loss,
              step=self.train_step_counter,
          )
  
      self.train_step_counter.assign_add(1)
      self._update_target()
  
      total_loss = critic_loss + actor_loss + alpha_loss
  
      extra = sac_agent.SacLossInfo(
          critic_loss=critic_loss,
          actor_loss=actor_loss,
          alpha_loss=alpha_loss,
      )
  
      return tf_agent.LossInfo(loss=total_loss, extra=extra)
  
imports: import functools
  import os
  from absl import app
  from absl import flags
  from absl import logging
  import gin
  import numpy
  import reverb
  from rigl.rl.tfagents import sparse_tanh_normal_projection_network
  import tensorflow
  from tf_agents.agents import tf_agent
  from tf_agents.agents.sac import sac_agent
  from tf_agents.environments import suite_mujoco
  from tf_agents.keras_layers import inner_reshape
  from tf_agents.metrics import py_metrics
  from tf_agents.networks import nest_map
  from tf_agents.networks import sequential
  from tf_agents.policies import greedy_policy
  from tf_agents.policies import py_tf_eager_policy
  from tf_agents.policies import random_py_policy
  from tf_agents.replay_buffers import reverb_replay_buffer
  from tf_agents.replay_buffers import reverb_utils
  from tf_agents.train import actor
  from tf_agents.train import learner
  from tf_agents.train import triggers
  from tf_agents.train.utils import spec_utils
  from tf_agents.train.utils import strategy_utils
  from tf_agents.train.utils import train_utils
  from tf_agents.utils import common
  from tf_agents.utils import object_identity
label_desc: Use image node_id 2 for calling the RecycledSacAgent obj's underlying member method code with example usage: obj._train(experience, weights) and returns: tf_agent