class_docstr: A TD3 Agent.
class_docstr_tok: ["A","TD3","Agent","."]
docstr: Computes the critic loss for TD3 training.
  
  Args:
    time_steps: A batch of timesteps.
    actions: A batch of actions.
    next_time_steps: A batch of next timesteps.
    weights: Optional scalar or element-wise (per-batch-entry) importance
      weights.
    training: Whether this loss is being used for training.
  
  Returns:
    critic_loss: A scalar critic loss.
docstr_tok: ["Computes","the","critic","loss","for","TD3","training",".","Args",":","time_steps",":","A","batch","of","timesteps",".","actions",":","A","batch","of","actions",".","next_time_steps",":","A","batch","of","next","timesteps",".","weights",":","Optional","scalar","or","element-wise","(","per-batch-entry",")","importance","weights",".","training",":","Whether","this","loss","is","being","used","for","training",".","Returns",":","critic_loss",":","A","scalar","critic","loss","."]
code: def critic_loss(
      self,
      time_steps: ts.TimeStep,
      actions: types.Tensor,
      next_time_steps: ts.TimeStep,
      weights: Optional[types.Tensor] = None,
      training: bool = False,
  ) -> types.Tensor:
      """Computes the critic loss for TD3 training.
  
      Args:
        time_steps: A batch of timesteps.
        actions: A batch of actions.
        next_time_steps: A batch of next timesteps.
        weights: Optional scalar or element-wise (per-batch-entry) importance
          weights.
        training: Whether this loss is being used for training.
  
      Returns:
        critic_loss: A scalar critic loss.
      """
      with tf.name_scope("critic_loss"):
          target_actions, _ = self._target_actor_network(
              next_time_steps.observation,
              next_time_steps.step_type,
              training=training,
          )
  
          # Add gaussian noise to each action before computing target q values
          def add_noise_to_action(
              action,
          ):  # pylint: disable=missing-docstring
              dist = tfp.distributions.Normal(
                  loc=tf.zeros_like(action),
                  scale=self._target_policy_noise
                  * tf.ones_like(action),
              )
              noise = dist.sample()
              noise = tf.clip_by_value(
                  noise,
                  -self._target_policy_noise_clip,
                  self._target_policy_noise_clip,
              )
              return action + noise
  
          noisy_target_actions = tf.nest.map_structure(
              add_noise_to_action, target_actions
          )
  
          # Target q-values are the min of the two networks
          target_q_input_1 = (
              next_time_steps.observation,
              noisy_target_actions,
          )
          target_q_values_1, _ = self._target_critic_network_1(
              target_q_input_1,
              next_time_steps.step_type,
              training=False,
          )
          target_q_input_2 = (
              next_time_steps.observation,
              noisy_target_actions,
          )
          target_q_values_2, _ = self._target_critic_network_2(
              target_q_input_2,
              next_time_steps.step_type,
              training=False,
          )
          target_q_values = tf.minimum(
              target_q_values_1, target_q_values_2
          )
  
          td_targets = tf.stop_gradient(
              self._reward_scale_factor * next_time_steps.reward
              + self._gamma * next_time_steps.discount * target_q_values
          )
  
          pred_input_1 = (time_steps.observation, actions)
          pred_td_targets_1, _ = self._critic_network_1(
              pred_input_1, time_steps.step_type, training=training
          )
          pred_input_2 = (time_steps.observation, actions)
          pred_td_targets_2, _ = self._critic_network_2(
              pred_input_2, time_steps.step_type, training=training
          )
          pred_td_targets_all = [pred_td_targets_1, pred_td_targets_2]
  
          if self._debug_summaries:
              tf.compat.v2.summary.histogram(
                  name="td_targets",
                  data=td_targets,
                  step=self.train_step_counter,
              )
              with tf.name_scope("td_targets"):
                  tf.compat.v2.summary.scalar(
                      name="mean",
                      data=tf.reduce_mean(input_tensor=td_targets),
                      step=self.train_step_counter,
                  )
                  tf.compat.v2.summary.scalar(
                      name="max",
                      data=tf.reduce_max(input_tensor=td_targets),
                      step=self.train_step_counter,
                  )
                  tf.compat.v2.summary.scalar(
                      name="min",
                      data=tf.reduce_min(input_tensor=td_targets),
                      step=self.train_step_counter,
                  )
  
              for td_target_idx in range(2):
                  pred_td_targets = pred_td_targets_all[td_target_idx]
                  td_errors = td_targets - pred_td_targets
                  with tf.name_scope(
                      "critic_net_%d" % (td_target_idx + 1)
                  ):
                      tf.compat.v2.summary.histogram(
                          name="td_errors",
                          data=td_errors,
                          step=self.train_step_counter,
                      )
                      tf.compat.v2.summary.histogram(
                          name="pred_td_targets",
                          data=pred_td_targets,
                          step=self.train_step_counter,
                      )
                      with tf.name_scope("td_errors"):
                          tf.compat.v2.summary.scalar(
                              name="mean",
                              data=tf.reduce_mean(
                                  input_tensor=td_errors
                              ),
                              step=self.train_step_counter,
                          )
                          tf.compat.v2.summary.scalar(
                              name="mean_abs",
                              data=tf.reduce_mean(
                                  input_tensor=tf.abs(td_errors)
                              ),
                              step=self.train_step_counter,
                          )
                          tf.compat.v2.summary.scalar(
                              name="max",
                              data=tf.reduce_max(
                                  input_tensor=td_errors
                              ),
                              step=self.train_step_counter,
                          )
                          tf.compat.v2.summary.scalar(
                              name="min",
                              data=tf.reduce_min(
                                  input_tensor=td_errors
                              ),
                              step=self.train_step_counter,
                          )
                      with tf.name_scope("pred_td_targets"):
                          tf.compat.v2.summary.scalar(
                              name="mean",
                              data=tf.reduce_mean(
                                  input_tensor=pred_td_targets
                              ),
                              step=self.train_step_counter,
                          )
                          tf.compat.v2.summary.scalar(
                              name="max",
                              data=tf.reduce_max(
                                  input_tensor=pred_td_targets
                              ),
                              step=self.train_step_counter,
                          )
                          tf.compat.v2.summary.scalar(
                              name="min",
                              data=tf.reduce_min(
                                  input_tensor=pred_td_targets
                              ),
                              step=self.train_step_counter,
                          )
  
          critic_loss = self._td_errors_loss_fn(
              td_targets, pred_td_targets_1
          ) + self._td_errors_loss_fn(td_targets, pred_td_targets_2)
          if nest_utils.is_batched_nested_tensors(
              time_steps, self.time_step_spec, num_outer_dims=2
          ):
              # Sum over the time dimension.
              critic_loss = tf.reduce_sum(
                  input_tensor=critic_loss,
                  axis=range(1, critic_loss.shape.rank),
              )
  
          if weights is not None:
              critic_loss *= weights
  
          return tf.reduce_mean(input_tensor=critic_loss)
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  from typing import Optional, Text
  import gin
  import tensorflow
  import tensorflow_probability
  from tf_agents.agents import data_converter
  from tf_agents.agents import tf_agent
  from tf_agents.networks import network
  from tf_agents.policies import actor_policy
  from tf_agents.policies import gaussian_policy
  from tf_agents.trajectories import time_step
  from tf_agents.typing import types
  from tf_agents.utils import common
  from tf_agents.utils import eager_utils
  from tf_agents.utils import nest_utils
  from tf_agents.utils import object_identity
label_desc: Use image node_id 6 for calling the Td3Agent obj's underlying member method code with example usage: obj.critic_loss(time_steps, actions, next_time_steps, weights, training) and returns: tf, unknown