docstr: Emulates DDPG loss functions for tf and torch.
docstr_tok: ["Emulates","DDPG","loss","functions","for","tf","and","torch","."]
code: def _ddpg_loss_helper(
      self,
      train_batch,
      weights,
      ks,
      fw,
      gamma,
      huber_threshold,
      l2_reg,
      sess,
  ):
      """Emulates DDPG loss functions for tf and torch."""
      model_out_t = train_batch[SampleBatch.CUR_OBS]
      target_model_out_tp1 = train_batch[SampleBatch.NEXT_OBS]
      # get_policy_output
      policy_t = sigmoid(
          2.0
          * fc(
              relu(
                  fc(
                      model_out_t,
                      weights[ks[1]],
                      weights[ks[0]],
                      framework=fw,
                  )
              ),
              weights[ks[5]],
              weights[ks[4]],
              framework=fw,
          )
      )
      # Get policy output for t+1 (target model).
      policy_tp1 = sigmoid(
          2.0
          * fc(
              relu(
                  fc(
                      target_model_out_tp1,
                      weights[ks[3]],
                      weights[ks[2]],
                      framework=fw,
                  )
              ),
              weights[ks[7]],
              weights[ks[6]],
              framework=fw,
          )
      )
      # Assume no smooth target policy.
      policy_tp1_smoothed = policy_tp1
  
      # Q-values for the actually selected actions.
      # get_q_values
      q_t = fc(
          relu(
              fc(
                  np.concatenate(
                      [model_out_t, train_batch[SampleBatch.ACTIONS]],
                      -1,
                  ),
                  weights[ks[9]],
                  weights[ks[8]],
                  framework=fw,
              )
          ),
          weights[ks[11]],
          weights[ks[10]],
          framework=fw,
      )
      twin_q_t = fc(
          relu(
              fc(
                  np.concatenate(
                      [model_out_t, train_batch[SampleBatch.ACTIONS]],
                      -1,
                  ),
                  weights[ks[13]],
                  weights[ks[12]],
                  framework=fw,
              )
          ),
          weights[ks[15]],
          weights[ks[14]],
          framework=fw,
      )
  
      # Q-values for current policy in given current state.
      # get_q_values
      q_t_det_policy = fc(
          relu(
              fc(
                  np.concatenate([model_out_t, policy_t], -1),
                  weights[ks[9]],
                  weights[ks[8]],
                  framework=fw,
              )
          ),
          weights[ks[11]],
          weights[ks[10]],
          framework=fw,
      )
  
      # Target q network evaluation.
      # target_model.get_q_values
      q_tp1 = fc(
          relu(
              fc(
                  np.concatenate(
                      [target_model_out_tp1, policy_tp1_smoothed], -1
                  ),
                  weights[ks[17]],
                  weights[ks[16]],
                  framework=fw,
              )
          ),
          weights[ks[19]],
          weights[ks[18]],
          framework=fw,
      )
      twin_q_tp1 = fc(
          relu(
              fc(
                  np.concatenate(
                      [target_model_out_tp1, policy_tp1_smoothed], -1
                  ),
                  weights[ks[21]],
                  weights[ks[20]],
                  framework=fw,
              )
          ),
          weights[ks[23]],
          weights[ks[22]],
          framework=fw,
      )
  
      q_t_selected = np.squeeze(q_t, axis=-1)
      twin_q_t_selected = np.squeeze(twin_q_t, axis=-1)
      q_tp1 = np.minimum(q_tp1, twin_q_tp1)
      q_tp1_best = np.squeeze(q_tp1, axis=-1)
  
      dones = train_batch[SampleBatch.TERMINATEDS]
      rewards = train_batch[SampleBatch.REWARDS]
      if fw == "torch":
          dones = dones.float().numpy()
          rewards = rewards.numpy()
  
      q_tp1_best_masked = (1.0 - dones) * q_tp1_best
      q_t_selected_target = rewards + gamma * q_tp1_best_masked
  
      td_error = q_t_selected - q_t_selected_target
      twin_td_error = twin_q_t_selected - q_t_selected_target
      errors = huber_loss(td_error, huber_threshold) + huber_loss(
          twin_td_error, huber_threshold
      )
  
      critic_loss = np.mean(errors)
      actor_loss = -np.mean(q_t_det_policy)
      # Add l2-regularization if required.
      for name, var in weights.items():
          if re.match(
              "default_policy/actor_(hidden_0|out)/kernel", name
          ):
              actor_loss += l2_reg * l2_loss(var)
          elif re.match(
              "default_policy/sequential(_1)?/\\w+/kernel", name
          ):
              critic_loss += l2_reg * l2_loss(var)
  
      return critic_loss, actor_loss, td_error
  
imports: import copy
  import re
  import unittest
  import gymnasium
  import numpy
  from gymnasium.spaces import Box
  from rllib_ddpg.ddpg import DDPGConfig
  import ray
  from ray.rllib.policy.sample_batch import SampleBatch
  from ray.rllib.utils.framework import try_import_tf, try_import_torch
  from ray.rllib.utils.numpy import convert_to_numpy, fc, huber_loss, l2_loss, relu, sigmoid
  from ray.rllib.utils.replay_buffers.utils import patch_buffer_with_fake_sampling_method
  from ray.rllib.utils.spaces.simplex import Simplex
  from ray.rllib.utils.test_utils import check, check_compute_single_action, check_train_results, framework_iterator
  from ray.rllib.utils.torch_utils import convert_to_torch_tensor
label_desc: Use image node_id 7 for calling the TestDDPG obj's underlying member method code with example usage: obj._ddpg_loss_helper(train_batch, weights, ks, fw, gamma, huber_threshold, l2_reg, sess) and returns: critic_loss,  actor_loss,  td_error