docstr: update all networks in TD3 
docstr_tok: ["update","all","networks","in","TD3"]
code: def update(
      self,
      batch_size,
      eval_noise_scale,
      reward_scale=10.0,
      gamma=0.9,
      soft_tau=1e-2,
  ):
      """update all networks in TD3"""
      self.update_cnt += 1
      (
          state,
          action,
          reward,
          next_state,
          done,
      ) = self.replay_buffer.sample(batch_size)
  
      reward = reward[:, np.newaxis]  # expand dim
      done = done[:, np.newaxis]
  
      new_next_action = self.target_policy_net.evaluate(
          next_state, eval_noise_scale=eval_noise_scale
      )  # clipped normal noise
      reward = (
          reward_scale
          * (reward - np.mean(reward, axis=0))
          / (np.std(reward, axis=0) + 1e-6)
      )  # normalize with batch mean and std; plus a small number to prevent numerical problem
  
      # Training Q Function
      target_q_input = tf.concat(
          [next_state, new_next_action], 1
      )  # the dim 0 is number of samples
      target_q_min = tf.minimum(
          self.target_q_net1(target_q_input),
          self.target_q_net2(target_q_input),
      )
  
      target_q_value = (
          reward + (1 - done) * gamma * target_q_min
      )  # if done==1, only reward
      q_input = tf.concat([state, action], 1)  # input of q_net
  
      with tf.GradientTape() as q1_tape:
          predicted_q_value1 = self.q_net1(q_input)
          q_value_loss1 = tf.reduce_mean(
              tf.square(predicted_q_value1 - target_q_value)
          )
      q1_grad = q1_tape.gradient(
          q_value_loss1, self.q_net1.trainable_weights
      )
      self.q_optimizer1.apply_gradients(
          zip(q1_grad, self.q_net1.trainable_weights)
      )
  
      with tf.GradientTape() as q2_tape:
          predicted_q_value2 = self.q_net2(q_input)
          q_value_loss2 = tf.reduce_mean(
              tf.square(predicted_q_value2 - target_q_value)
          )
      q2_grad = q2_tape.gradient(
          q_value_loss2, self.q_net2.trainable_weights
      )
      self.q_optimizer2.apply_gradients(
          zip(q2_grad, self.q_net2.trainable_weights)
      )
  
      # Training Policy Function
      if self.update_cnt % self.policy_target_update_interval == 0:
          with tf.GradientTape() as p_tape:
              new_action = self.policy_net.evaluate(
                  state, eval_noise_scale=0.0
              )  # no noise, deterministic policy gradients
              new_q_input = tf.concat([state, new_action], 1)
              # """ implementation 1 """
              # predicted_new_q_value = tf.minimum(self.q_net1(new_q_input),self.q_net2(new_q_input))
              """ implementation 2 """
              predicted_new_q_value = self.q_net1(new_q_input)
              policy_loss = -tf.reduce_mean(predicted_new_q_value)
          p_grad = p_tape.gradient(
              policy_loss, self.policy_net.trainable_weights
          )
          self.policy_optimizer.apply_gradients(
              zip(p_grad, self.policy_net.trainable_weights)
          )
  
          # Soft update the target nets
          self.target_q_net1 = self.target_soft_update(
              self.q_net1, self.target_q_net1, soft_tau
          )
          self.target_q_net2 = self.target_soft_update(
              self.q_net2, self.target_q_net2, soft_tau
          )
          self.target_policy_net = self.target_soft_update(
              self.policy_net, self.target_policy_net, soft_tau
          )
  
imports: import argparse
  import os
  import random
  import time
  import gym
  import matplotlib.pyplot
  import numpy
  import tensorflow
  import tensorflow_probability
  import tensorlayer
  from tensorlayer.layers import Dense
  from tensorlayer.models import Model
label_desc: Use image node_id 4 for calling the TD3 obj's underlying member method code with example usage: obj.update(batch_size, eval_noise_scale, reward_scale, gamma, soft_tau) without return types