docstr: update all networks in SAC 
docstr_tok: ["update","all","networks","in","SAC"]
code: def update(
      self,
      batch_size,
      reward_scale=10.0,
      auto_entropy=True,
      target_entropy=-2,
      gamma=0.99,
      soft_tau=1e-2,
  ):
      """update all networks in SAC"""
      (
          state,
          action,
          reward,
          next_state,
          done,
      ) = self.replay_buffer.sample(batch_size)
  
      reward = reward[:, np.newaxis]  # expand dim
      done = done[:, np.newaxis]
  
      reward = (
          reward_scale
          * (reward - np.mean(reward, axis=0))
          / (np.std(reward, axis=0) + 1e-6)
      )  # normalize with batch mean and std; plus a small number to prevent numerical problem
  
      # Training Q Function
      (
          new_next_action,
          next_log_prob,
          _,
          _,
          _,
      ) = self.policy_net.evaluate(next_state)
      target_q_input = tf.concat(
          [next_state, new_next_action], 1
      )  # the dim 0 is number of samples
      target_q_min = (
          tf.minimum(
              self.target_soft_q_net1(target_q_input),
              self.target_soft_q_net2(target_q_input),
          )
          - self.alpha * next_log_prob
      )
      target_q_value = (
          reward + (1 - done) * gamma * target_q_min
      )  # if done==1, only reward
      q_input = tf.concat(
          [state, action], 1
      )  # the dim 0 is number of samples
  
      with tf.GradientTape() as q1_tape:
          predicted_q_value1 = self.soft_q_net1(q_input)
          q_value_loss1 = tf.reduce_mean(
              tf.losses.mean_squared_error(
                  predicted_q_value1, target_q_value
              )
          )
      q1_grad = q1_tape.gradient(
          q_value_loss1, self.soft_q_net1.trainable_weights
      )
      self.soft_q_optimizer1.apply_gradients(
          zip(q1_grad, self.soft_q_net1.trainable_weights)
      )
  
      with tf.GradientTape() as q2_tape:
          predicted_q_value2 = self.soft_q_net2(q_input)
          q_value_loss2 = tf.reduce_mean(
              tf.losses.mean_squared_error(
                  predicted_q_value2, target_q_value
              )
          )
      q2_grad = q2_tape.gradient(
          q_value_loss2, self.soft_q_net2.trainable_weights
      )
      self.soft_q_optimizer2.apply_gradients(
          zip(q2_grad, self.soft_q_net2.trainable_weights)
      )
  
      # Training Policy Function
      with tf.GradientTape() as p_tape:
          (
              new_action,
              log_prob,
              z,
              mean,
              log_std,
          ) = self.policy_net.evaluate(state)
          new_q_input = tf.concat(
              [state, new_action], 1
          )  # the dim 0 is number of samples
          """ implementation 1 """
          predicted_new_q_value = tf.minimum(
              self.soft_q_net1(new_q_input),
              self.soft_q_net2(new_q_input),
          )
          # """ implementation 2 """
          # predicted_new_q_value = self.soft_q_net1(new_q_input)
          policy_loss = tf.reduce_mean(
              self.alpha * log_prob - predicted_new_q_value
          )
      p_grad = p_tape.gradient(
          policy_loss, self.policy_net.trainable_weights
      )
      self.policy_optimizer.apply_gradients(
          zip(p_grad, self.policy_net.trainable_weights)
      )
  
      # Updating alpha w.r.t entropy
      # alpha: trade-off between exploration (max entropy) and exploitation (max Q)
      if auto_entropy is True:
          with tf.GradientTape() as alpha_tape:
              alpha_loss = -tf.reduce_mean(
                  (self.log_alpha * (log_prob + target_entropy))
              )
          alpha_grad = alpha_tape.gradient(alpha_loss, [self.log_alpha])
          self.alpha_optimizer.apply_gradients(
              zip(alpha_grad, [self.log_alpha])
          )
          self.alpha = tf.math.exp(self.log_alpha)
      else:  # fixed alpha
          self.alpha = 1.0
          alpha_loss = 0
  
      # Soft update the target value nets
      self.target_soft_q_net1 = self.target_soft_update(
          self.soft_q_net1, self.target_soft_q_net1, soft_tau
      )
      self.target_soft_q_net2 = self.target_soft_update(
          self.soft_q_net2, self.target_soft_q_net2, soft_tau
      )
  
imports: import argparse
  import os
  import random
  import time
  import gym
  import matplotlib.pyplot
  import numpy
  import tensorflow
  import tensorflow_probability
  import tensorlayer
  from tensorlayer.layers import Dense
  from tensorlayer.models import Model
label_desc: Use image node_id 4 for calling the SAC obj's underlying member method code with example usage: obj.update(batch_size, reward_scale, auto_entropy, target_entropy, gamma, soft_tau) without return types