class_docstr: We use this object to :
      __init__:
          - Creates the step_model
          - Creates the train_model
  
      train():
          - Make the training part (feedforward and retropropagation of gradients)
  
      save/load():
          - Save load the model
class_docstr_tok: ["We","use","this","object","to",":","__init__",":","-","Creates","the","step_model","-","Creates","the","train_model","train","(",")",":","-","Make","the","training","part","(","feedforward","and","retropropagation","of","gradients",")","save\/load","(",")",":","-","Save","load","the","model"]
code: def __init__(
      self,
      *,
      policy,
      nbatch_act,
      nbatch_train,
      nsteps,
      ent_coef,
      vf_coef,
      max_grad_norm,
      microbatch_size=None,
      np_mask=None,
  ):
      self.sess = sess = get_session()
  
      with tf.variable_scope("ppo2_model", reuse=tf.AUTO_REUSE):
          # CREATE OUR TWO MODELS
          # act_model that is used for sampling
          act_model = policy(
              nbatch_act, 1, sess, np_mask=np_mask, is_act_model=True
          )
  
          # Train model for training
          if microbatch_size is None:
              train_model = policy(
                  nbatch_train,
                  nsteps,
                  sess,
                  np_mask=np_mask,
                  is_act_model=False,
              )
          else:
              train_model = policy(
                  microbatch_size,
                  nsteps,
                  sess,
                  np_mask=np_mask,
                  is_act_model=False,
              )
  
      # CREATE THE PLACEHOLDERS
      self.A = A = train_model.pdtype.sample_placeholder([None])
      self.ADV = ADV = tf.placeholder(tf.float32, [None])
      self.R = R = tf.placeholder(tf.float32, [None])
      # Keep track of old actor
      self.OLDNEGLOGPAC = OLDNEGLOGPAC = tf.placeholder(
          tf.float32, [None]
      )
      # Keep track of old critic
      self.OLDVPRED = OLDVPRED = tf.placeholder(tf.float32, [None])
      self.LR = LR = tf.placeholder(tf.float32, [])
      # Cliprange
      self.CLIPRANGE = CLIPRANGE = tf.placeholder(tf.float32, [])
  
      neglogpac = train_model.pd.neglogp(A)
  
      # Calculate the entropy
      # Entropy is used to improve exploration by limiting the premature convergence to suboptimal policy.
      entropy = tf.reduce_mean(train_model.pd.entropy())
  
      # CALCULATE THE LOSS
      # Total loss = Policy gradient loss - entropy * entropy coefficient + Value coefficient * value loss
  
      # Clip the value to reduce variability during Critic training
      # Get the predicted value
      vpred = train_model.vf
      vpredclipped = OLDVPRED + tf.clip_by_value(
          train_model.vf - OLDVPRED, -CLIPRANGE, CLIPRANGE
      )
      # Unclipped value
      vf_losses1 = tf.square(vpred - R)
      # Clipped value
      vf_losses2 = tf.square(vpredclipped - R)
  
      vf_loss = 0.5 * tf.reduce_mean(tf.maximum(vf_losses1, vf_losses2))
  
      # Calculate ratio (pi current policy / pi old policy)
      ratio = tf.exp(OLDNEGLOGPAC - neglogpac)
  
      # Defining Loss = - J is equivalent to max J
      pg_losses = -ADV * ratio
  
      pg_losses2 = -ADV * tf.clip_by_value(
          ratio, 1.0 - CLIPRANGE, 1.0 + CLIPRANGE
      )
  
      # Final PG loss
      pg_loss = tf.reduce_mean(tf.maximum(pg_losses, pg_losses2))
      approxkl = 0.5 * tf.reduce_mean(
          tf.square(neglogpac - OLDNEGLOGPAC)
      )
      clipfrac = tf.reduce_mean(
          tf.to_float(tf.greater(tf.abs(ratio - 1.0), CLIPRANGE))
      )
  
      # Total loss
      loss = pg_loss - entropy * ent_coef + vf_loss * vf_coef
  
      # UPDATE THE PARAMETERS USING LOSS
      # 1. Get the model parameters
      params = tf.trainable_variables("ppo2_model")
      # 2. Build our trainer
      self.trainer = tf.train.AdamOptimizer(
          learning_rate=LR, epsilon=1e-5
      )
      # 3. Calculate the gradients
      grads_and_var = self.trainer.compute_gradients(loss, params)
      grads, var = zip(*grads_and_var)
  
      if max_grad_norm is not None:
          # Clip the gradients (normalize)
          grads, _grad_norm = tf.clip_by_global_norm(
              grads, max_grad_norm
          )
      grads_and_var = list(zip(grads, var))
      # zip aggregate each gradient with parameters associated
      # For instance zip(ABCD, xyza) => Ax, By, Cz, Da
  
      self.grads = grads
      self.var = var
      self._train_op = self.trainer.apply_gradients(grads_and_var)
      self.loss_names = [
          "policy_loss",
          "value_loss",
          "policy_entropy",
          "approxkl",
          "clipfrac",
      ]
      self.stats_list = [pg_loss, vf_loss, entropy, approxkl, clipfrac]
  
      self.train_model = train_model
      self.act_model = act_model
      self.step = act_model.step
      self.value = act_model.value
      self.initial_state = act_model.initial_state
  
      initialize()
  
imports: import tensorflow.compat.v1
  from .util import initialize, get_session
label_desc: Use image node_id 1 to create a new Model object  with example: obj = Model()