class_docstr: Implements a recurrent DQN designed to produce melody sequences.
class_docstr_tok: ["Implements","a","recurrent","DQN","designed","to","produce","melody","sequences","."]
docstr: Builds the reinforcement learning tensorflow graph.
docstr_tok: ["Builds","the","reinforcement","learning","tensorflow","graph","."]
code: def build_graph(self):
      """Builds the reinforcement learning tensorflow graph."""
  
      tf.logging.info("Adding reward computation portion of the graph")
      with tf.name_scope("reward_computation"):
          self.reward_scores = tf.identity(
              self.reward_rnn(), name="reward_scores"
          )
  
      tf.logging.info("Adding taking action portion of graph")
      with tf.name_scope("taking_action"):
          # Output of the q network gives the value of taking each action (playing
          # each note).
          self.action_scores = tf.identity(
              self.q_network(), name="action_scores"
          )
          tf.summary.histogram("action_scores", self.action_scores)
  
          # The action values for the G algorithm are computed differently.
          if self.algorithm == "g":
              self.g_action_scores = (
                  self.action_scores + self.reward_scores
              )
  
              # Compute predicted action, which is the argmax of the action scores.
              self.action_softmax = tf.nn.softmax(
                  self.g_action_scores, name="action_softmax"
              )
              self.predicted_actions = tf.one_hot(
                  tf.argmax(
                      self.g_action_scores,
                      dimension=1,
                      name="predicted_actions",
                  ),
                  self.num_actions,
              )
          else:
              # Compute predicted action, which is the argmax of the action scores.
              self.action_softmax = tf.nn.softmax(
                  self.action_scores, name="action_softmax"
              )
              self.predicted_actions = tf.one_hot(
                  tf.argmax(
                      self.action_scores,
                      dimension=1,
                      name="predicted_actions",
                  ),
                  self.num_actions,
              )
  
      tf.logging.info("Add estimating future rewards portion of graph")
      with tf.name_scope("estimating_future_rewards"):
          # The target q network is used to estimate the value of the best action at
          # the state resulting from the current action.
          self.next_action_scores = tf.stop_gradient(
              self.target_q_network()
          )
          tf.summary.histogram(
              "target_action_scores", self.next_action_scores
          )
  
          # Rewards are observed from the environment and are fed in later.
          self.rewards = tf.placeholder(
              tf.float32, (None,), name="rewards"
          )
  
          # Each algorithm is attempting to model future rewards with a different
          # function.
          if self.algorithm == "psi":
              self.target_vals = tf.reduce_logsumexp(
                  self.next_action_scores,
                  reduction_indices=[
                      1,
                  ],
              )
          elif self.algorithm == "g":
              self.g_normalizer = tf.reduce_logsumexp(
                  self.reward_scores,
                  reduction_indices=[
                      1,
                  ],
              )
              self.g_normalizer = tf.reshape(self.g_normalizer, [-1, 1])
              self.g_normalizer = tf.tile(
                  self.g_normalizer, [1, self.num_actions]
              )
              self.g_action_scores = tf.subtract(
                  (self.next_action_scores + self.reward_scores),
                  self.g_normalizer,
              )
              self.target_vals = tf.reduce_logsumexp(
                  self.g_action_scores,
                  reduction_indices=[
                      1,
                  ],
              )
          else:
              # Use default based on Q learning.
              self.target_vals = tf.reduce_max(
                  self.next_action_scores,
                  reduction_indices=[
                      1,
                  ],
              )
  
          # Total rewards are the observed rewards plus discounted estimated future
          # rewards.
          self.future_rewards = (
              self.rewards + self.discount_rate * self.target_vals
          )
  
      tf.logging.info("Adding q value prediction portion of graph")
      with tf.name_scope("q_value_prediction"):
          # Action mask will be a one-hot encoding of the action the network
          # actually took.
          self.action_mask = tf.placeholder(
              tf.float32, (None, self.num_actions), name="action_mask"
          )
          self.masked_action_scores = tf.reduce_sum(
              self.action_scores * self.action_mask,
              reduction_indices=[
                  1,
              ],
          )
  
          temp_diff = self.masked_action_scores - self.future_rewards
  
          # Prediction error is the mean squared error between the reward the
          # network actually received for a given action, and what it expected to
          # receive.
          self.prediction_error = tf.reduce_mean(tf.square(temp_diff))
  
          # Compute gradients.
          self.params = tf.trainable_variables()
          self.gradients = self.optimizer.compute_gradients(
              self.prediction_error
          )
  
          # Clip gradients.
          for i, (grad, var) in enumerate(self.gradients):
              if grad is not None:
                  self.gradients[i] = (tf.clip_by_norm(grad, 5), var)
  
          for grad, var in self.gradients:
              tf.summary.histogram(var.name, var)
              if grad is not None:
                  tf.summary.histogram(var.name + "/gradients", grad)
  
          # Backprop.
          self.train_op = self.optimizer.apply_gradients(self.gradients)
  
      tf.logging.info("Adding target network update portion of graph")
      with tf.name_scope("target_network_update"):
          # Updates the target_q_network to be similar to the q_network based on
          # the target_network_update_rate.
          self.target_network_update = []
          for v_source, v_target in zip(
              self.q_network.variables(),
              self.target_q_network.variables(),
          ):
              # Equivalent to target = (1-alpha) * target + alpha * source
              update_op = v_target.assign_sub(
                  self.target_network_update_rate
                  * (v_target - v_source)
              )
              self.target_network_update.append(update_op)
          self.target_network_update = tf.group(
              *self.target_network_update
          )
  
      tf.summary.scalar("prediction_error", self.prediction_error)
  
      self.summarize = tf.summary.merge_all()
      self.no_op1 = tf.no_op()
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  import os
  import random
  import urllib
  from magenta.models.rl_tuner import note_rnn_loader
  from magenta.models.rl_tuner import rl_tuner_eval_metrics
  from magenta.models.rl_tuner import rl_tuner_ops
  import matplotlib.pyplot
  from note_seq import melodies_lib
  from note_seq import midi_io
  import numpy
  import scipy.special
  from six.moves import range
  from six.moves import reload_module
  from six.moves import urllib
  import tensorflow.compat.v1
label_desc: Use image node_id 7 for calling the RLTuner obj's underlying member method code with example usage: obj.build_graph() without return types