class_docstr: Implements a recurrent DQN designed to produce melody sequences.
class_docstr_tok: ["Implements","a","recurrent","DQN","designed","to","produce","melody","sequences","."]
docstr: Main training function that allows model to act, collects reward, trains.
  
  Iterates a number of times, getting the model to act each time, saving the
  experience, and performing backprop.
  
  Args:
    num_steps: The number of training steps to execute.
    exploration_period: The number of steps over which the probability of
      exploring (taking a random action) is annealed from 1.0 to the model's
      random_action_probability.
    enable_random: If False, the model will not be able to act randomly /
      explore.
docstr_tok: ["Main","training","function","that","allows","model","to","act",",","collects","reward",",","trains",".","Iterates","a","number","of","times",",","getting","the","model","to","act","each","time",",","saving","the","experience",",","and","performing","backprop",".","Args",":","num_steps",":","The","number","of","training","steps","to","execute",".","exploration_period",":","The","number","of","steps","over","which","the","probability","of","exploring","(","taking","a","random","action",")","is","annealed","from","1.0","to","the","model's","random_action_probability",".","enable_random",":","If","False",",","the","model","will","not","be","able","to","act","randomly","\/","explore","."]
code: def train(
      self, num_steps=10000, exploration_period=5000, enable_random=True
  ):
      """Main training function that allows model to act, collects reward, trains.
  
      Iterates a number of times, getting the model to act each time, saving the
      experience, and performing backprop.
  
      Args:
        num_steps: The number of training steps to execute.
        exploration_period: The number of steps over which the probability of
          exploring (taking a random action) is annealed from 1.0 to the model's
          random_action_probability.
        enable_random: If False, the model will not be able to act randomly /
          explore.
      """
      tf.logging.info("Evaluating initial model...")
      self.evaluate_model()
  
      self.actions_executed_so_far = 0
  
      if self.stochastic_observations:
          tf.logging.info("Using stochastic environment")
  
      sample_next_obs = False
      if (
          self.exploration_mode == "boltzmann"
          or self.stochastic_observations
      ):
          sample_next_obs = True
  
      self.reset_composition()
      last_observation = self.prime_internal_models()
  
      for i in range(num_steps):
          # Experiencing observation, state, action, reward, new observation,
          # new state tuples, and storing them.
          state = np.array(self.q_network.state_value).flatten()
  
          action, new_observation, reward_scores = self.action(
              last_observation,
              exploration_period,
              enable_random=enable_random,
              sample_next_obs=sample_next_obs,
          )
  
          new_state = np.array(self.q_network.state_value).flatten()
          new_reward_state = np.array(
              self.reward_rnn.state_value
          ).flatten()
  
          reward = self.collect_reward(
              last_observation, new_observation, reward_scores
          )
  
          self.store(
              last_observation,
              state,
              action,
              reward,
              new_observation,
              new_state,
              new_reward_state,
          )
  
          # Used to keep track of how the reward is changing over time.
          self.reward_last_n += reward
  
          # Used to keep track of the current musical composition and beat for
          # the reward functions.
          self.composition.append(np.argmax(new_observation))
          self.beat += 1
  
          if i > 0 and i % self.output_every_nth == 0:
              tf.logging.info("Evaluating model...")
              self.evaluate_model()
              self.save_model(self.algorithm)
  
              if self.algorithm == "g":
                  self.rewards_batched.append(
                      self.music_theory_reward_last_n
                      + self.note_rnn_reward_last_n
                  )
              else:
                  self.rewards_batched.append(self.reward_last_n)
              self.music_theory_rewards_batched.append(
                  self.music_theory_reward_last_n
              )
              self.note_rnn_rewards_batched.append(
                  self.note_rnn_reward_last_n
              )
  
              # Save a checkpoint.
              save_step = (
                  len(self.rewards_batched) * self.output_every_nth
              )
              self.saver.save(
                  self.session, self.save_path, global_step=save_step
              )
  
              r = self.reward_last_n
              tf.logging.info("Training iteration %s", i)
              tf.logging.info(
                  "\tReward for last %s steps: %s",
                  self.output_every_nth,
                  r,
              )
              tf.logging.info(
                  "\t\tMusic theory reward: %s",
                  self.music_theory_reward_last_n,
              )
              tf.logging.info(
                  "\t\tNote RNN reward: %s", self.note_rnn_reward_last_n
              )
  
              # TODO(natashamjaques): Remove print statement once tf.logging outputs
              # to Jupyter notebooks (once the following issue is resolved:
              # https://github.com/tensorflow/tensorflow/issues/3047)
              print("Training iteration", i)
              print(
                  "\tReward for last",
                  self.output_every_nth,
                  "steps:",
                  r,
              )
              print(
                  "\t\tMusic theory reward:",
                  self.music_theory_reward_last_n,
              )
              print("\t\tNote RNN reward:", self.note_rnn_reward_last_n)
  
              if self.exploration_mode == "egreedy":
                  exploration_p = rl_tuner_ops.linear_annealing(
                      self.actions_executed_so_far,
                      exploration_period,
                      1.0,
                      self.dqn_hparams.random_action_probability,
                  )
                  tf.logging.info(
                      "\tExploration probability is %s", exploration_p
                  )
  
              self.reward_last_n = 0
              self.music_theory_reward_last_n = 0
              self.note_rnn_reward_last_n = 0
  
          # Backprop.
          self.training_step()
  
          # Update current state as last state.
          last_observation = new_observation
  
          # Reset the state after each composition is complete.
          if self.beat % self.num_notes_in_melody == 0:
              tf.logging.debug("\nResetting composition!\n")
              self.reset_composition()
              last_observation = self.prime_internal_models()
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  import os
  import random
  import urllib
  from magenta.models.rl_tuner import note_rnn_loader
  from magenta.models.rl_tuner import rl_tuner_eval_metrics
  from magenta.models.rl_tuner import rl_tuner_ops
  import matplotlib.pyplot
  from note_seq import melodies_lib
  from note_seq import midi_io
  import numpy
  import scipy.special
  from six.moves import range
  from six.moves import reload_module
  from six.moves import urllib
  import tensorflow.compat.v1
label_desc: Use image node_id 8 for calling the RLTuner obj's underlying member method code with example usage: obj.train(num_steps, exploration_period, enable_random) without return types