class_docstr: Implements a recurrent DQN designed to produce melody sequences.
class_docstr_tok: ["Implements","a","recurrent","DQN","designed","to","produce","melody","sequences","."]
docstr: Backpropagate prediction error from a randomly sampled experience batch.
  
  A minibatch of experiences is randomly sampled from the model's experience
  replay buffer and used to update the weights of the q_network and
  target_q_network.
docstr_tok: ["Backpropagate","prediction","error","from","a","randomly","sampled","experience","batch",".","A","minibatch","of","experiences","is","randomly","sampled","from","the","model","'s","experience","replay","buffer","and","used","to","update","the","weights","of","the","q_network","and","target_q_network","."]
code: def training_step(self):
      """Backpropagate prediction error from a randomly sampled experience batch.
  
      A minibatch of experiences is randomly sampled from the model's experience
      replay buffer and used to update the weights of the q_network and
      target_q_network.
      """
      if (
          self.num_times_train_called % self.dqn_hparams.train_every_nth
          == 0
      ):
          if len(self.experience) < self.dqn_hparams.minibatch_size:
              return
  
          # Sample experience.
          samples = random.sample(
              range(len(self.experience)),
              self.dqn_hparams.minibatch_size,
          )
          samples = [self.experience[i] for i in samples]
  
          # Batch states.
          states = np.empty(
              (len(samples), self.q_network.cell.state_size)
          )
          new_states = np.empty(
              (len(samples), self.target_q_network.cell.state_size)
          )
          reward_new_states = np.empty(
              (len(samples), self.reward_rnn.cell.state_size)
          )
          observations = np.empty((len(samples), self.input_size))
          new_observations = np.empty((len(samples), self.input_size))
          action_mask = np.zeros((len(samples), self.num_actions))
          rewards = np.empty((len(samples),))
          lengths = np.full(len(samples), 1, dtype=int)
  
          for i, (o, s, a, r, new_o, new_s, reward_s) in enumerate(
              samples
          ):
              observations[i, :] = o
              new_observations[i, :] = new_o
              states[i, :] = s
              new_states[i, :] = new_s
              action_mask[i, :] = a
              rewards[i] = r
              reward_new_states[i, :] = reward_s
  
          observations = np.reshape(
              observations, (len(samples), 1, self.input_size)
          )
          new_observations = np.reshape(
              new_observations, (len(samples), 1, self.input_size)
          )
  
          calc_summaries = self.iteration % 100 == 0
          calc_summaries = (
              calc_summaries and self.summary_writer is not None
          )
  
          if self.algorithm == "g":
              _, _, target_vals, summary_str = self.session.run(
                  [
                      self.prediction_error,
                      self.train_op,
                      self.target_vals,
                      self.summarize if calc_summaries else self.no_op1,
                  ],
                  {
                      self.reward_rnn.melody_sequence: new_observations,
                      self.reward_rnn.initial_state: reward_new_states,
                      self.reward_rnn.lengths: lengths,
                      self.q_network.melody_sequence: observations,
                      self.q_network.initial_state: states,
                      self.q_network.lengths: lengths,
                      self.target_q_network.melody_sequence: new_observations,
                      self.target_q_network.initial_state: new_states,
                      self.target_q_network.lengths: lengths,
                      self.action_mask: action_mask,
                      self.rewards: rewards,
                  },
              )
          else:
              _, _, target_vals, summary_str = self.session.run(
                  [
                      self.prediction_error,
                      self.train_op,
                      self.target_vals,
                      self.summarize if calc_summaries else self.no_op1,
                  ],
                  {
                      self.q_network.melody_sequence: observations,
                      self.q_network.initial_state: states,
                      self.q_network.lengths: lengths,
                      self.target_q_network.melody_sequence: new_observations,
                      self.target_q_network.initial_state: new_states,
                      self.target_q_network.lengths: lengths,
                      self.action_mask: action_mask,
                      self.rewards: rewards,
                  },
              )
  
          total_logs = self.iteration * self.dqn_hparams.train_every_nth
          if total_logs % self.output_every_nth == 0:
              self.target_val_list.append(np.mean(target_vals))
  
          self.session.run(self.target_network_update)
  
          if calc_summaries:
              self.summary_writer.add_summary(
                  summary_str, self.iteration
              )
  
          self.iteration += 1
  
      self.num_times_train_called += 1
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  import os
  import random
  import urllib
  from magenta.models.rl_tuner import note_rnn_loader
  from magenta.models.rl_tuner import rl_tuner_eval_metrics
  from magenta.models.rl_tuner import rl_tuner_ops
  import matplotlib.pyplot
  from note_seq import melodies_lib
  from note_seq import midi_io
  import numpy
  import scipy.special
  from six.moves import range
  from six.moves import reload_module
  from six.moves import urllib
  import tensorflow.compat.v1
label_desc: Use image node_id 11 for calling the RLTuner obj's underlying member method code with example usage: obj.training_step() without return types