docstr: Tests DDPG's Exploration (w/ random actions for n timesteps).
docstr_tok: ["Tests","DDPG","'s","Exploration","(","w\/","random","actions","for","n","timesteps",")","."]
code: def test_ddpg_exploration_and_with_random_prerun(self):
      """Tests DDPG's Exploration (w/ random actions for n timesteps)."""
  
      core_config = (
          DDPGConfig()
          .environment("Pendulum-v1")
          .rollouts(num_rollout_workers=0)
          .training(num_steps_sampled_before_learning_starts=0)
      )
  
      obs = np.array([0.0, 0.1, -0.1])
  
      # Test against all frameworks.
      for _ in framework_iterator(core_config):
          config = copy.deepcopy(core_config)
          # Default OUNoise setup.
          algo = config.build()
          # Setting explore=False should always return the same action.
          a_ = algo.compute_single_action(obs, explore=False)
          check(convert_to_numpy(algo.get_policy().global_timestep), 1)
          for i in range(50):
              a = algo.compute_single_action(obs, explore=False)
              check(
                  convert_to_numpy(algo.get_policy().global_timestep),
                  i + 2,
              )
              check(a, a_)
          # explore=None (default: explore) should return different actions.
          actions = []
          for i in range(50):
              actions.append(algo.compute_single_action(obs))
              check(
                  convert_to_numpy(algo.get_policy().global_timestep),
                  i + 52,
              )
          check(np.std(actions), 0.0, false=True)
          algo.stop()
  
          # Check randomness at beginning.
          config.exploration(
              exploration_config={
                  # Act randomly at beginning ...
                  "random_timesteps": 50,
                  # Then act very closely to deterministic actions thereafter.
                  "ou_base_scale": 0.001,
                  "initial_scale": 0.001,
                  "final_scale": 0.001,
              }
          )
          algo = config.build()
          # ts=0 (get a deterministic action as per explore=False).
          deterministic_action = algo.compute_single_action(
              obs, explore=False
          )
          check(convert_to_numpy(algo.get_policy().global_timestep), 1)
          # ts=1-49 (in random window).
          random_a = []
          for i in range(1, 50):
              random_a.append(
                  algo.compute_single_action(obs, explore=True)
              )
              check(
                  convert_to_numpy(algo.get_policy().global_timestep),
                  i + 1,
              )
              check(random_a[-1], deterministic_action, false=True)
          self.assertTrue(np.std(random_a) > 0.5)
  
          # ts > 50 (a=deterministic_action + scale * N[0,1])
          for i in range(50):
              a = algo.compute_single_action(obs, explore=True)
              check(
                  convert_to_numpy(algo.get_policy().global_timestep),
                  i + 51,
              )
              check(a, deterministic_action, rtol=0.1)
  
          # ts >> 50 (BUT: explore=False -> expect deterministic action).
          for i in range(50):
              a = algo.compute_single_action(obs, explore=False)
              check(
                  convert_to_numpy(algo.get_policy().global_timestep),
                  i + 101,
              )
              check(a, deterministic_action)
          algo.stop()
  
imports: import copy
  import re
  import unittest
  import gymnasium
  import numpy
  from gymnasium.spaces import Box
  from rllib_ddpg.ddpg import DDPGConfig
  import ray
  from ray.rllib.policy.sample_batch import SampleBatch
  from ray.rllib.utils.framework import try_import_tf, try_import_torch
  from ray.rllib.utils.numpy import convert_to_numpy, fc, huber_loss, l2_loss, relu, sigmoid
  from ray.rllib.utils.replay_buffers.utils import patch_buffer_with_fake_sampling_method
  from ray.rllib.utils.spaces.simplex import Simplex
  from ray.rllib.utils.test_utils import check, check_compute_single_action, check_train_results, framework_iterator
  from ray.rllib.utils.torch_utils import convert_to_torch_tensor
label_desc: Use image node_id 4 for calling the TestDDPG obj's underlying member method code with example usage: obj.test_ddpg_exploration_and_with_random_prerun() without return types