code: def test_retry_with_max_failures(ray_start_4_cpus):
      """Tests trainer retry with max_failures > 0 when integrating with Tune."""
  
      def train_func():
          ckpt = train.get_checkpoint()
          restored = bool(ckpt)  # Does a previous checkpoint exist?
          itr = 0
          if ckpt:
              ckpt = load_dict_checkpoint(ckpt)
              itr = ckpt["iter"] + 1
  
          for i in range(itr, 4):
              if i == 2 and not restored:
                  raise Exception("try to fail me")
              with create_dict_checkpoint(dict(iter=i)) as checkpoint:
                  train.report(
                      dict(test=i, training_iteration=i),
                      checkpoint=checkpoint,
                  )
  
      trainer = DataParallelTrainer(
          train_func,
          backend_config=TestConfig(),
          scaling_config=ScalingConfig(num_workers=1),
      )
      tuner = Tuner(
          trainer,
          run_config=RunConfig(
              failure_config=FailureConfig(max_failures=3)
          ),
      )
  
      result_grid = tuner.fit()
      checkpoint = load_dict_checkpoint(result_grid[0].checkpoint)
      assert checkpoint["iter"] == 3
      df = result_grid[0].metrics_dataframe
      assert len(df[TRAINING_ITERATION]) == 4
  
imports: import logging
  import pytest
  import ray
  from ray import train, tune
  from ray.air.constants import TRAINING_ITERATION
  from ray.train import FailureConfig, RunConfig, ScalingConfig
  from ray.train._internal.worker_group import WorkerGroup
  from ray.train.backend import Backend, BackendConfig
  from ray.train.data_parallel_trainer import DataParallelTrainer
  from ray.train.examples.pytorch.torch_fashion_mnist_example import train_func_per_worker
  from ray.train.examples.tf.tensorflow_mnist_example import train_func
  from ray.train.tensorflow.tensorflow_trainer import TensorflowTrainer
  from ray.train.tests.util import create_dict_checkpoint, load_dict_checkpoint
  from ray.train.torch.torch_trainer import TorchTrainer
  from ray.tune.tune_config import TuneConfig
  from ray.tune.tuner import Tuner
label_desc: Use image node_id 10 for calling a global function with example usage: test_retry_with_max_failures(ray_start_4_cpus) without return types