code: def test_checkpointing(
      self, tmp_path: Path, xor_trial_controller: Callable
  ) -> None:
      checkpoint_dir = str(tmp_path.joinpath("checkpoint"))
      latest_checkpoint = None
      steps_completed = 0
      old_loss = -1
  
      def make_workloads_1() -> workload.Stream:
          nonlocal old_loss
  
          trainer = utils.TrainAndValidate()
  
          yield from trainer.send(steps=10, validation_freq=10)
          training_metrics, validation_metrics = trainer.result()
          old_loss = validation_metrics[-1]["val_loss"]
  
          interceptor = workload.WorkloadResponseInterceptor()
          yield from interceptor.send(workload.checkpoint_workload())
          nonlocal latest_checkpoint, steps_completed
          latest_checkpoint = interceptor.metrics_result()["uuid"]
          steps_completed = trainer.get_steps_completed()
  
      controller = xor_trial_controller(
          self.hparams,
          make_workloads_1(),
          trial_seed=self.trial_seed,
          checkpoint_dir=checkpoint_dir,
      )
      controller.run()
  
      # Restore the checkpoint on a new trial instance and recompute
      # validation. The validation error should be the same as it was
      # previously.
      def make_workloads_2() -> workload.Stream:
          interceptor = workload.WorkloadResponseInterceptor()
  
          yield from interceptor.send(workload.validation_workload())
          metrics = interceptor.metrics_result()
  
          new_loss = metrics["metrics"]["validation_metrics"][
              "val_loss"
          ]
          assert new_loss == pytest.approx(old_loss)
  
      controller = xor_trial_controller(
          self.hparams,
          make_workloads_2(),
          trial_seed=self.trial_seed,
          checkpoint_dir=checkpoint_dir,
          latest_checkpoint=latest_checkpoint,
          steps_completed=steps_completed,
      )
      controller.run()
  
      # Verify that we can load the model from a checkpoint dir
      ckpt_path = str(tmp_path / "checkpoint" / latest_checkpoint)
      model = keras.load_model_from_checkpoint_path(ckpt_path)
      assert isinstance(model, tf.keras.models.Model), type(model)
  
imports: import os
  from pathlib import Path
  from typing import Any, Callable, Dict, List, Optional
  import pytest
  import tensorflow
  from packaging import version
  import determined
  from determined import keras, workload
  from tests.experiment import utils
  from tests.experiment.fixtures import tf_keras_xor_model
label_desc: Use image node_id 7 for calling the TestKerasTrial obj's underlying member method code with example usage: obj.test_checkpointing(tmp_path, xor_trial_controller) without return types