class_docstr: Handles the main model training procedure. This includes performing evaluation on the dev set at regular
  intervals during training as well as evaluation on the test set at the end of training.
class_docstr_tok: ["Handles","the","main","model","training","procedure",".","This","includes","performing","evaluation","on","the","dev","set","at","regular","intervals","during","training","as","well","as","evaluation","on","the","test","set","at","the","end","of","training","."]
docstr: Save a train checkpoint at the Trainer's checkpoint_path.
  
  Some objects(eg, scheduler) in the Trainer are not serializable using the Pickle module. For these objects,
  the state_dict is stored for the checkpoint, that can be used to reconstruct a similar state upon resuming
  train from the checkpoint.
  
  #TODO The model is currently saved as a whole serialized object. The disadvantage of this approach is that it is
  bound to specifics Python version, FARM version, directory structures etc. A more modular and reusable approach
  is to save using AdaptiveModel's save() method where the model and the state_dict are stored separately.
  
  # TODO custom defined evaluators are not saved in the checkpoint.
docstr_tok: ["Save","a","train","checkpoint","at","the","Trainer","'s","checkpoint_path",".","Some","objects","(","eg",",","scheduler",")","in","the","Trainer","are","not","serializable","using","the","Pickle","module",".","For","these","objects",",","the","state_dict","is","stored","for","the","checkpoint",",","that","can","be","used","to","reconstruct","a","similar","state","upon","resuming","train","from","the","checkpoint",".","#","TODO","The","model","is","currently","saved","as","a","whole","serialized","object",".","The","disadvantage","of","this","approach","is","that","it","is","bound","to","specifics","Python","version",",","FARM","version",",","directory","structures","etc",".","A","more","modular","and","reusable","approach","is","to","save","using","AdaptiveModel","'s","save","(",")","method","where","the","model","and","the","state_dict","are","stored","separately",".","#","TODO","custom","defined","evaluators","are","not","saved","in","the","checkpoint","."]
code: def _save(self):
      """
      Save a train checkpoint at the Trainer's checkpoint_path.
  
      Some objects(eg, scheduler) in the Trainer are not serializable using the Pickle module. For these objects,
      the state_dict is stored for the checkpoint, that can be used to reconstruct a similar state upon resuming
      train from the checkpoint.
  
      #TODO The model is currently saved as a whole serialized object. The disadvantage of this approach is that it is
      bound to specifics Python version, FARM version, directory structures etc. A more modular and reusable approach
      is to save using AdaptiveModel's save() method where the model and the state_dict are stored separately.
  
      # TODO custom defined evaluators are not saved in the checkpoint.
      """
      logger.info("Saving a train checkpoint ...")
      checkpoint_path = (
          self.checkpoint_root_dir / "checkpoint_in_progress"
      )
      checkpoint_path.mkdir(parents=True, exist_ok=True)
  
      trainer_state_dict = self._get_state_dict()
  
      # save as a regular AdaptiveModel (e.g. for down-stream eval during training from scratch)
      self.model.save(checkpoint_path)
  
      # save all state dicst (incl. the model) to have full reproducibility
      torch.save(
          {
              "trainer_state": trainer_state_dict,
              "model_state": self.model.state_dict(),
              "optimizer_state": self.optimizer.state_dict(),
              "scheduler_opts": self.lr_schedule.opts,
              "scheduler_state": self.lr_schedule.state_dict(),
              "numpy_rng_state": numpy.random.get_state(),
              "rng_state": torch.get_rng_state(),
              "cuda_rng_state": torch.cuda.get_rng_state(),
          },
          checkpoint_path / "trainer",
          pickle_module=dill,
      )
  
      checkpoint_name = (
          f"epoch_{self.from_epoch}_step_{self.from_step-1}"
      )
      checkpoint_path.replace(
          Path(checkpoint_path.parent) / checkpoint_name
      )
  
      saved_checkpoints = self._get_checkpoints(
          self.checkpoint_root_dir
      )
      if len(saved_checkpoints) > self.checkpoints_to_keep:
          for cp in saved_checkpoints[self.checkpoints_to_keep :]:
              shutil.rmtree(cp)
  
      logger.info(
          f"Saved a training checkpoint after {checkpoint_name}"
      )
  
imports: import logging
  import sys
  import torch
  from pathlib import Path
  from tqdm import tqdm
  import numpy
  import shutil
  import dill
  from farm.utils import MLFlowLogger
  from farm.utils import GracefulKiller, set_all_seeds
  from farm.eval import Evaluator
  from farm.data_handler.data_silo import DataSilo
  from farm.visual.ascii.images import GROWING_TREE
  from farm.modeling.adaptive_model import AdaptiveModel
  from farm.modeling.biadaptive_model import BiAdaptiveModel
  from farm.modeling.optimization import get_scheduler
label_desc: Use image node_id 9 for calling the Trainer obj's underlying member method code with example usage: obj._save() without return types