class_docstr: Create a retryable version of a function.
  
  Calling this will call the passed function, retrying if any exceptions in
  retryable_exceptions are caught, with exponential backoff.
class_docstr_tok: ["Create","a","retryable","version","of","a","function",".","Calling","this","will","call","the","passed","function",",","retrying","if","any","exceptions","in","retryable_exceptions","are","caught",",","with","exponential","backoff","."]
docstr: Call the wrapped function, with retries.
  
  Arguments:
     retry_timedelta (kwarg): amount of time to retry before giving up.
     sleep_base (kwarg): amount of time to sleep upon first failure, all other sleeps
         are derived from this one.
docstr_tok: ["Call","the","wrapped","function",",","with","retries",".","Arguments",":","retry_timedelta","(","kwarg",")",":","amount","of","time","to","retry","before","giving","up",".","sleep_base","(","kwarg",")",":","amount","of","time","to","sleep","upon","first","failure",",","all","other","sleeps","are","derived","from","this","one","."]
code: def __call__(self, *args: Any, **kwargs: Any) -> _R:  # noqa: C901
      """Call the wrapped function, with retries.
  
      Arguments:
         retry_timedelta (kwarg): amount of time to retry before giving up.
         sleep_base (kwarg): amount of time to sleep upon first failure, all other sleeps
             are derived from this one.
      """
      retry_timedelta = kwargs.pop(
          "retry_timedelta", self._retry_timedelta
      )
      if retry_timedelta is None:
          retry_timedelta = datetime.timedelta(days=365)
  
      retry_cancel_event = kwargs.pop(
          "retry_cancel_event", self._retry_cancel_event
      )
  
      num_retries = kwargs.pop("num_retries", self._num_retries)
      if num_retries is None:
          num_retries = 1000000
  
      if os.environ.get("WANDB_TEST"):
          num_retries = 0
  
      sleep_base: float = kwargs.pop("retry_sleep_base", 1)
  
      # an extra function to allow performing more logic on the filtered exception
      check_retry_fn: CheckRetryFnType = kwargs.pop(
          "check_retry_fn", self._check_retry_fn
      )
  
      sleep = sleep_base
      now = NOW_FN()
      start_time = now
      start_time_triggered = None
  
      self._num_iter = 0
  
      while True:
          try:
              result = self._call_fn(*args, **kwargs)
              # Only print resolved attempts once every minute
              if (
                  self._num_iter > 2
                  and now - self._last_print
                  > datetime.timedelta(minutes=1)
              ):
                  self._last_print = NOW_FN()
                  if self.retry_callback:
                      self.retry_callback(
                          200,
                          "{} resolved after {}, resuming normal operation.".format(
                              self._error_prefix, NOW_FN() - start_time
                          ),
                      )
              return result
          except self._retryable_exceptions as e:
              # if the secondary check fails, re-raise
              retry_timedelta_triggered = check_retry_fn(e)
              if not retry_timedelta_triggered:
                  raise
  
              # always enforce num_retries no matter which type of exception was seen
              if self._num_iter >= num_retries:
                  raise
  
              now = NOW_FN()
  
              # handle a triggered secondary check which could have a shortened timeout
              if isinstance(
                  retry_timedelta_triggered, datetime.timedelta
              ):
                  # save the time of the first secondary trigger
                  if not start_time_triggered:
                      start_time_triggered = now
  
                  # make sure that we haven't run out of time from secondary trigger
                  if (
                      now - start_time_triggered
                      >= retry_timedelta_triggered
                  ):
                      raise
  
              # always enforce the default timeout from start of retries
              if now - start_time >= retry_timedelta:
                  raise
  
              if self._num_iter == 2:
                  logger.info("Retry attempt failed:", exc_info=e)
                  if (
                      isinstance(e, HTTPError)
                      and e.response is not None
                      and self.retry_callback is not None
                  ):
                      self.retry_callback(
                          e.response.status_code, e.response.text
                      )
                  else:
                      # todo: would like to catch other errors, eg wandb.errors.Error, ConnectionError etc
                      # but some of these can be raised before the retry handler thread (RunStatusChecker) is
                      # spawned in wandb_init
                      wandb.termlog(
                          "{} ({}), entering retry loop.".format(
                              self._error_prefix, e.__class__.__name__
                          )
                      )
              # if wandb.env.is_debug():
              #     traceback.print_exc()
          cancelled = self._sleep_check_cancelled(
              sleep + random.random() * 0.25 * sleep,
              cancel_event=retry_cancel_event,
          )
          if cancelled:
              raise ContextCancelledError("retry timeout")
          sleep *= 2
          if sleep > self.MAX_SLEEP_SECONDS:
              sleep = self.MAX_SLEEP_SECONDS
          now = NOW_FN()
  
          self._num_iter += 1
  
imports: import abc
  import asyncio
  import datetime
  import functools
  import logging
  import os
  import random
  import threading
  import time
  from typing import Any, Awaitable, Callable, Generic, Optional, Tuple, Type, TypeVar
  from requests import HTTPError
  import wandb
  from wandb.util import CheckRetryFnType
  from .mailbox import ContextCancelledError
label_desc: Use image node_id 4 for calling the Retry obj's underlying member method code with example usage: obj.__call__() and returns: result