docstr: Performs the learning rate range test.
  Arguments:
      start_lr (float, optional): the starting learning rate for the range test.
          Default: None (uses the learning rate from the optimizer).
      end_lr (float, optional): the maximum learning rate to test. Default: 10.
      num_iter (int, optional): the number of iterations over which the test
          occurs. Default: 100.
      step_mode (str, optional): one of the available learning rate policies,
          linear or exponential ("linear", "exp"). Default: "exp".
      smooth_f (float, optional): the loss smoothing factor within the [0, 1[
          interval. Disabled if set to 0, otherwise the loss is smoothed using
          exponential smoothing. Default: 0.05.
      diverge_th (int, optional): the test is stopped when the loss surpasses the
          threshold:  diverge_th * best_loss. Default: 5.
  Reference:
  [Training Neural Nets on Larger Batches: Practical Tips for 1-GPU, Multi-GPU & Distributed setups](
  https://medium.com/huggingface/ec88c3e51255)
  [thomwolf/gradient_accumulation](https://gist.github.com/thomwolf/ac7a7da6b1888c2eeac8ac8b9b05d3d3)
docstr_tok: ["Performs","the","learning","rate","range","test",".","Arguments",":","start_lr","(","float",",","optional",")",":","the","starting","learning","rate","for","the","range","test",".","Default",":","None","(","uses","the","learning","rate","from","the","optimizer",")",".","end_lr","(","float",",","optional",")",":","the","maximum","learning","rate","to","test",".","Default",":","10.","num_iter","(","int",",","optional",")",":","the","number","of","iterations","over","which","the","test","occurs",".","Default",":","100.","step_mode","(","str",",","optional",")",":","one","of","the","available","learning","rate","policies",",","linear","or","exponential","(","``","linear","''",",","``","exp","''",")",".","Default",":","``","exp","''",".","smooth_f","(","float",",","optional",")",":","the","loss","smoothing","factor","within","the","[","0",",","1","[","interval",".","Disabled","if","set","to","0",",","otherwise","the","loss","is","smoothed","using","exponential","smoothing",".","Default",":","0.05.","diverge_th","(","int",",","optional",")",":","the","test","is","stopped","when","the","loss","surpasses","the","threshold",":","diverge_th","*","best_loss",".","Default",":","5",".","Reference",":","[","Training","Neural","Nets","on","Larger","Batches",":","Practical","Tips","for","1-GPU",",","Multi-GPU","&","Distributed","setups","]","(","https",":","\/\/medium.com\/huggingface\/ec88c3e51255",")","[","thomwolf\/gradient_accumulation","]","(","https",":","\/\/gist.github.com\/thomwolf\/ac7a7da6b1888c2eeac8ac8b9b05d3d3",")"]
code: def lr_find(
      self,
      start_lr,
      end_lr=10,
      use_val_loss=True,
      optimizer_type="lamb",
      num_iter=100,
      step_mode="exp",
      smooth_f=0.05,
      diverge_th=5,
  ):
      """Performs the learning rate range test.
      Arguments:
          start_lr (float, optional): the starting learning rate for the range test.
              Default: None (uses the learning rate from the optimizer).
          end_lr (float, optional): the maximum learning rate to test. Default: 10.
          num_iter (int, optional): the number of iterations over which the test
              occurs. Default: 100.
          step_mode (str, optional): one of the available learning rate policies,
              linear or exponential ("linear", "exp"). Default: "exp".
          smooth_f (float, optional): the loss smoothing factor within the [0, 1[
              interval. Disabled if set to 0, otherwise the loss is smoothed using
              exponential smoothing. Default: 0.05.
          diverge_th (int, optional): the test is stopped when the loss surpasses the
              threshold:  diverge_th * best_loss. Default: 5.
      Reference:
      [Training Neural Nets on Larger Batches: Practical Tips for 1-GPU, Multi-GPU & Distributed setups](
      https://medium.com/huggingface/ec88c3e51255)
      [thomwolf/gradient_accumulation](https://gist.github.com/thomwolf/ac7a7da6b1888c2eeac8ac8b9b05d3d3)
      """
  
      # Reset test results
      self.history = {"lr": [], "loss": []}
      self.best_loss = None
      self.state_cacher = StateCacher(True, cache_dir=self.output_dir)
  
      self.optimizer = self.get_optimizer(
          lr=start_lr, optimizer_type=optimizer_type
      )
  
      if hasattr(self.model, "module"):
          self.model = self.model.module
  
      self.state_cacher.store("model", self.model.state_dict())
      self.state_cacher.store("optimizer", self.optimizer.state_dict())
  
      # Parallelize the model architecture
      if self.multi_gpu is True:
          self.model = torch.nn.DataParallel(self.model)
  
      # Check if the optimizer is already attached to a scheduler
      self._check_for_scheduler()
  
      # Set the starting learning rate
      if start_lr:
          self._set_learning_rate(start_lr)
  
      # Initialize the proper learning rate policy
      if step_mode.lower() == "exp":
          lr_schedule = ExponentialLR(self.optimizer, end_lr, num_iter)
      elif step_mode.lower() == "linear":
          lr_schedule = LinearLR(self.optimizer, end_lr, num_iter)
      else:
          raise ValueError(
              "expected one of (exp, linear), got {}".format(step_mode)
          )
  
      if smooth_f < 0 or smooth_f >= 1:
          raise ValueError("smooth_f is outside the range [0, 1]")
  
      train_iter = TrainDataLoaderIter(self.data.train_dl)
  
      for iteration in tqdm(range(num_iter)):
          # train on batch and retrieve loss
          loss = self._train_batch(train_iter)
          if use_val_loss:
              loss = self.validate(quiet=True, loss_only=True)["loss"]
  
          # Update the learning rate
          self.history["lr"].append(lr_schedule.get_lr()[0])
          lr_schedule.step()
  
          # Track the best loss and smooth it if smooth_f is specified
          if iteration == 0:
              self.best_loss = loss
          else:
              if smooth_f > 0:
                  loss = (
                      smooth_f * loss
                      + (1 - smooth_f) * self.history["loss"][-1]
                  )
              if loss < self.best_loss:
                  self.best_loss = loss
  
          # Check if the loss has diverged; if it has, stop the test
          self.history["loss"].append(loss)
          if loss > diverge_th * self.best_loss:
              print("Stopping early, the loss has diverged")
              break
  
      print(
          "Learning rate search finished. See the graph with {finder_name}.plot()"
      )
      self.reset()
      self.plot()
  
imports: import os
  import copy
  import logging
  from packaging import version
  from .data_cls import BertDataBunch
  from tqdm.autonotebook import tqdm
  import matplotlib.pyplot
  from .learner_util import Learner
  from torch import nn
  from typing import List, Optional
  from .modeling import BertForMultiLabelSequenceClassification, XLNetForMultiLabelSequenceClassification, RobertaForMultiLabelSequenceClassification, DistilBertForMultiLabelSequenceClassification, CamembertForMultiLabelSequenceClassification, AlbertForMultiLabelSequenceClassification, ElectraForMultiLabelSequenceClassification, FlaubertForMultiLabelSequenceClassification
  from fastprogress.fastprogress import master_bar, progress_bar
  import torch
  import pandas
  import numpy
  from sklearn.metrics import roc_curve, auc
  from pathlib import Path
  from torch.optim.lr_scheduler import _LRScheduler, Optimizer
  from tensorboardX import SummaryWriter
  from transformers import WEIGHTS_NAME, BertConfig, BertForSequenceClassification, BertTokenizer, XLMConfig, XLMForSequenceClassification, XLMTokenizer, XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer, RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer, CamembertConfig, CamembertForSequenceClassification, CamembertTokenizer, AlbertConfig, AlbertForSequenceClassification, AlbertTokenizer, DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer, ElectraConfig, ElectraForSequenceClassification, ElectraTokenizer, FlaubertConfig, FlaubertForSequenceClassification, FlaubertTokenizer
  from transformers import AutoModelForSequenceClassification, AutoConfig, TrainingArguments
  from transformers.trainer_callback import TrainerCallback, TrainerControl, TrainerState, CallbackHandler, DefaultFlowCallback
label_desc: Use image node_id 8 for calling the BertLearner obj's underlying member method code with example usage: obj.lr_find(start_lr, end_lr, use_val_loss, optimizer_type, num_iter, step_mode, smooth_f, diverge_th) without return types