class_docstr: This module wraps a standard criterion and adds an extra knowledge distillation loss by
  taking a teacher model prediction and using it as additional supervision.
class_docstr_tok: ["This","module","wraps","a","standard","criterion","and","adds","an","extra","knowledge","distillation","loss","by","taking","a","teacher","model","prediction","and","using","it","as","additional","supervision","."]
docstr: Args:
      inputs: The original inputs that are feed to the teacher model.
      outputs: Output tensor from model being trained.
      labels: the labels for the base criterion.
docstr_tok: ["Args",":","inputs",":","The","original","inputs","that","are","feed","to","the","teacher","model",".","outputs",":","Output","tensor","from","model","being","trained",".","labels",":","the","labels","for","the","base","criterion","."]
code: def forward(self, inputs, outputs, labels):
      """
      Args:
          inputs: The original inputs that are feed to the teacher model.
          outputs: Output tensor from model being trained.
          labels: the labels for the base criterion.
      """
      base_loss = self.base_criterion(outputs, labels)
      if self.distillation_type == "none":
          return base_loss
  
      # don't backprop throught the teacher
      with torch.no_grad():
          teacher_outputs = self.teacher_model(inputs)
  
      if self.distillation_type == "soft":
          T = self.tau
          # taken from https://github.com/peterliht/knowledge-distillation-pytorch/blob/master/model/net.py#L100
          # with slight modifications
          distillation_loss = (
              F.kl_div(
                  F.log_softmax(outputs / T, dim=1),
                  # We provide the teacher's targets in log probability because we use log_target=True
                  # (as recommended in pytorch https://github.com/pytorch/pytorch/blob/9324181d0ac7b4f7949a574dbc3e8be30abe7041/torch/nn/functional.py#L2719)
                  # but it is possible to give just the probabilities and set log_target=False. In our experiments we tried both.
                  F.log_softmax(teacher_outputs / T, dim=1),
                  reduction="sum",
                  log_target=True,
              )
              * (T * T)
              / outputs.numel()
          )
          # We divide by outputs_kd.numel() to have the legacy PyTorch behavior.
          # But we also experiments output_kd.size(0)
          # see issue 61(https://github.com/facebookresearch/deit/issues/61) for more details
      elif self.distillation_type == "hard":
          distillation_loss = F.cross_entropy(
              outputs, teacher_outputs.argmax(dim=1)
          )
  
      loss = (
          base_loss * (1 - self.alpha) + distillation_loss * self.alpha
      )
      return loss
  
imports: import torch
  from torch.nn import functional
label_desc: Use image node_id 2 for calling the DistillationLoss obj's underlying member method code with example usage: obj.forward(inputs, outputs, labels) and returns: loss, base_loss