class_docstr: TextClassificationModel to be trained on two- or three-class textual entailment data, to be used for zero shot intent recognition.
class_docstr_tok: ["TextClassificationModel","to","be","trained","on","two-","or","three-class","textual","entailment","data",",","to","be","used","for","zero","shot","intent","recognition","."]
docstr: Get metrics based on the candidate label with the highest predicted likelihood and the ground truth label for intent
docstr_tok: ["Get","metrics","based","on","the","candidate","label","with","the","highest","predicted","likelihood","and","the","ground","truth","label","for","intent"]
code: def on_validation_epoch_end(self, split="val"):
      """
      Get metrics based on the candidate label with the highest predicted likelihood and the ground truth label for intent
      """
      output_logits = torch.cat(
          [output["logits"] for output in self.validation_step_outputs],
          dim=0,
      )
      output_input_ids = torch.cat(
          [
              output["input_ids"]
              for output in self.validation_step_outputs
          ],
          dim=0,
      )
      output_labels = torch.cat(
          [output["labels"] for output in self.validation_step_outputs],
          dim=0,
      )
  
      if self.cfg.library == "huggingface":
          entail_logits = output_logits[..., 2]
          decoded_input_ids = [
              self.tokenizer.decode(output_input_ids[i])
              for i in range(len(output_input_ids))
          ]
          utterance_candidate_pairs = [
              i.split(self.tokenizer.sep_token)
              for i in decoded_input_ids
          ]
          utterances = [
              i[0]
              .replace(self.tokenizer.bos_token, "")
              .replace(self.tokenizer.eos_token, "")
              for i in utterance_candidate_pairs
          ]
  
      elif self.cfg.library == "megatron":
          entail_logits = output_logits[..., 1]
          decoded_input_ids = [
              self.tokenizer.tokenizer.decode(output_input_ids[i])
              for i in range(len(output_input_ids))
          ]
          utterance_candidate_pairs = [
              i.split(self.tokenizer.tokenizer.sep_token)
              for i in decoded_input_ids
          ]
          utterances = [
              i[0]
              .replace(self.tokenizer.tokenizer.bos_token, "")
              .replace(self.tokenizer.tokenizer.eos_token, "")
              for i in utterance_candidate_pairs
          ]
  
      # account for uncased tokenization
      candidates = [
          i[1]
          .replace(self.cfg.dataset.prompt_template.lower(), "")
          .replace(self.cfg.dataset.prompt_template, "")
          .strip()
          for i in utterance_candidate_pairs
      ]
      utterance_to_idx = defaultdict(list)
      for idx, utterance in enumerate(utterances):
          utterance_to_idx[utterance].append(idx)
  
      predicted_labels = []
      ground_truth_labels = []
      utterances = []
      for utterance, idxs in utterance_to_idx.items():
          utterance_candidates = [candidates[idx] for idx in idxs]
          logits = [entail_logits[idx].item() for idx in idxs]
          labels = [output_labels[idx].item() for idx in idxs]
          correct_candidate = utterance_candidates[np.argmax(labels)]
          predicted_candidate = utterance_candidates[np.argmax(logits)]
          predicted_labels.append(predicted_candidate)
          ground_truth_labels.append(correct_candidate)
          utterances.append(utterance)
  
      os.makedirs(self.cfg.dataset.dialogues_example_dir, exist_ok=True)
      filename = os.path.join(
          self.cfg.dataset.dialogues_example_dir,
          "test_predictions.jsonl",
      )
  
      DialogueGenerationMetrics.save_predictions(
          filename,
          predicted_labels,
          ground_truth_labels,
          utterances,
      )
  
      label_to_ids = {
          label: idx
          for idx, label in enumerate(
              list(set(predicted_labels + ground_truth_labels))
          )
      }
      self.classification_report = ClassificationReport(
          num_classes=len(label_to_ids),
          mode="micro",
          label_ids=label_to_ids,
          dist_sync_on_step=True,
      ).to(output_logits[0].device)
      predicted_label_ids = torch.tensor(
          [label_to_ids[label] for label in predicted_labels]
      ).to(output_logits[0].device)
      ground_truth_label_ids = torch.tensor(
          [label_to_ids[label] for label in ground_truth_labels]
      ).to(output_logits[0].device)
  
      tp, fn, fp, _ = self.classification_report(
          predicted_label_ids, ground_truth_label_ids
      )
      (
          precision,
          recall,
          f1,
          report,
      ) = self.classification_report.compute()
      label_acc = np.mean(
          [
              int(predicted_labels[i] == ground_truth_labels[i])
              for i in range(len(predicted_labels))
          ]
      )
  
      avg_loss = torch.stack(
          [x[f"val_loss"] for x in self.validation_step_outputs]
      ).mean()
  
      logging.info(report)
  
      self.log("unified_precision", precision)
      self.log("unified_f1", f1)
      self.log("unified_recall", recall)
      self.log("unfied_accuracy", label_acc * 100)
      self.log("val_loss", avg_loss, prog_bar=True)
  
      self.validation_step_outputs.clear()  # free memory
      self.classification_report.reset()
  
imports: import os
  from collections import defaultdict
  from typing import Dict, List, Optional, Union
  import numpy
  import torch
  from omegaconf import DictConfig
  from pytorch_lightning import Trainer
  from transformers import AutoModelForSequenceClassification, AutoTokenizer
  from nemo.collections.nlp.data.dialogue import DialogueSGDDataProcessor
  from nemo.collections.nlp.data.dialogue.data_processor.assistant_data_processor import DialogueAssistantDataProcessor
  from nemo.collections.nlp.data.dialogue.data_processor.design_data_processor import DialogueDesignDataProcessor
  from nemo.collections.nlp.data.dialogue.dataset.dialogue_zero_shot_intent_dataset import DialogueZeroShotIntentDataset
  from nemo.collections.nlp.data.zero_shot_intent_recognition.zero_shot_intent_dataset import ZeroShotIntentInferenceDataset, calc_class_weights_from_dataloader
  from nemo.collections.nlp.metrics.classification_report import ClassificationReport
  from nemo.collections.nlp.metrics.dialogue_metrics import DialogueGenerationMetrics
  from nemo.collections.nlp.models import TextClassificationModel
  from nemo.core.classes.common import PretrainedModelInfo
  from nemo.utils import logging
label_desc: Use image node_id 9 for calling the DialogueZeroShotIntentModel obj's underlying member method code with example usage: obj.on_validation_epoch_end(split) without return types