class_docstr: Wrapper class for Hugging Face Transformers PyTorch T5 model.
class_docstr_tok: ["Wrapper","class","for","Hugging","Face","Transformers","PyTorch","T5","model","."]
docstr: Evaluate the model on the given Mixture or Task.
  
  *Note*: If a checkpoint step is provided (i.e. `checkpoint_steps is not
  None`), the model's state will be replaced by the state in those
  checkpoints. If you have not saved your model before calling `eval`, you
  should call `save_checkpoint` before `eval` to avoid losing its parameter
  values and state.
  
  Args:
    mixture_or_task_name: str, the name of the Mixture or Task to evaluate
      on.  Must be pre-registered in the global `t5.data.TaskRegistry` or
      `t5.data.MixtureRegistry.`
    sequence_length: dict of int, a dict mapping feature name to length.
    batch_size: int, the number of padded sequences in each batch.
    checkpoint_steps: int, list of ints, "all", or None. If None, eval in the
      model in its current state without loading any checkpoints. If an int
      or list of ints, evaluation will be run on the checkpoint files in
      `model_dir` whose global steps are those provided. If -1, eval on the
      latest checkpoint from the model directory. If "all", evaluate all
      checkpoints in the model directory.
    summary_dir: str, path to write TensorBoard events file summaries for
      eval. If None, use model_dir/{split}_eval.
    split: str, the mixture/task split to evaluate on.
    compute_sequence_length: bool, automatically compute sequence length
      during eval mode.
    **generate_kwargs: Additional keyword arguments to pass to
      `transformers.PretrainedModel.generate()`, for example to change the
      decoding strategy. See the documentation for
      `transformers.PretrainedModel.generate()` for options.
docstr_tok: ["Evaluate","the","model","on","the","given","Mixture","or","Task",".","*","Note","*",":","If","a","checkpoint","step","is","provided","(","i.e",".","`","checkpoint_steps","is","not","None","`",")",",","the","model","'s","state","will","be","replaced","by","the","state","in","those","checkpoints",".","If","you","have","not","saved","your","model","before","calling","`","eval","`",",","you","should","call","`","save_checkpoint","`","before","`","eval","`","to","avoid","losing","its","parameter","values","and","state",".","Args",":","mixture_or_task_name",":","str",",","the","name","of","the","Mixture","or","Task","to","evaluate","on",".","Must","be","pre-registered","in","the","global","`","t5.data.TaskRegistry","`","or","`","t5.data.MixtureRegistry.","`","sequence_length",":","dict","of","int",",","a","dict","mapping","feature","name","to","length",".","batch_size",":","int",",","the","number","of","padded","sequences","in","each","batch",".","checkpoint_steps",":","int",",","list","of","ints",",","``","all","''",",","or","None",".","If","None",",","eval","in","the","model","in","its","current","state","without","loading","any","checkpoints",".","If","an","int","or","list","of","ints",",","evaluation","will","be","run","on","the","checkpoint","files","in","`","model_dir","`","whose","global","steps","are","those","provided",".","If","-1",",","eval","on","the","latest","checkpoint","from","the","model","directory",".","If","``","all","''",",","evaluate","all","checkpoints","in","the","model","directory",".","summary_dir",":","str",",","path","to","write","TensorBoard","events","file","summaries","for","eval",".","If","None",",","use","model_dir\/","{","split","}","_eval",".","split",":","str",",","the","mixture\/task","split","to","evaluate","on",".","compute_sequence_length",":","bool",",","automatically","compute","sequence","length","during","eval","mode",".","*","*","generate_kwargs",":","Additional","keyword","arguments","to","pass","to","`","transformers.PretrainedModel.generate","(",")","`",",","for","example","to","change","the","decoding","strategy",".","See","the","documentation","for","`","transformers.PretrainedModel.generate","(",")","`","for","options","."]
code: def eval(
      self,
      mixture_or_task_name,
      sequence_length,
      batch_size,
      checkpoint_steps=None,
      summary_dir=None,
      split="validation",
      compute_sequence_length=False,
      **generate_kwargs,
  ):
      """Evaluate the model on the given Mixture or Task.
  
      *Note*: If a checkpoint step is provided (i.e. `checkpoint_steps is not
      None`), the model's state will be replaced by the state in those
      checkpoints. If you have not saved your model before calling `eval`, you
      should call `save_checkpoint` before `eval` to avoid losing its parameter
      values and state.
  
      Args:
        mixture_or_task_name: str, the name of the Mixture or Task to evaluate
          on.  Must be pre-registered in the global `t5.data.TaskRegistry` or
          `t5.data.MixtureRegistry.`
        sequence_length: dict of int, a dict mapping feature name to length.
        batch_size: int, the number of padded sequences in each batch.
        checkpoint_steps: int, list of ints, "all", or None. If None, eval in the
          model in its current state without loading any checkpoints. If an int
          or list of ints, evaluation will be run on the checkpoint files in
          `model_dir` whose global steps are those provided. If -1, eval on the
          latest checkpoint from the model directory. If "all", evaluate all
          checkpoints in the model directory.
        summary_dir: str, path to write TensorBoard events file summaries for
          eval. If None, use model_dir/{split}_eval.
        split: str, the mixture/task split to evaluate on.
        compute_sequence_length: bool, automatically compute sequence length
          during eval mode.
        **generate_kwargs: Additional keyword arguments to pass to
          `transformers.PretrainedModel.generate()`, for example to change the
          decoding strategy. See the documentation for
          `transformers.PretrainedModel.generate()` for options.
      """
  
      def _predict_from_tasks(
          tasks,
          vocabulary,
          checkpoint_step,
          sequence_length,
          datasets,
          **unused_kwargs,
      ):
          if isinstance(vocabulary, tuple):
              vocab = vocabulary[1]
  
          if checkpoint_step != self._step:
              self.load_checkpoint(checkpoint_step)
          self._model.eval()
          outputs = []
          for task in tasks:
              if compute_sequence_length:
                  ds = _get_dataset(
                      task.name, sequence_length, split, shuffle=False
                  )
              else:
                  ds = datasets[task.name]
  
              ds = list(
                  tokens_to_batches(
                      ds,
                      sequence_length,
                      batch_size,
                      tuple(task.output_features),
                      task,
                  )
              )
              for batch in ds:
                  predicted_tokens = self._model.generate(
                      input_ids=self.to_tensor(batch["inputs"]),
                      **generate_kwargs,
                  )
                  predicted_tokens = (
                      predicted_tokens.cpu().numpy().tolist()
                  )
                  predictions = [
                      vocab.decode(p) for p in predicted_tokens
                  ]
  
                  outputs.extend(predictions)
  
          return outputs
  
      if checkpoint_steps is None:
          checkpoint_steps = [self._step]
      elif isinstance(checkpoint_steps, int):
          checkpoint_steps = [checkpoint_steps]
      elif checkpoint_steps == "all":
          checkpoint_steps = self.get_all_checkpoint_steps()
      elif not isinstance(checkpoint_steps, (list, tuple)):
          raise ValueError(
              f"checkpoint_steps must be None, int or list; got {checkpoint_steps}"
          )
  
      summary_dir = summary_dir or os.path.join(
          self._model_dir, f"{split}_eval"
      )
      tf.io.gfile.makedirs(summary_dir)
  
      utils.run_eval(
          mixture_or_task_name=mixture_or_task_name,
          predict_or_score_fn=_predict_from_tasks,
          checkpoint_steps=checkpoint_steps,
          dataset_fn=functools.partial(_get_dataset, shuffle=False),
          summary_dir=summary_dir,
          split=split,
          sequence_length=None
          if compute_sequence_length
          else sequence_length,
          batch_size=batch_size,
      )
  
imports: import functools
  import itertools
  import os
  import re
  import time
  from absl import logging
  import mesh_tensorflow.transformer.dataset
  import seqio
  import t5.data
  from t5.models import utils
  from t5.models.t5_model import T5Model
  import tensorflow.compat.v1
  import tensorflow_datasets
  import torch
  import torch.utils.tensorboard
label_desc: Use image node_id 10 for calling the HfPyTorchModel obj's underlying member method code with example usage: obj.eval(mixture_or_task_name, sequence_length, batch_size, checkpoint_steps, summary_dir, split, compute_sequence_length) and returns: outputs