speed up inference

ShaderEval.py

@@ -106,14 +106,14 @@ class ReturnGenerationEvaluator(evaluate.TextGenerationEvaluator):
     
     def _estimate_stopping(self, labels, **kwargs):
         """ estimates max_new_tokens for the pipeline call
-            by counting the characters in the  longest string of the references and multiplying by 2 (for good measure but probably not needed)
+            by counting the characters in the  longest string of the references adding 5 (for good measure but probably not needed)
         Args:
             labels: A list of dicts by knowing the labels
         Returns:
             `int`: the estimated max_new_tokens, should be smaller than context_lenght in all cases
         """
         context_lenght = self._resolve_context_lenght(**kwargs)
-        estimate = min(max([len(ref) for ref in labels])*2, context_lenght)
+        estimate = min(max([len(ref) for ref in labels]) + 5, context_lenght)
         return estimate
 
     # this one needs to be adjusted