Upload app.py

app.py

@@ -226,30 +226,22 @@ def evaluate_prompt(prompt: str, dataset_name: str, split: str, num_samples: int
                     max_tokens=500,
                 )
 
-                prediction = response.choices[0].message.content.strip().lower()
+                prediction = response.choices[0].message.content.strip()
 
                 # IMDB labels: 0 = negative, 1 = positive
                 true_label = int(target)  # 0 or 1
 
-                # Check for sentiment classification in first 100 chars (to avoid long explanations)
-                pred_start = prediction[:100]
+                # STRICT FORMAT REQUIREMENT: Must start with exactly "Sentiment: positive" or "Sentiment: negative"
+                # This teaches evolution to add proper format instructions
+                pred_lower = prediction.lower()
 
-                # Look for clear positive/negative indicators
-                has_positive = ("positive" in pred_start and "sentiment" in pred_start) or \
-                               ("this is positive" in pred_start) or \
-                               ("sentiment: positive" in pred_start)
-
-                has_negative = ("negative" in pred_start and "sentiment" in pred_start) or \
-                               ("this is negative" in pred_start) or \
-                               ("sentiment: negative" in pred_start)
-
-                # Prediction must be unambiguous
-                if has_positive and not has_negative:
+                # Check if response starts with the exact format (allow some whitespace)
+                if pred_lower.startswith("sentiment: positive") or pred_lower.startswith("sentiment:positive"):
                     predicted_label = 1
-                elif has_negative and not has_positive:
+                elif pred_lower.startswith("sentiment: negative") or pred_lower.startswith("sentiment:negative"):
                     predicted_label = 0
                 else:
-                    # Ambiguous or no clear signal = wrong
+                    # Wrong format = incorrect (even if sentiment is mentioned elsewhere)
                     predicted_label = -1
 
                 is_correct = (predicted_label == true_label)
@@ -572,30 +564,22 @@ def evaluate(prompt: str) -> dict:
                     max_tokens=500,
                 )
 
-                prediction = response.choices[0].message.content.strip().lower()
+                prediction = response.choices[0].message.content.strip()
 
                 # IMDB labels: 0 = negative, 1 = positive
                 true_label = int(target)  # 0 or 1
 
-                # Check for sentiment classification in first 100 chars (to avoid long explanations)
-                pred_start = prediction[:100]
-
-                # Look for clear positive/negative indicators
-                has_positive = ("positive" in pred_start and "sentiment" in pred_start) or \
-                               ("this is positive" in pred_start) or \
-                               ("sentiment: positive" in pred_start)
-
-                has_negative = ("negative" in pred_start and "sentiment" in pred_start) or \
-                               ("this is negative" in pred_start) or \
-                               ("sentiment: negative" in pred_start)
+                # STRICT FORMAT REQUIREMENT: Must start with exactly "Sentiment: positive" or "Sentiment: negative"
+                # This teaches evolution to add proper format instructions
+                pred_lower = prediction.lower()
 
-                # Prediction must be unambiguous
-                if has_positive and not has_negative:
+                # Check if response starts with the exact format (allow some whitespace)
+                if pred_lower.startswith("sentiment: positive") or pred_lower.startswith("sentiment:positive"):
                     predicted_label = 1
-                elif has_negative and not has_positive:
+                elif pred_lower.startswith("sentiment: negative") or pred_lower.startswith("sentiment:negative"):
                     predicted_label = 0
                 else:
-                    # Ambiguous or no clear signal = wrong
+                    # Wrong format = incorrect (even if sentiment is mentioned elsewhere)
                     predicted_label = -1
 
                 is_correct = (predicted_label == true_label)