Upload app.py

app.py

@@ -516,10 +516,22 @@ def evaluate(prompt: str) -> dict:
     """
     Evaluate a prompt using 50 fixed samples - SAME as initial and final evaluation.
 
+    OpenEvolve passes a file path, so we need to read the prompt from the file.
     Using the same 50 samples ensures evolution optimizes for the exact test set.
     Includes early stopping and rate limit handling.
     """
     try:
+        # CRITICAL: OpenEvolve passes a FILE PATH, not the prompt text!
+        # Check if prompt is a file path and read it
+        if os.path.exists(prompt):
+            with open(prompt, 'r') as f:
+                prompt_text = f.read()
+            # Strip EVOLVE-BLOCK markers if present
+            prompt_text = prompt_text.replace("# EVOLVE-BLOCK-START", "").replace("# EVOLVE-BLOCK-END", "").strip()
+        else:
+            # If not a file path, use as-is (for backward compatibility)
+            prompt_text = prompt
+
         # IMPORTANT: Use fixed seed for consistent sampling across all evaluations
         random.seed(42)
 
@@ -569,8 +581,8 @@ def evaluate(prompt: str) -> dict:
                 if isinstance(target, dict):
                     target = str(target)
 
-                # Format the prompt
-                formatted_prompt = prompt.replace("{{input}}", str(input_text))
+                # Format the prompt (use prompt_text that we read from file)
+                formatted_prompt = prompt_text.replace("{{input}}", str(input_text))
 
                 # Call the model with retry logic for transient failures
                 max_retries = 3
@@ -646,8 +658,8 @@ def evaluate(prompt: str) -> dict:
 
         print(f"Final: {{correct}}/{{total}} = {{accuracy:.2%}}")
 
-        # DEBUG: Log the prompt being evaluated and its score
-        prompt_preview = prompt[:80].replace('\\n', ' ') if len(prompt) > 80 else prompt.replace('\\n', ' ')
+        # DEBUG: Log the prompt being evaluated and its score (use prompt_text, not file path)
+        prompt_preview = prompt_text[:80].replace('\\n', ' ') if len(prompt_text) > 80 else prompt_text.replace('\\n', ' ')
         print(f"[EVAL DEBUG] Prompt: '{{prompt_preview}}...' → Score: {{accuracy:.2%}}")
 
         return {{
@@ -826,10 +838,14 @@ def optimize_prompt(initial_prompt: str, dataset_name: str, dataset_split: str,
     work_dir = tempfile.mkdtemp(prefix="openevolve_")
 
     try:
-        # Save initial prompt
+        # Save initial prompt with EVOLVE-BLOCK markers for OpenEvolve
+        # These markers tell OpenEvolve which part to optimize
         initial_prompt_path = os.path.join(work_dir, "initial_prompt.txt")
         with open(initial_prompt_path, "w") as f:
+            # Wrap prompt in evolve markers so OpenEvolve knows what to optimize
+            f.write("# EVOLVE-BLOCK-START\n")
             f.write(initial_prompt)
+            f.write("\n# EVOLVE-BLOCK-END\n")
 
         # Create evaluator
         progress(0.1, desc="Creating evaluator...")
@@ -929,16 +945,22 @@ def optimize_prompt(initial_prompt: str, dataset_name: str, dataset_split: str,
             best_prompt_path = os.path.join(output_dir, "best", "best_program.txt")
             if os.path.exists(best_prompt_path):
                 with open(best_prompt_path, "r") as f:
-                    best_prompt = f.read()
+                    best_prompt_raw = f.read()
+                # Strip EVOLVE-BLOCK markers that we added
+                best_prompt = best_prompt_raw.replace("# EVOLVE-BLOCK-START", "").replace("# EVOLVE-BLOCK-END", "").strip()
                 print(f"\n[SELECTION] OpenEvolve selected best prompt from: {best_prompt_path}")
+                print(f"[SELECTION] Raw prompt length: {len(best_prompt_raw)} chars")
                 print(f"[SELECTION] Best prompt: '{best_prompt[:100].replace(chr(10), ' ')}...'")
             else:
                 # Fallback: try without the "best" subdirectory
                 best_prompt_path_alt = os.path.join(output_dir, "best_program.txt")
                 if os.path.exists(best_prompt_path_alt):
                     with open(best_prompt_path_alt, "r") as f:
-                        best_prompt = f.read()
+                        best_prompt_raw = f.read()
+                    # Strip EVOLVE-BLOCK markers
+                    best_prompt = best_prompt_raw.replace("# EVOLVE-BLOCK-START", "").replace("# EVOLVE-BLOCK-END", "").strip()
                     print(f"\n[SELECTION] OpenEvolve selected best prompt from: {best_prompt_path_alt}")
+                    print(f"[SELECTION] Raw prompt length: {len(best_prompt_raw)} chars")
                     print(f"[SELECTION] Best prompt: '{best_prompt[:100].replace(chr(10), ' ')}...'")
                 else:
                     best_prompt = initial_prompt