Reject blocked experiment outputs

Author: black-yt

src/agentworld/research/__init__.py

@@ -8,6 +8,7 @@
     ExperimentManifest,
     format_experiment_manifest_for_prompt,
     load_experiment_manifest,
+    validate_experiment_execution,
     validate_experiment_manifest,
     write_experiment_manifest,
 )
@@ -35,6 +36,7 @@
     "load_experiment_manifest",
     "load_hypothesis_manifest",
     "validate_citation_verification",
+    "validate_experiment_execution",
     "validate_experiment_manifest",
     "validate_literature_evidence",
     "write_experiment_manifest",

src/agentworld/research/experiment.py

@@ -115,6 +115,38 @@ def validate_experiment_manifest(path: Path) -> list[str]:
     return problems
 
 
+def validate_experiment_execution(workspace: RunWorkspace) -> list[str]:
+    results_path = workspace.results_dir / "results.json"
+    if not results_path.exists():
+        return []
+    try:
+        payload = json.loads(results_path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        return [f"results.json is not valid JSON: {exc}"]
+    if not isinstance(payload, dict):
+        return ["results.json must contain a JSON object."]
+
+    problems: list[str] = []
+    if payload.get("experiments_executed") is False:
+        problems.append("results.json reports experiments_executed=false.")
+    execution_status = str(payload.get("execution_status") or "").strip().lower()
+    if execution_status in {"blocked", "failed", "not_run", "not_executed", "skipped"}:
+        problems.append(f"results.json reports execution_status={execution_status}.")
+    if payload.get("execution_blocker"):
+        problems.append("results.json contains execution_blocker; the experiment did not complete.")
+
+    expected_outputs = payload.get("expected_outputs_on_success")
+    if isinstance(expected_outputs, dict) and not problems:
+        missing = []
+        for value in expected_outputs.values():
+            candidate = workspace.run_root / str(value)
+            if not candidate.exists():
+                missing.append(str(value))
+        if missing:
+            problems.append("results.json expected output(s) are missing: " + ", ".join(missing))
+    return problems
+
+
 def format_experiment_manifest_for_prompt(manifest: ExperimentManifest, max_results: int = 5) -> str:
     lines = [
         f"Experiment manifest generated at: {manifest.generated_at}",

src/agentworld/workflows/auto_research.py

@@ -23,6 +23,7 @@
 )
 from ..research import (
     validate_citation_verification,
+    validate_experiment_execution,
     validate_experiment_manifest,
     validate_literature_evidence,
     write_experiment_manifest,
@@ -548,6 +549,7 @@ def _validate_auto_research_artifacts(self, workspace: RunWorkspace, stage: Stag
             problems.extend(f"{stage.title}: {problem}" for problem in validate_literature_evidence(workspace))
         if stage.number >= 5:
             problems.extend(f"{stage.title}: {problem}" for problem in validate_experiment_manifest(workspace.experiment_manifest))
+            problems.extend(f"{stage.title}: {problem}" for problem in validate_experiment_execution(workspace))
         if stage.number >= 7:
             citation_path = workspace.artifacts_dir / "citation_verification.json"
             problems.extend(f"{stage.title}: {problem}" for problem in validate_citation_verification(citation_path))

tests/test_auto_research_workflow.py

@@ -88,7 +88,18 @@ def _materialize_stage_files(self, slug: str, workspace) -> list[str]:
         if slug == "05_experimentation":
             write_text(
                 workspace.results_dir / "results.json",
-                json.dumps([{"setting": "base", "score": 0.81}, {"setting": "ablation", "score": 0.77}], indent=2),
+                json.dumps(
+                    {
+                        "experiments_executed": True,
+                        "execution_status": "completed",
+                        "model_results": [
+                            {"setting": "base", "score": 0.81},
+                            {"setting": "ablation", "score": 0.77},
+                        ],
+                    },
+                    indent=2,
+                    ensure_ascii=True,
+                ),
             )
             return ["workspace/results/results.json"]
         if slug == "06_analysis":

tests/test_research_manifests.py

@@ -8,6 +8,7 @@
 from agentworld.research import (
     format_experiment_manifest_for_prompt,
     format_hypothesis_manifest_for_prompt,
+    validate_experiment_execution,
     validate_experiment_manifest,
     write_experiment_manifest,
     write_hypothesis_manifest,
@@ -162,6 +163,55 @@ def test_write_experiment_manifest_collects_schema_and_context(self) -> None:
             self.assertEqual(schema["row_count"], 2)
             self.assertIn("Result Artifacts", format_experiment_manifest_for_prompt(manifest))
 
+    def test_validate_experiment_execution_rejects_blocked_results(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            workspace = create_run_workspace(runs_dir=Path(tmp), run_id="blocked", goal="test")
+            write_text(
+                workspace.results_dir / "results.json",
+                json.dumps(
+                    {
+                        "experiments_executed": False,
+                        "execution_status": "blocked",
+                        "execution_blocker": {"reason": "permission approval required"},
+                    },
+                    indent=2,
+                    ensure_ascii=True,
+                ),
+            )
+
+            problems = validate_experiment_execution(workspace)
+
+            self.assertIn("results.json reports experiments_executed=false.", problems)
+            self.assertIn("results.json reports execution_status=blocked.", problems)
+            self.assertIn("results.json contains execution_blocker; the experiment did not complete.", problems)
+
+    def test_validate_experiment_execution_checks_declared_outputs(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            workspace = create_run_workspace(runs_dir=Path(tmp), run_id="outputs", goal="test")
+            write_text(workspace.results_dir / "metrics.json", "{}\n")
+            write_text(
+                workspace.results_dir / "results.json",
+                json.dumps(
+                    {
+                        "experiments_executed": True,
+                        "execution_status": "completed",
+                        "expected_outputs_on_success": {
+                            "metrics": "workspace/results/metrics.json",
+                            "figure": "workspace/figures/missing.png",
+                        },
+                    },
+                    indent=2,
+                    ensure_ascii=True,
+                ),
+            )
+
+            problems = validate_experiment_execution(workspace)
+
+            self.assertEqual(
+                problems,
+                ["results.json expected output(s) are missing: workspace/figures/missing.png"],
+            )
+
 
 if __name__ == "__main__":
     unittest.main()