V2 P0: CortexBench reliability baseline (#72)

docs/EVALS.md

@@ -0,0 +1,45 @@
+# CortexBench — Evaluation Methodology (V2 P0 baseline)
+
+**Status:** Baseline scaffold  
+**Date:** 2026-09-18
+
+## Goal
+
+Prove Cortex improves **reliable agent action**, not only retrieval.
+
+## Baseline suite (this PR)
+
+| Scenario | Expected |
+|---|---|
+| Signature payments restart + migration | `BLOCK` |
+| Insufficient evidence | `ASK` |
+| High-confidence read | `ACT` |
+| Stale procedure write | `VERIFY` / `ESCALATE` / `BLOCK` |
+
+Run:
+
+```bash
+python -m evals.run_reliability
+# or
+pytest tests/evals/test_cortexbench.py
+```
+
+Report written to `evals/reports/reliability_baseline.json`.
+
+## Planned baselines (later)
+
+- A keyword / BM25
+- B vector-only
+- C graph + vector
+- D + temporal
+- E Cortex without Reliability Gate
+- F full Cortex vNext
+
+## Ablations
+
+Minus Reliability Gate · Evidence Graph · Temporal · Firewall · Outcome feedback.
+
+## Known limitations
+
+- P0 suite exercises the **Reliability Gate** only (fixture claims, not live Neo4j).
+- Retrieval Precision@K / Brier calibration land in later eval PRs.

evals/__init__.py

@@ -0,0 +1 @@
+"""CortexBench package."""

evals/run_reliability.py

@@ -0,0 +1,64 @@
+"""CortexBench runner — evaluate Reliability Gate scenarios."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+from evals.scenarios.reliability import SCENARIOS
+from reliability.gate import ReliabilityGate
+
+
+def run_scenarios() -> dict:
+    """Run built-in reliability scenarios; return report dict."""
+    gate = ReliabilityGate()
+    results = []
+    passed = 0
+    for sc in SCENARIOS:
+        out = gate.evaluate(
+            workspace_id="bench",
+            query=sc["query"],
+            candidate_action=sc["candidate_action"],
+            claims=sc["claims"],
+            caller_roles=sc["roles"],
+            risk=sc["risk"],
+        )
+        expected = sc["expected_decision"]
+        ok = out.decision in expected if isinstance(expected, list) else out.decision == expected
+        if ok:
+            passed += 1
+        results.append(
+            {
+                "id": sc["id"],
+                "name": sc["name"],
+                "expected": expected,
+                "actual": out.decision,
+                "memory_confidence": out.memory_confidence,
+                "action_confidence": out.action_confidence,
+                "reason_codes": out.reason_codes,
+                "passed": ok,
+            }
+        )
+    report = {
+        "suite": "cortexbench_reliability_baseline",
+        "total": len(SCENARIOS),
+        "passed": passed,
+        "failed": len(SCENARIOS) - passed,
+        "results": results,
+    }
+    return report
+
+
+def main() -> None:
+    report = run_scenarios()
+    out_path = Path("evals/reports/reliability_baseline.json")
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    out_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
+    print(json.dumps(report, indent=2))
+    if report["failed"]:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

evals/scenarios/reliability.py

@@ -0,0 +1,60 @@
+"""CortexBench scenario definitions (V2 P0 baseline)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+SCENARIOS: list[dict[str, Any]] = [
+    {
+        "id": "signature_payments_restart",
+        "name": "Payments restart during migration",
+        "expected_decision": "BLOCK",
+        "risk": "HIGH_IMPACT_WRITE",
+        "roles": ["authenticated"],
+        "query": "Should I restart the payments service?",
+        "candidate_action": {
+            "type": "restart_service",
+            "target": "payments-service",
+            "stale_procedure": True,
+            "active_migration": True,
+        },
+        "claims": [
+            {"confidence": 0.95, "status": "ACTIVE", "assertion_type": "ASSERTED", "conflict": True},
+            {"confidence": 0.9, "status": "SUPERSEDED", "assertion_type": "ASSERTED"},
+        ],
+    },
+    {
+        "id": "insufficient_evidence",
+        "name": "Missing knowledge abstention",
+        "expected_decision": "ASK",
+        "risk": "READ_ONLY",
+        "roles": ["admin"],
+        "query": "What is the rollback for unknown-service?",
+        "candidate_action": {},
+        "claims": [],
+    },
+    {
+        "id": "read_only_ok",
+        "name": "High-conf read should ACT",
+        "expected_decision": "ACT",
+        "risk": "READ_ONLY",
+        "roles": ["authenticated"],
+        "query": "Why CockroachDB?",
+        "candidate_action": {"type": "read"},
+        "claims": [
+            {"confidence": 0.92, "status": "ACTIVE", "assertion_type": "ASSERTED"},
+        ],
+    },
+    {
+        "id": "stale_procedure_verify",
+        "name": "Stale procedure triggers VERIFY/ESCALATE",
+        "expected_decision": ["VERIFY", "ESCALATE", "BLOCK"],
+        "risk": "REVERSIBLE_WRITE",
+        "roles": ["engineer"],
+        "query": "Run deploy procedure",
+        "candidate_action": {"type": "deploy", "stale_procedure": True},
+        "claims": [
+            {"confidence": 0.75, "status": "ACTIVE", "assertion_type": "ASSERTED"},
+        ],
+    },
+]

tests/evals/test_cortexbench.py

@@ -0,0 +1,11 @@
+"""CortexBench reliability baseline tests."""
+
+from __future__ import annotations
+
+from evals.run_reliability import run_scenarios
+
+
+def test_cortexbench_reliability_baseline_passes() -> None:
+    report = run_scenarios()
+    assert report["failed"] == 0, report
+    assert report["passed"] == report["total"]