diff --git a/docs/EVALS.md b/docs/EVALS.md new file mode 100644 index 0000000..29673f8 --- /dev/null +++ b/docs/EVALS.md @@ -0,0 +1,45 @@ +# CortexBench — Evaluation Methodology (V2 P0 baseline) + +**Status:** Baseline scaffold +**Date:** 2026-09-18 + +## Goal + +Prove Cortex improves **reliable agent action**, not only retrieval. + +## Baseline suite (this PR) + +| Scenario | Expected | +|---|---| +| Signature payments restart + migration | `BLOCK` | +| Insufficient evidence | `ASK` | +| High-confidence read | `ACT` | +| Stale procedure write | `VERIFY` / `ESCALATE` / `BLOCK` | + +Run: + +```bash +python -m evals.run_reliability +# or +pytest tests/evals/test_cortexbench.py +``` + +Report written to `evals/reports/reliability_baseline.json`. + +## Planned baselines (later) + +- A keyword / BM25 +- B vector-only +- C graph + vector +- D + temporal +- E Cortex without Reliability Gate +- F full Cortex vNext + +## Ablations + +Minus Reliability Gate · Evidence Graph · Temporal · Firewall · Outcome feedback. + +## Known limitations + +- P0 suite exercises the **Reliability Gate** only (fixture claims, not live Neo4j). +- Retrieval Precision@K / Brier calibration land in later eval PRs. diff --git a/evals/__init__.py b/evals/__init__.py new file mode 100644 index 0000000..6852bd2 --- /dev/null +++ b/evals/__init__.py @@ -0,0 +1 @@ +"""CortexBench package.""" diff --git a/evals/run_reliability.py b/evals/run_reliability.py new file mode 100644 index 0000000..3a12037 --- /dev/null +++ b/evals/run_reliability.py @@ -0,0 +1,64 @@ +"""CortexBench runner — evaluate Reliability Gate scenarios.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +from evals.scenarios.reliability import SCENARIOS +from reliability.gate import ReliabilityGate + + +def run_scenarios() -> dict: + """Run built-in reliability scenarios; return report dict.""" + gate = ReliabilityGate() + results = [] + passed = 0 + for sc in SCENARIOS: + out = gate.evaluate( + workspace_id="bench", + query=sc["query"], + candidate_action=sc["candidate_action"], + claims=sc["claims"], + caller_roles=sc["roles"], + risk=sc["risk"], + ) + expected = sc["expected_decision"] + ok = out.decision in expected if isinstance(expected, list) else out.decision == expected + if ok: + passed += 1 + results.append( + { + "id": sc["id"], + "name": sc["name"], + "expected": expected, + "actual": out.decision, + "memory_confidence": out.memory_confidence, + "action_confidence": out.action_confidence, + "reason_codes": out.reason_codes, + "passed": ok, + } + ) + report = { + "suite": "cortexbench_reliability_baseline", + "total": len(SCENARIOS), + "passed": passed, + "failed": len(SCENARIOS) - passed, + "results": results, + } + return report + + +def main() -> None: + report = run_scenarios() + out_path = Path("evals/reports/reliability_baseline.json") + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(report, indent=2), encoding="utf-8") + print(json.dumps(report, indent=2)) + if report["failed"]: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/evals/scenarios/reliability.py b/evals/scenarios/reliability.py new file mode 100644 index 0000000..8e6cab3 --- /dev/null +++ b/evals/scenarios/reliability.py @@ -0,0 +1,60 @@ +"""CortexBench scenario definitions (V2 P0 baseline).""" + +from __future__ import annotations + +from typing import Any + +SCENARIOS: list[dict[str, Any]] = [ + { + "id": "signature_payments_restart", + "name": "Payments restart during migration", + "expected_decision": "BLOCK", + "risk": "HIGH_IMPACT_WRITE", + "roles": ["authenticated"], + "query": "Should I restart the payments service?", + "candidate_action": { + "type": "restart_service", + "target": "payments-service", + "stale_procedure": True, + "active_migration": True, + }, + "claims": [ + {"confidence": 0.95, "status": "ACTIVE", "assertion_type": "ASSERTED", "conflict": True}, + {"confidence": 0.9, "status": "SUPERSEDED", "assertion_type": "ASSERTED"}, + ], + }, + { + "id": "insufficient_evidence", + "name": "Missing knowledge abstention", + "expected_decision": "ASK", + "risk": "READ_ONLY", + "roles": ["admin"], + "query": "What is the rollback for unknown-service?", + "candidate_action": {}, + "claims": [], + }, + { + "id": "read_only_ok", + "name": "High-conf read should ACT", + "expected_decision": "ACT", + "risk": "READ_ONLY", + "roles": ["authenticated"], + "query": "Why CockroachDB?", + "candidate_action": {"type": "read"}, + "claims": [ + {"confidence": 0.92, "status": "ACTIVE", "assertion_type": "ASSERTED"}, + ], + }, + { + "id": "stale_procedure_verify", + "name": "Stale procedure triggers VERIFY/ESCALATE", + "expected_decision": ["VERIFY", "ESCALATE", "BLOCK"], + "risk": "REVERSIBLE_WRITE", + "roles": ["engineer"], + "query": "Run deploy procedure", + "candidate_action": {"type": "deploy", "stale_procedure": True}, + "claims": [ + {"confidence": 0.75, "status": "ACTIVE", "assertion_type": "ASSERTED"}, + ], + }, +] diff --git a/tests/evals/test_cortexbench.py b/tests/evals/test_cortexbench.py new file mode 100644 index 0000000..357515a --- /dev/null +++ b/tests/evals/test_cortexbench.py @@ -0,0 +1,11 @@ +"""CortexBench reliability baseline tests.""" + +from __future__ import annotations + +from evals.run_reliability import run_scenarios + + +def test_cortexbench_reliability_baseline_passes() -> None: + report = run_scenarios() + assert report["failed"] == 0, report + assert report["passed"] == report["total"]