diff --git a/test/battery.test.ts b/test/battery.test.ts index 063a5cb..44cb653 100644 --- a/test/battery.test.ts +++ b/test/battery.test.ts @@ -1,6 +1,11 @@ import assert from "node:assert/strict"; import { join } from "node:path"; import test from "node:test"; +import { + parseProofRun, + PROOF_ARTIFACT_VERSION, + scoreProofRun, +} from "@creative-int/evals/proof"; import { createWebsTransport, type WebsTransport } from "../extensions/webs.ts"; import { createDeterministicWebsTransport, @@ -16,11 +21,24 @@ test("emits a passing canonical ProofRun for the deterministic MCP journey", asy }); assert.ok(result); + const proof = parseProofRun(JSON.parse(JSON.stringify(result.proof))); + assert.deepEqual(proof, result.proof); assert.equal(result.pass, true); assert.equal(result.gate.pass, true); assert.equal(result.score.score, 1); assert.deepEqual(result.score.blockers, []); - assert.equal(result.proof.artifactVersion, "creative-int.proof-run.v1"); + // Keep the released v1 baseline compatible with the intentional v2 upgrade. + assert.ok( + ["creative-int.proof-run.v1", "creative-int.proof-run.v2"].includes( + PROOF_ARTIFACT_VERSION, + ), + ); + assert.equal(proof.artifactVersion, PROOF_ARTIFACT_VERSION); + assert.equal(proof.proofClass, "deterministic-proof"); + assert.equal( + proof.intent.userGoal, + "Prove the public agent-facing Webs MCP memory journey without skip-into-green.", + ); assert.equal(result.proof.packageVersions["@creative-int/evals"], "1.8.0"); assert.equal(result.proof.intent.agentRoute, "source-ingest"); assert.equal(result.proof.evidence.artifacts.length, 5); @@ -40,6 +58,59 @@ test("emits a passing canonical ProofRun for the deterministic MCP journey", asy ); }); +test("rejects malformed serialized battery proof through the canonical parser", async () => { + const [result] = await runWebsBattery({ + transport: createDeterministicWebsTransport(), + }); + assert.ok(result); + const proof = result.proof; + const malformed = [ + [ + { ...proof, artifactVersion: "creative-int.proof-run.v999" }, + /artifactVersion/, + ], + [{ ...proof, intent: undefined }, /intent/], + [ + { ...proof, intent: { ...proof.intent, userGoal: undefined } }, + /userGoal/, + ], + [ + { ...proof, intent: { ...proof.intent, agentRoute: "unknown" } }, + /agentRoute/, + ], + [{ ...proof, evidence: { artifacts: [{ kind: "unknown" }] } }, /kind/], + ] as const; + for (const [input, error] of malformed) { + assert.throws( + () => parseProofRun(JSON.parse(JSON.stringify(input))), + error, + ); + } +}); + +test("legacy v1 saved proof retains failed assertions and blockers when parsed", () => { + const legacy = { + artifactVersion: "creative-int.proof-run.v1", + intent: { + userGoal: "Verify a saved Webs MCP journey", + agentRoute: "source-ingest", + }, + eval: { + assertions: [{ name: "MCP response", status: "fail" }], + blockers: ["MCP response unavailable"], + }, + }; + const proof = parseProofRun(JSON.parse(JSON.stringify(legacy))); + assert.equal(proof.artifactVersion, PROOF_ARTIFACT_VERSION); + assert.equal(proof.intent.userGoal, legacy.intent.userGoal); + assert.equal(proof.intent.agentRoute, legacy.intent.agentRoute); + assert.deepEqual(proof.eval.assertions, legacy.eval.assertions); + assert.deepEqual(proof.eval.blockers, legacy.eval.blockers); + const score = scoreProofRun(proof); + assert.ok(score.score < 1); + assert.ok(score.blockers.includes("MCP response unavailable")); +}); + test("rejects zero-case battery runs instead of skipping into green", async () => { await assert.rejects( runWebsBattery({