diff --git a/vinci/test/402-escalation-no-downgrade.mjs b/vinci/test/402-escalation-no-downgrade.mjs index 25e227be0..e5d2aa8e8 100644 --- a/vinci/test/402-escalation-no-downgrade.mjs +++ b/vinci/test/402-escalation-no-downgrade.mjs @@ -1,37 +1,180 @@ import assert from "node:assert/strict"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { + createAssistantMessageEventStream, + fauxAssistantMessage, + registerApiProvider, + unregisterApiProviders, +} from "@earendil-works/pi-ai/compat"; import { createJiti } from "jiti/static"; const here = dirname(fileURLToPath(import.meta.url)); const loader = createJiti(import.meta.url, { moduleCache: false, tryNative: false }); -const provenance = await loader.import(resolve(here, "../extensions/vinci-model-provenance.ts"), { default: false }); +const advisorExtension = await loader.import(resolve(here, "../extensions/vinci-advisor.ts"), { default: false }); +const { runCouncil } = await loader.import(resolve(here, "../extensions/vinci-council.ts"), { default: false }); +const { judgeScope } = await loader.import(resolve(here, "../extensions/vinci-scope.ts"), { default: false }); +const { getUnstuck } = await loader.import(resolve(here, "../extensions/vinci-loopbreak.ts"), { default: false }); -const { classifyVinciModelError } = provenance; +const BILLING_ERRORS = [ + ["in-flight", "in_flight_budget_exhausted"], + ["affordability", "You requested up to 131072 tokens, but can only afford 23014"], +]; -const IN_FLIGHT_402 = Object.assign(new Error("in_flight_budget_exhausted"), { status: 402 }); -const AFFORDABILITY_402 = Object.assign(new Error("You requested up to 131072 tokens, but can only afford 23014"), { status: 402 }); +function model(api, id) { + return { + id, + name: `Vinci ${id}`, + api, + provider: "vinci", + baseUrl: "http://localhost:0", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 16384, + }; +} -// ASSERTION 1 & 2: Classifier must return "account" for marked 402s -const inFlightKind = classifyVinciModelError(IN_FLIGHT_402); -assert.equal(inFlightKind, "account", "in-flight 402 must classify as account"); +function successStream(text, requestModel) { + const stream = createAssistantMessageEventStream(); + const message = { + ...fauxAssistantMessage(text), + api: requestModel.api, + provider: requestModel.provider, + model: requestModel.id, + }; + queueMicrotask(() => stream.push({ type: "done", reason: "stop", message })); + return stream; +} -const affordabilityKind = classifyVinciModelError(AFFORDABILITY_402); -assert.equal(affordabilityKind, "account", "affordability 402 must classify as account"); +async function assert402StopsEscalation(site, body, run, leadingResponses = []) { + const api = `test:402-${site}`; + const sourceId = `402-escalation-${site}`; + const calls = []; + const notices = []; + const terminalError = Object.assign(new Error(body), { status: 402 }); + const steps = [ + ...leadingResponses, + { error: terminalError }, + { error: Object.assign(new Error(body), { status: 402 }) }, + { text: "CHEAPER FALLBACK MUST NOT RUN" }, + ]; + const models = { + forte: model(api, "forte"), + fortissimo: model(api, "fortissimo"), + }; + const stream = (requestModel) => { + calls.push(requestModel.id); + const step = steps.shift(); + assert.ok(step, `${site}: no unplanned model call may occur`); + if (step.error) throw step.error; + return successStream(step.text, requestModel); + }; + registerApiProvider({ api, stream, streamSimple: stream }, sourceId); -// ASSERTION 3: Escalation site logic - no downgrade on marked 402 -const unavailableClasses = []; -const kind = inFlightKind; -const attempt = 0; -const SAME_CLASS_ATTEMPTS = 2; + const registry = { + find(provider, id) { + return provider === "vinci" ? models[id] : undefined; + }, + async getApiKeyAndHeaders() { + return { ok: true, apiKey: "test-key" }; + }, + }; + const ctx = { + cwd: process.cwd(), + hasUI: true, + model: models.forte, + modelRegistry: registry, + sessionManager: { getSessionId: () => `402-${site}` }, + ui: { notify: (message, level) => notices.push({ message, level }) }, + }; + usageSessionStart({}, ctx); -if (kind === "transient" && attempt < SAME_CLASS_ATTEMPTS) { - throw new Error("DEFECT: would retry same class"); + try { + await assert.rejects( + () => run({ ctx, models, registry, announce: (message, level) => notices.push({ message, level }) }), + (error) => { + assert.match(error.message, /will not downgrade after an account or terminal error/i); + assert.equal(error.cause?.status, 402, `${site}: the terminal error must retain the real HTTP status`); + assert.strictEqual(error.cause, terminalError, `${site}: the terminal wrapper must preserve the provider error`); + return true; + }, + `${site}: a 402 must throw instead of returning a cheaper-class result`, + ); + assert.deepEqual( + calls, + [...leadingResponses.map(() => "forte"), "fortissimo"], + `${site}: no retry or cheaper class may be attempted after a 402`, + ); + assert.equal(steps.length, 2, `${site}: the same-class retry and cheaper fallback must remain unused`); + assert.equal( + notices.some(({ level, message }) => level === "warning" || /unavailable|continuing with/i.test(message)), + false, + `${site}: a 402 must not enter the unavailable-class downgrade path`, + ); + } finally { + unregisterApiProviders(sourceId); + } } -if (kind === "transient" || kind === "unavailable") { - unavailableClasses.push("cheaper"); - throw new Error("DEFECT: would downgrade"); + +let advisorTool; +let usageSessionStart; +advisorExtension.default({ + appendEntry() {}, + on(event, handler) { + if (event === "session_start") usageSessionStart = handler; + }, + registerTool(tool) { + if (tool.name === "advisor") advisorTool = tool; + }, +}); +assert.ok(advisorTool, "advisor tool must register"); +assert.ok(usageSessionStart, "usage accumulator must bind to the active test task"); + +const consumers = [ + [ + "advisor", + ({ ctx }) => advisorTool.execute("402-advisor", { question: "Review this" }, undefined, undefined, ctx), + [], + ], + [ + "council", + ({ ctx, registry, announce }) => + runCouncil( + ctx.model, + { apiKey: "test-key" }, + "Choose an approach", + registry, + ctx.sessionManager.getSessionId(), + announce, + ), + [ + { text: "optimist take" }, + { text: "skeptic take" }, + { text: "realist take" }, + { text: "strategist take" }, + ], + ], + ["scope", ({ ctx }) => judgeScope(ctx, "unrelated.ts"), []], + ["loopbreak", ({ ctx }) => getUnstuck(ctx, "bash repeated", [], "Fix the loop"), []], +]; + +const requestedConsumer = process.env.VINCI_TEST_402_CONSUMER?.trim(); +const selectedConsumers = requestedConsumer + ? consumers.filter(([site]) => site === requestedConsumer) + : consumers; +assert.ok( + selectedConsumers.length > 0, + `VINCI_TEST_402_CONSUMER must name one of: ${consumers.map(([site]) => site).join(", ")}`, +); + +for (const [variant, body] of BILLING_ERRORS) { + for (const [site, run, leadingResponses] of selectedConsumers) { + await assert402StopsEscalation(`${site}-${variant}`, body, run, leadingResponses); + } } -assert.deepEqual(unavailableClasses, [], "no downgrade attempted"); -console.log("402-escalation-no-downgrade: all assertions passed"); +const consumerSummary = requestedConsumer ?? "all four consumers"; +const consumerVerb = requestedConsumer ? "rejects" : "reject"; +console.log(`402-escalation-no-downgrade: ${consumerSummary} ${consumerVerb} both 402 variants without downgrade`); diff --git a/vinci/test/run.sh b/vinci/test/run.sh index 0be60aa8f..a78a554a4 100644 --- a/vinci/test/run.sh +++ b/vinci/test/run.sh @@ -273,6 +273,7 @@ run_group ask-checklist-integration node "${ROOT}/vinci/test/ask-checklist-integ run_group issue-integration node "${ROOT}/vinci/test/issue-integration.mjs" run_group billing-codes-integration node "${ROOT}/vinci/test/billing-codes-integration.mjs" run_group 402-classification-integration node "${ROOT}/vinci/test/402-classification-integration.mjs" +run_group 402-escalation-no-downgrade node "${ROOT}/vinci/test/402-escalation-no-downgrade.mjs" run_group no-downgrade-integration node "${ROOT}/vinci/test/no-downgrade-integration.mjs" run_group units node "${ROOT}/vinci/test/units.mjs" # Worker daemon: every vinci/test/worker-*.mjs runs, discovered by glob so a new file cannot be