From 790a3ddbd1b653c8999f9c5b66f36be018fa07e6 Mon Sep 17 00:00:00 2001 From: George Pu <19347973+thegeorgepu@users.noreply.github.com> Date: Fri, 4 Sep 2026 14:25:57 -0400 Subject: [PATCH 1/9] feat(coding-agent): add qualified Qwen H200 worker lane --- packages/coding-agent/CHANGELOG.md | 1 + vinci/extensions/lib/qwen-runtime.ts | 748 +++++++++++++++++++++++++++ vinci/extensions/vinci-provider.ts | 153 +++++- vinci/test/worker-qwen-provider.mjs | 330 ++++++++++++ vinci/worker/README.md | 119 ++++- vinci/worker/cleanroom.mjs | 34 +- vinci/worker/economics.mjs | 10 +- vinci/worker/run.mjs | 41 ++ vinci/worker/task.mjs | 1 + 9 files changed, 1431 insertions(+), 6 deletions(-) create mode 100644 vinci/extensions/lib/qwen-runtime.ts create mode 100644 vinci/test/worker-qwen-provider.mjs diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index d35d99def..62a2f9e38 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -13,6 +13,7 @@ - Added public SDK exports for CLI-equivalent model and scoped-model resolution ([#6201](https://github.com/earendil-works/pi/issues/6201)). - Added extension entry renderers for persisted display-only session entries that are rendered in interactive mode without being sent to the model context. - Added optional terminal and process-handler injection to `InteractiveMode` for deterministic embedded and UI-test environments. +- Added a fail-closed, qualified Vinci Worker lane for the exact non-authoritative `Qwen/Qwen3.8-27B` H200 endpoint. ### Changed diff --git a/vinci/extensions/lib/qwen-runtime.ts b/vinci/extensions/lib/qwen-runtime.ts new file mode 100644 index 000000000..cb1836db6 --- /dev/null +++ b/vinci/extensions/lib/qwen-runtime.ts @@ -0,0 +1,748 @@ +import { createHash } from "node:crypto"; +import { + existsSync, + lstatSync, + mkdirSync, + readFileSync, + renameSync, + writeFileSync, +} from "node:fs"; +import { dirname, isAbsolute } from "node:path"; +import { pathToFileURL } from "node:url"; + +export const QWEN_PROVIDER = "qwen-h200"; +export const QWEN_MODEL = "Qwen/Qwen3.8-27B"; +export const QWEN_API = "vinci-qwen-openai-completions"; + +const QUALIFICATION_SCHEMA = "vinci.qwen-worker-qualification.v1"; +const CIRCUIT_SCHEMA = "vinci.qwen-worker-circuit.v1"; +const MAX_RESPONSE_BYTES = 256 * 1024; +const FALLBACK_POLICY = "explicit-openrouter-separate-attempt-only"; +const AUTHORITY_ROLE = "non-authoritative-evidence-and-proposals-only"; +const HEX64 = /^[0-9a-f]{64}$/; +const IMMUTABLE_REVISION = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/; +const ENV_NAME = /^[A-Z][A-Z0-9_]{0,127}$/; + +type RuntimeTuple = { + engine: string; + version: string; + artifact_sha256: string; + arguments_sha256: string; +}; + +type Qualification = { + schema: string; + status: string; + authority_role: string; + fallback_policy: string; + model: string; + revision: string; + runtime: RuntimeTuple; + endpoint_sha256: string; + prompt_sha256: string; + tools_sha256: string; + capabilities: { + streaming_sse: boolean; + tool_calls: boolean; + structured_output: string; + }; + limits: { + timeout_ms: number; + max_retries: number; + max_retry_delay_ms: number; + max_concurrency: number; + context_window: number; + max_tokens: number; + }; + pricing: { + input_per_million_usd: number; + output_per_million_usd: number; + cache_read_per_million_usd: number; + cache_write_per_million_usd: number; + }; +}; + +export type QwenRuntimeConfig = { + baseUrl: string; + healthUrl: string; + modelsUrl: string; + chatUrl: string; + secret: string; + secretRef: string; + qualification: Qualification; + qualificationSha256: string; + circuitFile: string; + circuitThreshold: number; + circuitOpenMs: number; + attribution: { + workOrderId: string; + runId: string; + attemptId: string; + }; +}; + +type CircuitState = { + schema: string; + failures: number; + open_until_ms: number; + last_reason: string | null; +}; + +export class QwenReadinessError extends Error { + code: string; + + constructor(code: string, message: string) { + super(`qwen_${code}: ${message}`); + this.name = "QwenReadinessError"; + this.code = code; + } +} + +function fail(code: string, message: string): never { + throw new QwenReadinessError(code, message); +} + +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function canonical(value: unknown): string { + if (value === null) return "null"; + if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; + if (typeof value === "object") { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonical(record[key])}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +function exactKeys(value: unknown, expected: string[], label: string): asserts value is Record { + if (!value || typeof value !== "object" || Array.isArray(value)) fail("qualification_invalid", `${label} must be an object`); + const actual = Object.keys(value).sort(); + const wanted = [...expected].sort(); + if (canonical(actual) !== canonical(wanted)) fail("qualification_invalid", `${label} has unexpected or missing fields`); +} + +function boundedInteger(value: unknown, minimum: number, maximum: number, label: string): number { + if (!Number.isSafeInteger(value) || (value as number) < minimum || (value as number) > maximum) { + fail("qualification_invalid", `${label} must be an integer in [${minimum}, ${maximum}]`); + } + return value as number; +} + +function nonNegativeNumber(value: unknown, label: string): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + fail("qualification_invalid", `${label} must be a finite non-negative number`); + } + return value; +} + +export function normalizeQwenBaseUrl(raw: string | undefined): { + baseUrl: string; + healthUrl: string; + modelsUrl: string; + chatUrl: string; +} { + if (!raw) fail("config_missing", "VINCI_QWEN_BASE_URL is required"); + let parsed: URL; + try { + parsed = new URL(raw); + } catch { + fail("config_invalid", "VINCI_QWEN_BASE_URL must be an absolute URL"); + } + if (parsed.username || parsed.password || parsed.search || parsed.hash) { + fail("config_invalid", "VINCI_QWEN_BASE_URL may not contain credentials, a query, or a fragment"); + } + const loopback = parsed.hostname === "127.0.0.1" || parsed.hostname === "localhost" || parsed.hostname === "[::1]"; + if (parsed.protocol !== "https:" && !(parsed.protocol === "http:" && loopback)) { + fail("config_invalid", "VINCI_QWEN_BASE_URL must use HTTPS (HTTP is allowed only on loopback)"); + } + const withoutSlashes = parsed.toString().replace(/\/+$/, ""); + const root = withoutSlashes.endsWith("/v1") ? withoutSlashes.slice(0, -3) : withoutSlashes; + return { + baseUrl: `${root}/v1`, + healthUrl: `${root}/health`, + modelsUrl: `${root}/v1/models`, + chatUrl: `${root}/v1/chat/completions`, + }; +} + +function readSecretReference(reference: string | undefined, env: NodeJS.ProcessEnv): string { + if (!reference) fail("config_missing", "VINCI_QWEN_SECRET_REF is required"); + let secret: string; + if (reference.startsWith("env:")) { + const name = reference.slice(4); + if (!ENV_NAME.test(name)) fail("config_invalid", "VINCI_QWEN_SECRET_REF env name is invalid"); + secret = env[name] ?? ""; + // Keep the resolved value only in the provider closure. Repository tools inherit this process + // environment, so leaving a dynamically named credential here would bypass the static key + // inventory even though the reference itself is scrubbed later. + delete env[name]; + } else if (reference.startsWith("file:")) { + const path = reference.slice(5); + if (!isAbsolute(path)) fail("config_invalid", "VINCI_QWEN_SECRET_REF file path must be absolute"); + let stat; + try { + stat = lstatSync(path); + } catch { + fail("credential_unavailable", "the referenced Qwen credential file is unavailable"); + } + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || (stat.mode & 0o077) !== 0) { + fail("credential_unsafe", "the referenced Qwen credential must be a private regular file"); + } + secret = readFileSync(path, "utf8").trim(); + } else { + fail("config_invalid", "VINCI_QWEN_SECRET_REF must use env:NAME or file:/absolute/path"); + } + if (!secret || secret.length > 16_384 || /\s/.test(secret)) { + fail("credential_invalid", "the referenced Qwen credential is empty, oversized, or contains whitespace"); + } + return secret; +} + +function validateRuntime(value: unknown): RuntimeTuple { + exactKeys(value, ["engine", "version", "artifact_sha256", "arguments_sha256"], "runtime"); + for (const key of ["engine", "version"] as const) { + if (typeof value[key] !== "string" || !value[key]) fail("qualification_invalid", `runtime.${key} must be a non-empty string`); + } + for (const key of ["artifact_sha256", "arguments_sha256"] as const) { + if (typeof value[key] !== "string" || !HEX64.test(value[key])) fail("qualification_invalid", `runtime.${key} must be lowercase SHA-256`); + } + return value as RuntimeTuple; +} + +function validateQualification(raw: unknown): Qualification { + exactKeys( + raw, + [ + "schema", + "status", + "authority_role", + "fallback_policy", + "model", + "revision", + "runtime", + "endpoint_sha256", + "prompt_sha256", + "tools_sha256", + "capabilities", + "limits", + "pricing", + ], + "qualification", + ); + if (raw.schema !== QUALIFICATION_SCHEMA || raw.status !== "qualified") fail("qualification_invalid", "qualification is not an admitted v1 qualified record"); + if (raw.authority_role !== AUTHORITY_ROLE) fail("authority_forbidden", "Qwen must remain non-authoritative"); + if (raw.fallback_policy !== FALLBACK_POLICY) fail("fallback_forbidden", "fallback must be a separately authorized OpenRouter attempt"); + if (raw.model !== QWEN_MODEL) fail("model_mismatch", `qualification must name ${QWEN_MODEL}`); + if (typeof raw.revision !== "string" || !IMMUTABLE_REVISION.test(raw.revision)) { + fail("qualification_invalid", "revision must be an immutable lowercase 40- or 64-hex commit/digest"); + } + const runtime = validateRuntime(raw.runtime); + for (const key of ["endpoint_sha256", "prompt_sha256", "tools_sha256"] as const) { + if (typeof raw[key] !== "string" || !HEX64.test(raw[key])) fail("qualification_invalid", `${key} must be lowercase SHA-256`); + } + + exactKeys(raw.capabilities, ["streaming_sse", "tool_calls", "structured_output"], "capabilities"); + if (raw.capabilities.streaming_sse !== true || raw.capabilities.tool_calls !== true) { + fail("capability_missing", "streaming SSE and structured tool calls must both be qualified"); + } + if (raw.capabilities.structured_output !== "tool-arguments-json") { + fail("capability_missing", "the worker-required structured output is tool-arguments JSON"); + } + + exactKeys(raw.limits, ["timeout_ms", "max_retries", "max_retry_delay_ms", "max_concurrency", "context_window", "max_tokens"], "limits"); + const limits = { + timeout_ms: boundedInteger(raw.limits.timeout_ms, 1_000, 300_000, "limits.timeout_ms"), + max_retries: boundedInteger(raw.limits.max_retries, 0, 2, "limits.max_retries"), + max_retry_delay_ms: boundedInteger(raw.limits.max_retry_delay_ms, 0, 30_000, "limits.max_retry_delay_ms"), + max_concurrency: boundedInteger(raw.limits.max_concurrency, 1, 8, "limits.max_concurrency"), + context_window: boundedInteger(raw.limits.context_window, 8_192, 2_000_000, "limits.context_window"), + max_tokens: boundedInteger(raw.limits.max_tokens, 256, 131_072, "limits.max_tokens"), + }; + if (limits.max_tokens > limits.context_window) fail("qualification_invalid", "limits.max_tokens exceeds limits.context_window"); + + exactKeys(raw.pricing, ["input_per_million_usd", "output_per_million_usd", "cache_read_per_million_usd", "cache_write_per_million_usd"], "pricing"); + const pricing = { + input_per_million_usd: nonNegativeNumber(raw.pricing.input_per_million_usd, "pricing.input_per_million_usd"), + output_per_million_usd: nonNegativeNumber(raw.pricing.output_per_million_usd, "pricing.output_per_million_usd"), + cache_read_per_million_usd: nonNegativeNumber(raw.pricing.cache_read_per_million_usd, "pricing.cache_read_per_million_usd"), + cache_write_per_million_usd: nonNegativeNumber(raw.pricing.cache_write_per_million_usd, "pricing.cache_write_per_million_usd"), + }; + + return { ...(raw as unknown as Qualification), runtime, limits, pricing }; +} + +function readQualification(env: NodeJS.ProcessEnv): { qualification: Qualification; digest: string } { + const path = env.VINCI_QWEN_QUALIFICATION_FILE; + const expectedDigest = env.VINCI_QWEN_QUALIFICATION_SHA256; + if (!path || !isAbsolute(path)) fail("config_missing", "VINCI_QWEN_QUALIFICATION_FILE must be an absolute path"); + if (!expectedDigest || !HEX64.test(expectedDigest)) fail("config_missing", "VINCI_QWEN_QUALIFICATION_SHA256 must pin the qualification bytes"); + let stat; + let bytes: Buffer; + try { + stat = lstatSync(path); + bytes = readFileSync(path); + } catch { + fail("qualification_unavailable", "the pinned qualification artifact is unavailable"); + } + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || (stat.mode & 0o022) !== 0) { + fail("qualification_unsafe", "the qualification artifact must be a non-writable regular file"); + } + if (sha256(bytes) !== expectedDigest) fail("qualification_digest_mismatch", "qualification bytes do not match the process pin"); + let parsed: unknown; + try { + parsed = JSON.parse(bytes.toString("utf8")); + } catch { + fail("qualification_invalid", "qualification is not JSON"); + } + return { qualification: validateQualification(parsed), digest: expectedDigest }; +} + +export function loadQwenRuntimeConfig(env: NodeJS.ProcessEnv = process.env): QwenRuntimeConfig { + const urls = normalizeQwenBaseUrl(env.VINCI_QWEN_BASE_URL); + const secretRef = env.VINCI_QWEN_SECRET_REF; + const secret = readSecretReference(secretRef, env); + const admitted = readQualification(env); + const qualification = admitted.qualification; + const expectedEndpoint = sha256(urls.baseUrl); + if (qualification.endpoint_sha256 !== expectedEndpoint) fail("endpoint_mismatch", "qualification is bound to a different base URL"); + if (qualification.prompt_sha256 !== env.VINCI_QWEN_PROMPT_SHA256) fail("prompt_mismatch", "task prompt is not the qualified prompt"); + if (qualification.tools_sha256 !== env.VINCI_QWEN_TOOLS_SHA256) fail("tools_mismatch", "task tools are not the qualified tools"); + if (env.VINCI_UNATTENDED_POLICY !== "governed" || !env.VINCI_UNATTENDED_LEASE) { + fail("authority_forbidden", "Qwen worker runs require a deterministic Governor lease"); + } + const workOrderId = env.VINCI_QWEN_WORK_ORDER_ID; + const runId = env.VINCI_QWEN_RUN_ID; + const attemptId = env.VINCI_QWEN_ATTEMPT_ID; + if (!workOrderId || !runId || !attemptId) fail("attribution_missing", "WorkOrder, Run, and Attempt attribution are required"); + const circuitFile = env.VINCI_QWEN_CIRCUIT_FILE; + if (!circuitFile || !isAbsolute(circuitFile)) fail("config_missing", "VINCI_QWEN_CIRCUIT_FILE must be an absolute path"); + return { + ...urls, + secret, + secretRef: secretRef as string, + qualification, + qualificationSha256: admitted.digest, + circuitFile, + circuitThreshold: boundedInteger(Number(env.VINCI_QWEN_CIRCUIT_THRESHOLD ?? "3"), 1, 10, "VINCI_QWEN_CIRCUIT_THRESHOLD"), + circuitOpenMs: boundedInteger(Number(env.VINCI_QWEN_CIRCUIT_OPEN_MS ?? "60000"), 1_000, 3_600_000, "VINCI_QWEN_CIRCUIT_OPEN_MS"), + attribution: { workOrderId, runId, attemptId }, + }; +} + +function emptyCircuit(): CircuitState { + return { schema: CIRCUIT_SCHEMA, failures: 0, open_until_ms: 0, last_reason: null }; +} + +function readCircuit(path: string): CircuitState { + if (!existsSync(path)) return emptyCircuit(); + try { + const stat = lstatSync(path); + const value = JSON.parse(readFileSync(path, "utf8")) as CircuitState; + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || value.schema !== CIRCUIT_SCHEMA || !Number.isSafeInteger(value.failures) || value.failures < 0 || !Number.isSafeInteger(value.open_until_ms) || value.open_until_ms < 0) { + fail("circuit_invalid", "circuit state is malformed or unsafe"); + } + return value; + } catch (error) { + if (error instanceof QwenReadinessError) throw error; + fail("circuit_invalid", "circuit state is unreadable"); + } +} + +function writeCircuit(path: string, state: CircuitState): void { + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const temporary = `${path}.tmp-${process.pid}`; + writeFileSync(temporary, `${canonical(state)}\n`, { mode: 0o600 }); + renameSync(temporary, path); +} + +export function assertQwenCircuitClosed(config: QwenRuntimeConfig, nowMs = Date.now()): void { + const state = readCircuit(config.circuitFile); + if (state.open_until_ms > nowMs) fail("circuit_open", `endpoint circuit is open until ${new Date(state.open_until_ms).toISOString()}`); +} + +export function recordQwenCircuitOutcome(config: QwenRuntimeConfig, ok: boolean, reason: string, nowMs = Date.now()): void { + if (ok) { + writeCircuit(config.circuitFile, emptyCircuit()); + return; + } + const current = readCircuit(config.circuitFile); + const failures = current.failures + 1; + writeCircuit(config.circuitFile, { + schema: CIRCUIT_SCHEMA, + failures, + open_until_ms: failures >= config.circuitThreshold ? nowMs + config.circuitOpenMs : 0, + last_reason: reason.slice(0, 128), + }); +} + +async function readBoundedText(response: Response): Promise { + if (!response.body) return ""; + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + while (true) { + const next = await reader.read(); + if (next.done) break; + total += next.value.length; + if (total > MAX_RESPONSE_BYTES) fail("response_oversized", "endpoint response exceeded 256 KiB"); + chunks.push(next.value); + } + const bytes = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.length; + } + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); +} + +async function request( + url: string, + init: RequestInit, + timeoutMs: number, + retries: number, + fetchImpl: typeof fetch, + signal?: AbortSignal, +): Promise { + let lastError: unknown; + for (let attempt = 0; attempt <= retries; attempt += 1) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort("timeout"), timeoutMs); + const abort = () => controller.abort(signal?.reason ?? "cancelled"); + if (signal?.aborted) abort(); + else signal?.addEventListener("abort", abort, { once: true }); + try { + return await fetchImpl(url, { ...init, redirect: "error", signal: controller.signal }); + } catch (error) { + lastError = error; + if (signal?.aborted) fail("cancelled", "readiness probe was cancelled"); + if (attempt === retries) break; + } finally { + clearTimeout(timeout); + signal?.removeEventListener("abort", abort); + } + } + const suffix = lastError instanceof Error && lastError.name === "AbortError" ? "timed out" : "failed"; + fail("endpoint_unavailable", `request ${suffix} after ${retries + 1} bounded attempt(s)`); +} + +function authHeaders(config: Pick): Record { + return { + authorization: `Bearer ${config.secret}`, + accept: "application/json", + "x-vinci-work-order-id": config.attribution.workOrderId, + "x-vinci-run-id": config.attribution.runId, + "x-vinci-attempt-id": config.attribution.attemptId, + "x-vinci-qwen-output-authority": "non-authoritative", + }; +} + +function servedIdentity(response: Response, payload: unknown): { revision: string; runtime: RuntimeTuple } { + exactKeys(payload, ["object", "data"], "models response"); + if (payload.object !== "list" || !Array.isArray(payload.data)) fail("models_invalid", "/v1/models must return an OpenAI list"); + const matches = payload.data.filter((entry) => entry && typeof entry === "object" && (entry as Record).id === QWEN_MODEL); + if (matches.length !== 1) fail("model_mismatch", `/v1/models must expose exactly one ${QWEN_MODEL}`); + const model = matches[0] as Record; + const runtimeValue = model.runtime; + const runtime = runtimeValue && typeof runtimeValue === "object" + ? validateRuntime(runtimeValue) + : validateRuntime({ + engine: response.headers.get("x-vinci-runtime-engine"), + version: response.headers.get("x-vinci-runtime-version"), + artifact_sha256: response.headers.get("x-vinci-runtime-artifact-sha256"), + arguments_sha256: response.headers.get("x-vinci-runtime-arguments-sha256"), + }); + const revision = typeof model.revision === "string" ? model.revision : response.headers.get("x-vinci-model-revision"); + if (!revision || !IMMUTABLE_REVISION.test(revision)) fail("identity_missing", "/v1/models omitted the immutable served revision"); + return { revision, runtime }; +} + +async function validateHealthResponse(response: Response): Promise { + if (!response.ok) fail("health_failed", `authenticated /health returned ${response.status}`); + const healthText = await readBoundedText(response); + if (!healthText) fail("health_invalid", "/health returned an empty response"); + let healthBody: unknown; + try { + healthBody = JSON.parse(healthText); + } catch { + fail("health_invalid", "/health returned non-JSON content"); + } + const status = healthBody && typeof healthBody === "object" ? (healthBody as Record).status : undefined; + if (status !== "ok" && status !== "ready") fail("health_failed", "/health did not report ready"); +} + +export async function probeQwenReadiness( + config: QwenRuntimeConfig, + options: { fetchImpl?: typeof fetch; signal?: AbortSignal; nowMs?: number } = {}, +): Promise<{ revision: string; runtime: RuntimeTuple }> { + const fetchImpl = options.fetchImpl ?? fetch; + const nowMs = options.nowMs ?? Date.now(); + assertQwenCircuitClosed(config, nowMs); + const timeoutMs = config.qualification.limits.timeout_ms; + const retries = config.qualification.limits.max_retries; + try { + const health = await request(config.healthUrl, { headers: authHeaders(config) }, timeoutMs, retries, fetchImpl, options.signal); + await validateHealthResponse(health); + + const models = await request(config.modelsUrl, { headers: authHeaders(config) }, timeoutMs, retries, fetchImpl, options.signal); + if (!models.ok) fail("models_failed", `authenticated /v1/models returned ${models.status}`); + const modelsText = await readBoundedText(models); + let modelsBody: unknown; + try { + modelsBody = JSON.parse(modelsText); + } catch { + fail("models_invalid", "/v1/models returned non-JSON content"); + } + const identity = servedIdentity(models, modelsBody); + if (identity.revision !== config.qualification.revision || canonical(identity.runtime) !== canonical(config.qualification.runtime)) { + fail("runtime_mismatch", "served model revision/runtime differs from the qualification tuple"); + } + + for (const [url, path] of [[config.healthUrl, "/health"], [config.modelsUrl, "/v1/models"]] as const) { + const anonymous = await request(url, { headers: { accept: "application/json" } }, timeoutMs, 0, fetchImpl, options.signal); + await readBoundedText(anonymous); + if (anonymous.status !== 401 && anonymous.status !== 403) { + fail("auth_not_enforced", `unauthenticated ${path} was not refused`); + } + } + recordQwenCircuitOutcome(config, true, "ready", nowMs); + return identity; + } catch (error) { + if (!(error instanceof QwenReadinessError) || error.code !== "circuit_open") { + recordQwenCircuitOutcome(config, false, error instanceof QwenReadinessError ? error.code : "probe_failed", nowMs); + } + throw error; + } +} + +export async function ensureQwenReady( + env: NodeJS.ProcessEnv = process.env, + options: { fetchImpl?: typeof fetch; signal?: AbortSignal; nowMs?: number } = {}, +): Promise { + const config = loadQwenRuntimeConfig(env); + await probeQwenReadiness(config, options); + return config; +} + +function canaryEndpointConfig(env: NodeJS.ProcessEnv): Pick { + const urls = normalizeQwenBaseUrl(env.VINCI_QWEN_BASE_URL); + return { + ...urls, + secret: readSecretReference(env.VINCI_QWEN_SECRET_REF, env), + attribution: { + workOrderId: "canary-read-only", + runId: "canary-read-only", + attemptId: "canary-read-only/1", + }, + }; +} + +export async function runQwenCanary(env: NodeJS.ProcessEnv = process.env, fetchImpl: typeof fetch = fetch): Promise> { + const config = canaryEndpointConfig(env); + const timeoutMs = boundedInteger(Number(env.VINCI_QWEN_CANARY_TIMEOUT_MS ?? "30000"), 1_000, 300_000, "VINCI_QWEN_CANARY_TIMEOUT_MS"); + const started = Date.now(); + const health = await request(config.healthUrl, { headers: authHeaders(config) }, timeoutMs, 0, fetchImpl); + await validateHealthResponse(health); + const models = await request(config.modelsUrl, { headers: authHeaders(config) }, timeoutMs, 0, fetchImpl); + if (!models.ok) fail("models_failed", `authenticated /v1/models returned ${models.status}`); + const identity = servedIdentity(models, JSON.parse(await readBoundedText(models))); + for (const [url, path] of [[config.healthUrl, "/health"], [config.modelsUrl, "/v1/models"]] as const) { + const anonymous = await request(url, { headers: { accept: "application/json" } }, timeoutMs, 0, fetchImpl); + await readBoundedText(anonymous); + if (anonymous.status !== 401 && anonymous.status !== 403) fail("auth_not_enforced", `unauthenticated ${path} was not refused`); + } + + const response = await request( + config.chatUrl, + { + method: "POST", + headers: { ...authHeaders(config), "content-type": "application/json" }, + body: JSON.stringify({ + model: QWEN_MODEL, + stream: true, + stream_options: { include_usage: true }, + temperature: 0, + max_tokens: 64, + messages: [ + { role: "system", content: "Call report_ready exactly once. Do not return prose." }, + { role: "user", content: "Report readiness." }, + ], + tools: [{ + type: "function", + function: { + name: "report_ready", + description: "Reports deterministic worker compatibility.", + strict: false, + parameters: { + type: "object", + properties: { status: { type: "string", enum: ["ready"] } }, + required: ["status"], + additionalProperties: false, + }, + }, + }], + tool_choice: { type: "function", function: { name: "report_ready" } }, + }), + }, + timeoutMs, + 0, + fetchImpl, + ); + if (!response.ok) fail("canary_failed", `streaming tool-call inference returned ${response.status}`); + if (!response.headers.get("content-type")?.toLowerCase().startsWith("text/event-stream")) { + fail("canary_invalid", "streaming tool-call inference did not return text/event-stream"); + } + const stream = await readBoundedText(response); + let toolName = ""; + let argumentsText = ""; + let usageSeen = false; + for (const line of stream.split(/\r?\n/)) { + if (!line.startsWith("data:")) continue; + const data = line.slice(5).trim(); + if (!data || data === "[DONE]") continue; + let chunk: Record; + try { + chunk = JSON.parse(data) as Record; + } catch { + fail("canary_invalid", "stream contained malformed JSON"); + } + if (chunk.usage && typeof chunk.usage === "object") usageSeen = true; + const choices = Array.isArray(chunk.choices) ? chunk.choices : []; + for (const choice of choices) { + const delta = choice && typeof choice === "object" ? (choice as Record).delta : null; + const toolCalls = delta && typeof delta === "object" && Array.isArray((delta as Record).tool_calls) + ? ((delta as Record).tool_calls as unknown[]) + : []; + for (const call of toolCalls) { + const fn = call && typeof call === "object" ? (call as Record).function : null; + if (!fn || typeof fn !== "object") continue; + const name = (fn as Record).name; + const args = (fn as Record).arguments; + if (typeof name === "string") toolName += name; + if (typeof args === "string") argumentsText += args; + } + } + } + let argumentsValue: unknown; + try { + argumentsValue = JSON.parse(argumentsText); + } catch { + fail("canary_invalid", "tool-call arguments were not complete JSON"); + } + if (toolName !== "report_ready" || canonical(argumentsValue) !== canonical({ status: "ready" })) { + fail("canary_invalid", "stream did not return the required structured tool call"); + } + if (!usageSeen) fail("canary_invalid", "stream omitted the usage chunk required for token telemetry"); + return { + schema: "vinci.qwen-worker-canary.v1", + model: QWEN_MODEL, + revision: identity.revision, + runtime: identity.runtime, + authenticated: true, + anonymous_refused: true, + capabilities: { streaming_sse: true, tool_calls: true, structured_output: "tool-arguments-json", usage_chunk: usageSeen }, + latency_ms: Date.now() - started, + authority_role: AUTHORITY_ROLE, + fallback_policy: FALLBACK_POLICY, + }; +} + +function requiredEnv(env: NodeJS.ProcessEnv, name: string): string { + const value = env[name]; + if (!value) fail("config_missing", `${name} is required`); + return value; +} + +export function buildQwenQualificationTemplate(env: NodeJS.ProcessEnv = process.env): Qualification { + if (env.VINCI_QWEN_ADMIT !== "qualified") { + fail("qualification_not_admitted", "VINCI_QWEN_ADMIT=qualified is required after independent canary review"); + } + const urls = normalizeQwenBaseUrl(env.VINCI_QWEN_BASE_URL); + const promptFile = requiredEnv(env, "VINCI_QWEN_QUALIFICATION_PROMPT_FILE"); + if (!isAbsolute(promptFile)) fail("config_invalid", "VINCI_QWEN_QUALIFICATION_PROMPT_FILE must be absolute"); + let prompt: string; + try { + prompt = readFileSync(promptFile, "utf8"); + } catch { + fail("config_invalid", "qualification prompt file is unreadable"); + } + let tools: unknown; + try { + tools = JSON.parse(requiredEnv(env, "VINCI_QWEN_QUALIFICATION_TOOLS")); + } catch { + fail("config_invalid", "VINCI_QWEN_QUALIFICATION_TOOLS must be a JSON array"); + } + if (!Array.isArray(tools) || tools.length === 0 || !tools.every((tool) => typeof tool === "string" && tool.length > 0)) { + fail("config_invalid", "VINCI_QWEN_QUALIFICATION_TOOLS must be a non-empty array of tool names"); + } + const revision = requiredEnv(env, "VINCI_QWEN_SERVED_REVISION"); + if (!IMMUTABLE_REVISION.test(revision)) { + fail("config_invalid", "VINCI_QWEN_SERVED_REVISION must be an immutable lowercase 40- or 64-hex commit/digest"); + } + const runtime = validateRuntime({ + engine: requiredEnv(env, "VINCI_QWEN_RUNTIME_ENGINE"), + version: requiredEnv(env, "VINCI_QWEN_RUNTIME_VERSION"), + artifact_sha256: requiredEnv(env, "VINCI_QWEN_RUNTIME_ARTIFACT_SHA256"), + arguments_sha256: requiredEnv(env, "VINCI_QWEN_RUNTIME_ARGUMENTS_SHA256"), + }); + const numberEnv = (name: string) => Number(requiredEnv(env, name)); + return validateQualification({ + schema: QUALIFICATION_SCHEMA, + status: "qualified", + authority_role: AUTHORITY_ROLE, + fallback_policy: FALLBACK_POLICY, + model: QWEN_MODEL, + revision, + runtime, + endpoint_sha256: sha256(urls.baseUrl), + prompt_sha256: sha256(prompt), + tools_sha256: sha256(canonical(tools)), + capabilities: { + streaming_sse: true, + tool_calls: true, + structured_output: "tool-arguments-json", + }, + limits: { + timeout_ms: Number(env.VINCI_QWEN_TIMEOUT_MS ?? "120000"), + max_retries: Number(env.VINCI_QWEN_MAX_RETRIES ?? "1"), + max_retry_delay_ms: Number(env.VINCI_QWEN_MAX_RETRY_DELAY_MS ?? "5000"), + max_concurrency: Number(env.VINCI_QWEN_MAX_CONCURRENCY ?? "1"), + context_window: numberEnv("VINCI_QWEN_CONTEXT_WINDOW"), + max_tokens: numberEnv("VINCI_QWEN_MAX_TOKENS"), + }, + pricing: { + input_per_million_usd: numberEnv("VINCI_QWEN_INPUT_PER_MILLION_USD"), + output_per_million_usd: numberEnv("VINCI_QWEN_OUTPUT_PER_MILLION_USD"), + cache_read_per_million_usd: numberEnv("VINCI_QWEN_CACHE_READ_PER_MILLION_USD"), + cache_write_per_million_usd: numberEnv("VINCI_QWEN_CACHE_WRITE_PER_MILLION_USD"), + }, + }); +} + +export function scrubQwenBootstrapEnvironment(env: NodeJS.ProcessEnv = process.env): void { + for (const name of ["VINCI_QWEN_SECRET_REF", "VINCI_QWEN_QUALIFICATION_FILE", "VINCI_QWEN_QUALIFICATION_SHA256"]) delete env[name]; +} + +const invokedPath = process.argv[1] ? pathToFileURL(process.argv[1]).href : ""; +if (import.meta.url === invokedPath) { + if (process.argv.includes("--canary")) { + runQwenCanary() + .then((report) => process.stdout.write(`${canonical(report)}\n`)) + .catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + }); + } else if (process.argv.includes("--qualification-template")) { + try { + process.stdout.write(`${canonical(buildQwenQualificationTemplate())}\n`); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } + } +} diff --git a/vinci/extensions/vinci-provider.ts b/vinci/extensions/vinci-provider.ts index b6676bd33..9750e697e 100644 --- a/vinci/extensions/vinci-provider.ts +++ b/vinci/extensions/vinci-provider.ts @@ -1,5 +1,23 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { + type Api, + createAssistantMessageEventStream, + type Context, + type Model, + type SimpleStreamOptions, +} from "@earendil-works/pi-ai"; +import { streamSimple as streamOpenAICompletions } from "@earendil-works/pi-ai/api/openai-completions"; import type { OAuthCredentials, OAuthLoginCallbacks } from "@earendil-works/pi-ai/compat"; +import { + assertQwenCircuitClosed, + ensureQwenReady, + QWEN_API, + QWEN_MODEL, + QWEN_PROVIDER, + recordQwenCircuitOutcome, + scrubQwenBootstrapEnvironment, + type QwenRuntimeConfig, +} from "./lib/qwen-runtime.ts"; import { setVinciConnection } from "./lib/ui-state.ts"; import { VINCI_BILLING_URL, VINCI_GATEWAY_BASE_URL, VINCI_PLATFORM_BASE_URL } from "./vinci-links.ts"; @@ -134,6 +152,101 @@ function vinciClassModel(id: string, name: string) { }; } +function qwenProviderConfig(runtime: QwenRuntimeConfig) { + let inFlight = 0; + return { + name: "Qwen 3.8 27B (Vinci H200, non-authoritative)", + baseUrl: runtime.baseUrl, + // A non-secret sentinel satisfies provider registration. The resolved credential is held only + // in this closure and replaces this value at the actual OpenAI-compatible request boundary. + apiKey: "runtime-resolved-secret-reference", + api: QWEN_API, + streamSimple(model: Model, context: Context, options?: SimpleStreamOptions) { + assertQwenCircuitClosed(runtime); + if (inFlight >= runtime.qualification.limits.max_concurrency) { + throw new Error("qwen_concurrency_exceeded: the qualified single-request bound is already in use"); + } + inFlight += 1; + let source; + try { + source = streamOpenAICompletions( + { ...model, api: "openai-completions" } as Model<"openai-completions">, + context, + { + ...options, + apiKey: runtime.secret, + timeoutMs: runtime.qualification.limits.timeout_ms, + maxRetries: runtime.qualification.limits.max_retries, + maxRetryDelayMs: runtime.qualification.limits.max_retry_delay_ms, + }, + ); + } catch (error) { + inFlight -= 1; + throw error; + } + const bounded = createAssistantMessageEventStream(); + void (async () => { + try { + for await (const event of source) bounded.push(event); + } catch (error) { + recordQwenCircuitOutcome(runtime, false, "stream_error"); + const message = { + role: "assistant" as const, + content: [], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "error" as const, + errorMessage: error instanceof Error ? error.message : String(error), + timestamp: Date.now(), + }; + bounded.push({ type: "error", reason: "error", error: message }); + bounded.end(message); + } finally { + inFlight -= 1; + } + })(); + return bounded; + }, + models: [ + { + id: QWEN_MODEL, + name: "Qwen 3.8 27B (qualified, non-authoritative)", + reasoning: true, + thinkingLevelMap: { off: "off", minimal: "low", low: "low", medium: "medium", high: "high", xhigh: "high" }, + input: ["text"] as Array<"text">, + contextWindow: runtime.qualification.limits.context_window, + maxTokens: runtime.qualification.limits.max_tokens, + cost: { + input: runtime.qualification.pricing.input_per_million_usd, + output: runtime.qualification.pricing.output_per_million_usd, + cacheRead: runtime.qualification.pricing.cache_read_per_million_usd, + cacheWrite: runtime.qualification.pricing.cache_write_per_million_usd, + }, + compat: { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + supportsUsageInStreaming: true, + maxTokensField: "max_tokens" as const, + requiresToolResultName: true, + supportsStrictMode: false, + supportsLongCacheRetention: false, + thinkingFormat: "qwen-chat-template" as const, + }, + }, + ], + }; +} + /** * Compose a terminal message for a Vinci billing refusal. * Accepts either a full error body (for structured codes) or just the error message text (for backward compat). @@ -253,7 +366,7 @@ async function loginVinci(callbacks: OAuthLoginCallbacks): Promise { + if (ctx.model?.provider !== QWEN_PROVIDER || !qwenRuntime) return; + event.headers["x-vinci-work-order-id"] = qwenRuntime.attribution.workOrderId; + event.headers["x-vinci-run-id"] = qwenRuntime.attribution.runId; + event.headers["x-vinci-attempt-id"] = qwenRuntime.attribution.attemptId; + event.headers["x-vinci-qwen-output-authority"] = "non-authoritative"; + event.headers["x-vinci-qwen-qualification-sha256"] = qwenRuntime.qualificationSha256; + }); + + pi.on("after_provider_response", (event, ctx) => { + if (ctx.model?.provider !== QWEN_PROVIDER || !qwenRuntime) return; + recordQwenCircuitOutcome(qwenRuntime, event.status >= 200 && event.status < 300, `http_${event.status}`); + }); + + pi.on("message_end", (event, ctx) => { + if (event.message.role !== "assistant" || event.message.provider !== QWEN_PROVIDER || !qwenRuntime) return; + pi.appendEntry("vinci-qwen-output-label", { + authority: "non-authoritative", + independent_check_required: true, + model: QWEN_MODEL, + revision: qwenRuntime.qualification.revision, + runtime: qwenRuntime.qualification.runtime, + qualification_sha256: qwenRuntime.qualificationSha256, + work_order_id: qwenRuntime.attribution.workOrderId, + run_id: qwenRuntime.attribution.runId, + attempt_id: qwenRuntime.attribution.attemptId, + outcome: event.message.stopReason, + session_id: ctx.sessionManager.getSessionId(), + }); + }); + } + pi.on("after_provider_response", (event, ctx) => { if (ctx.model?.provider !== "vinci") return; if (event.status >= 200 && event.status < 300) { diff --git a/vinci/test/worker-qwen-provider.mjs b/vinci/test/worker-qwen-provider.mjs new file mode 100644 index 000000000..407be13c9 --- /dev/null +++ b/vinci/test/worker-qwen-provider.mjs @@ -0,0 +1,330 @@ +import assert from "node:assert/strict"; +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { createHash } from "node:crypto"; +import * as runtime from "../extensions/lib/qwen-runtime.ts"; +import providerExtension from "../extensions/vinci-provider.ts"; +import * as cleanroom from "../worker/cleanroom.mjs"; +import * as digest from "../worker/contracts/digest.mjs"; +import * as economics from "../worker/economics.mjs"; +import * as workerRun from "../worker/run.mjs"; + +const root = resolve(import.meta.dirname, "../.."); + +const temp = mkdtempSync(join(tmpdir(), "vinci-qwen-test-")); +const secretFile = join(temp, "secret"); +const promptFile = join(temp, "prompt.txt"); +const qualificationFile = join(temp, "qualification.json"); +writeFileSync(secretFile, "test-secret\n", { mode: 0o600 }); +writeFileSync(promptFile, "inspect the bounded fixture\n", { mode: 0o600 }); + +const hex = (pair) => pair.repeat(32); +const sha256 = (value) => createHash("sha256").update(value).digest("hex"); +const baseEnv = { + VINCI_QWEN_ADMIT: "qualified", + VINCI_QWEN_BASE_URL: "https://qwen.example.test/v1", + VINCI_QWEN_SECRET_REF: `file:${secretFile}`, + VINCI_QWEN_QUALIFICATION_PROMPT_FILE: promptFile, + VINCI_QWEN_QUALIFICATION_TOOLS: '["read","grep"]', + VINCI_QWEN_SERVED_REVISION: hex("ab"), + VINCI_QWEN_RUNTIME_ENGINE: "vllm", + VINCI_QWEN_RUNTIME_VERSION: "0.10.2", + VINCI_QWEN_RUNTIME_ARTIFACT_SHA256: hex("cd"), + VINCI_QWEN_RUNTIME_ARGUMENTS_SHA256: hex("ef"), + VINCI_QWEN_CONTEXT_WINDOW: "262144", + VINCI_QWEN_MAX_TOKENS: "8192", + VINCI_QWEN_INPUT_PER_MILLION_USD: "0.25", + VINCI_QWEN_OUTPUT_PER_MILLION_USD: "0.75", + VINCI_QWEN_CACHE_READ_PER_MILLION_USD: "0.05", + VINCI_QWEN_CACHE_WRITE_PER_MILLION_USD: "0.25", + VINCI_QWEN_PROMPT_SHA256: sha256(readFileSync(promptFile)), + VINCI_QWEN_TOOLS_SHA256: sha256('["read","grep"]'), + VINCI_UNATTENDED_POLICY: "governed", + VINCI_UNATTENDED_LEASE: "lease-test", + VINCI_QWEN_WORK_ORDER_ID: "wo-test", + VINCI_QWEN_RUN_ID: "run-test", + VINCI_QWEN_ATTEMPT_ID: "task-test/1", + VINCI_QWEN_CIRCUIT_FILE: join(temp, "circuit.json"), + VINCI_QWEN_CIRCUIT_THRESHOLD: "2", + VINCI_QWEN_CIRCUIT_OPEN_MS: "60000", +}; + +try { + assert.throws( + () => runtime.buildQwenQualificationTemplate({ ...baseEnv, VINCI_QWEN_ADMIT: undefined }), + /qualification_not_admitted/, + ); + const qualification = runtime.buildQwenQualificationTemplate(baseEnv); + assert.equal(qualification.model, "Qwen/Qwen3.8-27B"); + assert.equal(qualification.limits.max_concurrency, 1, "concurrency must start conservative"); + assert.equal(qualification.limits.max_retries, 1); + assert.equal(qualification.authority_role, "non-authoritative-evidence-and-proposals-only"); + assert.equal(qualification.fallback_policy, "explicit-openrouter-separate-attempt-only"); + writeFileSync(qualificationFile, `${JSON.stringify(qualification)}\n`, { mode: 0o400 }); + chmodSync(qualificationFile, 0o400); + const env = { + ...baseEnv, + VINCI_QWEN_QUALIFICATION_FILE: qualificationFile, + VINCI_QWEN_QUALIFICATION_SHA256: sha256(readFileSync(qualificationFile)), + }; + const config = runtime.loadQwenRuntimeConfig(env); + assert.equal(config.baseUrl, "https://qwen.example.test/v1"); + assert.equal(config.secret, "test-secret"); + + const runtimeTuple = qualification.runtime; + const observed = []; + const readyFetch = async (url, init = {}) => { + observed.push({ url: String(url), authorization: new Headers(init.headers).get("authorization"), method: init.method ?? "GET", body: init.body }); + if (String(url).endsWith("/health") && !new Headers(init.headers).has("authorization")) { + return Response.json({ error: "unauthorized" }, { status: 401 }); + } + if (String(url).endsWith("/health")) return Response.json({ status: "ready" }); + if (String(url).endsWith("/v1/models") && !new Headers(init.headers).has("authorization")) { + return Response.json({ error: "unauthorized" }, { status: 401 }); + } + if (String(url).endsWith("/v1/models")) { + return Response.json({ object: "list", data: [{ id: runtime.QWEN_MODEL, revision: qualification.revision, runtime: runtimeTuple }] }); + } + throw new Error(`unexpected URL ${url}`); + }; + const identity = await runtime.probeQwenReadiness(config, { fetchImpl: readyFetch, nowMs: 1_000 }); + assert.deepEqual(identity, { revision: qualification.revision, runtime: runtimeTuple }); + assert.deepEqual(observed.map(({ authorization }) => authorization), ["Bearer test-secret", "Bearer test-secret", null, null]); + assert.doesNotMatch(JSON.stringify(observed), /VINCI_QWEN_SECRET_REF/); + + const promptMismatch = { ...env, VINCI_QWEN_PROMPT_SHA256: hex("01") }; + assert.throws(() => runtime.loadQwenRuntimeConfig(promptMismatch), /qwen_prompt_mismatch/); + assert.throws( + () => runtime.loadQwenRuntimeConfig({ ...env, VINCI_UNATTENDED_POLICY: "off" }), + /qwen_authority_forbidden/, + ); + assert.throws( + () => runtime.buildQwenQualificationTemplate({ ...baseEnv, VINCI_QWEN_MAX_CONCURRENCY: "9" }), + /max_concurrency/, + ); + + const vectors = join(root, "vinci/test/fixtures/contract-vectors"); + const emptyCriteriaOrder = { + ...JSON.parse(readFileSync(join(vectors, "work-order-1-minimal/input.json"), "utf8")), + acceptanceCriteria: [], + }; + assert.throws( + () => digest.workOrderDigest(emptyCriteriaOrder), + /criteria_required/, + "the existing contract gate must reject a Qwen batch without acceptance criteria before materialization", + ); + assert.throws( + () => workerRun.runVinci({ + envelope: { provider: "qwen-h200", model: runtime.QWEN_MODEL, ref: "legacy-prose-ref", tools: ["read"], spec: "legacy prose" }, + repoDir: temp, + stateDir: temp, + taskId: "task-test", + sessionId: "run-test", + }), + /validated digest WorkOrder identity and acceptance criteria/, + "legacy prose cannot bypass the WorkOrder acceptance-criteria gate", + ); + + const wrongModelConfig = { ...config, circuitFile: join(temp, "wrong-model-circuit.json") }; + await assert.rejects( + runtime.probeQwenReadiness(wrongModelConfig, { + fetchImpl: async (url, init = {}) => { + if (String(url).endsWith("/health")) return Response.json({ status: "ready" }); + if (!new Headers(init.headers).has("authorization")) return Response.json({}, { status: 401 }); + return Response.json({ object: "list", data: [{ id: "Qwen/Qwen3.8-27B-alias", revision: qualification.revision, runtime: runtimeTuple }] }); + }, + nowMs: 2_000, + }), + /qwen_model_mismatch/, + ); + + const authOpenConfig = { ...config, circuitFile: join(temp, "auth-open-circuit.json") }; + await assert.rejects( + runtime.probeQwenReadiness(authOpenConfig, { + fetchImpl: async (url) => String(url).endsWith("/health") + ? Response.json({ status: "ready" }) + : Response.json({ object: "list", data: [{ id: runtime.QWEN_MODEL, revision: qualification.revision, runtime: runtimeTuple }] }), + nowMs: 3_000, + }), + /qwen_auth_not_enforced/, + ); + + const circuitConfig = { ...config, circuitFile: join(temp, "breaker.json") }; + let failedCalls = 0; + const unavailable = async () => { + failedCalls += 1; + throw new Error("offline fake"); + }; + await assert.rejects(runtime.probeQwenReadiness(circuitConfig, { fetchImpl: unavailable, nowMs: 10_000 }), /endpoint_unavailable/); + assert.equal(failedCalls, 2, "one retry means exactly two bounded attempts"); + await assert.rejects(runtime.probeQwenReadiness(circuitConfig, { fetchImpl: unavailable, nowMs: 11_000 }), /endpoint_unavailable/); + const callsAtOpen = failedCalls; + await assert.rejects(runtime.probeQwenReadiness(circuitConfig, { fetchImpl: unavailable, nowMs: 12_000 }), /qwen_circuit_open/); + assert.equal(failedCalls, callsAtOpen, "open circuit must make no endpoint call"); + + const cancelledConfig = { ...config, circuitFile: join(temp, "cancelled.json") }; + const controller = new AbortController(); + controller.abort("fixture cancellation"); + await assert.rejects( + runtime.probeQwenReadiness(cancelledConfig, { + signal: controller.signal, + fetchImpl: async (_url, init = {}) => { + assert.equal(init.signal.aborted, true); + throw new DOMException("aborted", "AbortError"); + }, + }), + /qwen_cancelled/, + ); + + const sse = [ + 'data: {"choices":[{"delta":{"tool_calls":[{"function":{"name":"report_","arguments":"{\\"status\\":\\""}}]}}]}', + 'data: {"choices":[{"delta":{"tool_calls":[{"function":{"name":"ready","arguments":"ready\\"}"}}]}}]}', + 'data: {"choices":[],"usage":{"prompt_tokens":10,"completion_tokens":2}}', + "data: [DONE]", + "", + ].join("\n"); + const canaryCalls = []; + const canary = await runtime.runQwenCanary( + { ...baseEnv, VINCI_QWEN_CANARY_TIMEOUT_MS: "1000" }, + async (url, init = {}) => { + canaryCalls.push({ url: String(url), init }); + if (String(url).endsWith("/health") && !new Headers(init.headers).has("authorization")) return Response.json({}, { status: 401 }); + if (String(url).endsWith("/health")) return Response.json({ status: "ready" }); + if (String(url).endsWith("/v1/models") && !new Headers(init.headers).has("authorization")) return Response.json({}, { status: 401 }); + if (String(url).endsWith("/v1/models")) return Response.json({ object: "list", data: [{ id: runtime.QWEN_MODEL, revision: qualification.revision, runtime: runtimeTuple }] }); + if (String(url).endsWith("/v1/chat/completions")) return new Response(sse, { headers: { "content-type": "text/event-stream" } }); + throw new Error(`unexpected URL ${url}`); + }, + ); + assert.equal(canary.capabilities.structured_output, "tool-arguments-json"); + assert.equal(canary.capabilities.usage_chunk, true); + const inference = canaryCalls.find(({ url }) => url.endsWith("/chat/completions")); + const payload = JSON.parse(inference.init.body); + assert.equal(payload.model, "Qwen/Qwen3.8-27B"); + assert.equal(payload.stream, true); + assert.equal(payload.tools[0].function.name, "report_ready"); + + const extensionEnvNames = [ + ...Object.keys(env), + "VINCI_QWEN_SELECTED", + ]; + const prior = new Map(extensionEnvNames.map((name) => [name, process.env[name]])); + const nativeFetch = globalThis.fetch; + try { + Object.assign(process.env, env, { VINCI_QWEN_SELECTED: "1", VINCI_QWEN_CIRCUIT_FILE: join(temp, "extension-circuit.json") }); + globalThis.fetch = readyFetch; + const registrations = []; + const handlers = {}; + const labels = []; + await providerExtension({ + registerProvider(name, providerConfig) { registrations.push({ name, providerConfig }); }, + on(name, handler) { (handlers[name] ??= []).push(handler); }, + appendEntry(name, value) { labels.push({ name, value }); }, + }); + assert.deepEqual(registrations.map(({ name }) => name), ["vinci", "qwen-h200"]); + const qwen = registrations[1].providerConfig; + assert.equal(qwen.api, "vinci-qwen-openai-completions"); + assert.equal(qwen.apiKey, "runtime-resolved-secret-reference", "provider config must not contain the secret value"); + assert.equal(qwen.models[0].id, "Qwen/Qwen3.8-27B"); + assert.equal(qwen.models[0].cost.input, 0.25); + assert.equal(process.env.VINCI_QWEN_SECRET_REF, undefined, "bootstrap secret reference must be scrubbed"); + + const headers = {}; + for (const handler of handlers.before_provider_headers ?? []) { + await handler({ headers }, { model: { provider: "qwen-h200" } }); + } + assert.equal(headers["x-vinci-work-order-id"], "wo-test"); + assert.equal(headers["x-vinci-run-id"], "run-test"); + assert.equal(headers["x-vinci-attempt-id"], "task-test/1"); + assert.equal(headers["x-vinci-qwen-output-authority"], "non-authoritative"); + + for (let index = 0; index < 2; index += 1) { + for (const handler of handlers.after_provider_response ?? []) { + await handler({ status: 401 }, { model: { provider: "qwen-h200" } }); + } + } + assert.throws( + () => qwen.streamSimple( + { ...qwen.models[0], provider: "qwen-h200", api: qwen.api }, + { messages: [] }, + ), + /qwen_circuit_open/, + "authentication failures must open the circuit before another inference call", + ); + + for (const handler of handlers.message_end ?? []) { + await handler( + { message: { role: "assistant", provider: "qwen-h200", stopReason: "stop" } }, + { sessionManager: { getSessionId: () => "run-test" } }, + ); + } + assert.equal(labels[0].name, "vinci-qwen-output-label"); + assert.equal(labels[0].value.authority, "non-authoritative"); + assert.equal(labels[0].value.independent_check_required, true); + } finally { + globalThis.fetch = nativeFetch; + for (const [name, value] of prior) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + } + + assert.equal(cleanroom.CLEAN_ROOM_ENV_ALLOWLIST.includes("VINCI_QWEN_SECRET_REF"), false); + assert.ok(cleanroom.PROVIDER_KEY_ENV["qwen-h200"].includes("VINCI_QWEN_SECRET_REF")); + const scoped = cleanroom.providerScopedEnv({ + base: { + OPENROUTER_API_KEY: "must-drop", + VINCI_QWEN_SECRET_REF: "env:QWEN_DYNAMIC_TEST_SECRET", + QWEN_DYNAMIC_TEST_SECRET: "dynamic-test-secret", + }, + provider: "qwen-h200", + agentDir: join(temp, "agent"), + }); + assert.equal(scoped.OPENROUTER_API_KEY, undefined); + assert.equal(scoped.VINCI_QWEN_SECRET_REF, "env:QWEN_DYNAMIC_TEST_SECRET"); + assert.equal(scoped.QWEN_DYNAMIC_TEST_SECRET, "dynamic-test-secret"); + const cleanScoped = cleanroom.cleanRoomEnv({ + base: scoped, + provider: "qwen-h200", + homeDir: join(temp, "clean-home"), + tmpDir: join(temp, "clean-tmp"), + }); + assert.equal(cleanScoped.QWEN_DYNAMIC_TEST_SECRET, "dynamic-test-secret"); + const otherProvider = cleanroom.providerScopedEnv({ + base: scoped, + provider: "openrouter", + agentDir: join(temp, "other-agent"), + }); + assert.equal(otherProvider.VINCI_QWEN_SECRET_REF, undefined); + assert.equal(otherProvider.QWEN_DYNAMIC_TEST_SECRET, undefined, "a dynamic Qwen secret must not cross provider boundaries"); + const envSecretConfig = { ...env, VINCI_QWEN_SECRET_REF: "env:QWEN_DYNAMIC_TEST_SECRET", QWEN_DYNAMIC_TEST_SECRET: "dynamic-test-secret" }; + assert.equal(runtime.loadQwenRuntimeConfig(envSecretConfig).secret, "dynamic-test-secret"); + assert.equal(envSecretConfig.QWEN_DYNAMIC_TEST_SECRET, undefined, "the resolved secret must be scrubbed before repository tools run"); + + const summary = economics.buildEconomicsSummary({ + workOrderId: "wo-test", + attemptLabel: "task-test/1", + sessionId: "run-test", + started: "2026-09-04T10:00:00.000Z", + finished: "2026-09-04T10:00:02.000Z", + usageEntries: [{ provider: "qwen-h200", model: runtime.QWEN_MODEL, model_calls: 1, input_tokens: 10, output_tokens: 2, cost_microusd: 4 }], + sessionState: { path: "/fake/session", source: "usage_entries", costUsd: 0.000004 }, + receipt: { verificationStatus: "passed" }, + run: { exit_code: 0, limit_tripped: null, harness_stops: [] }, + taskState: "UNVERIFIED", + }); + assert.equal(summary.route.policy_id, "single-provider-no-automatic-fallback"); + assert.equal(summary.route.initial_provider, "qwen-h200"); + assert.equal(summary.route.initial_model, "Qwen/Qwen3.8-27B"); + assert.equal(summary.work_order_id, "wo-test"); + assert.equal(summary.session_id, "run-test"); + assert.equal(summary.attempt_label, "task-test/1"); + assert.equal(summary.started_at, "2026-09-04T10:00:00.000Z"); + assert.equal(summary.finished_at, "2026-09-04T10:00:02.000Z"); +} finally { + chmodSync(qualificationFile, 0o600); + rmSync(temp, { recursive: true, force: true }); +} + +process.stdout.write(" Qwen H200 provider: qualification, readiness, auth, circuit, canary, attribution, and telemetry guards pass\n"); diff --git a/vinci/worker/README.md b/vinci/worker/README.md index 25610cfe9..4226a8dee 100644 --- a/vinci/worker/README.md +++ b/vinci/worker/README.md @@ -298,6 +298,117 @@ id (`auto` is deliberately never a class — a contract names a class, not "what resolves to"). A spec-level `provider` pin must EQUAL the configured provider for the class (`provider_mismatch` otherwise); it never overrides it. +### Qualified direct Qwen H200 lane + +`qwen-h200` is an internal OpenAI-compatible provider for the exact model +`Qwen/Qwen3.8-27B`. It is not an inference service and never manages a GPU: the operator supplies +Ayush's already-running endpoint. A digest-form class entry is: + +``` +{"qwen-38-27b":{"provider":"qwen-h200","model":"Qwen/Qwen3.8-27B"}} +``` + +Enable it only in the Worker process that should admit the lane by including `qwen-h200` in +`VINCI_WORKER_ALLOWED_PROVIDERS` and the entry above in `VINCI_WORKER_MODEL_CLASSES`. Qwen is +digest-WorkOrder-only; legacy prose handoffs are refused because they cannot carry the validated +acceptance criteria and immutable contract binding required by this lane. + +The lane is fail-closed. Before it is registered, the client requires: + +- `VINCI_QWEN_BASE_URL`: the operator-supplied HTTPS endpoint (loopback HTTP is allowed for local + tests). Credentials, query strings, and fragments are refused. +- `VINCI_QWEN_SECRET_REF`: `file:/absolute/private/path` or `env:NAME`; this setting is a reference, + never a secret value. Credential files must be private, regular, and non-symlinked. The bootstrap + reference is scrubbed before repository tools run. +- `VINCI_QWEN_QUALIFICATION_FILE` and `VINCI_QWEN_QUALIFICATION_SHA256`: an operator-owned, + non-writable qualification record and an exact process-level byte pin. +- a deterministic Governor lease, plus WorkOrder, Run, and Attempt ids derived by the worker. + Model output supplies none of these identities. + +Ayush's non-secret handoff is intentionally small: the externally reachable base URL; confirmation +that bearer authentication is required by `/health`, `/v1/models`, and `/v1/chat/completions`; the +exact `Qwen/Qwen3.8-27B` identifier and immutable served revision; runtime engine and version; the +SHA-256 of the runtime artifact and canonical launch arguments; and the served context/output +limits. `/v1/models` must return the exact revision and runtime tuple either on its one matching +model object or through the documented `X-Vinci-Model-Revision` and `X-Vinci-Runtime-*` headers. +Ayush supplies only the name of the operator-installed secret reference mechanism, never the +credential itself in a WorkOrder, issue, log, or qualification record. Runtime launch flags, model +download, GPU placement, and endpoint operation remain exclusively his lane. + +The closed qualification record binds the endpoint hash, exact model, immutable served revision, +runtime engine/version/artifact/arguments tuple, exact task-prompt hash, exact ordered tool-list +hash, streaming SSE and structured tool-call JSON, request timeout, concurrency, retries, +retry-delay cap, context/output bounds, and operator token-cost estimates. `/health` and +`/v1/models` are probed with authentication; an anonymous models request must be refused, and the +model response must repeat the exact revision/runtime tuple. Three consecutive readiness failures +open the persistent circuit for 60 seconds by default. `VINCI_QWEN_CIRCUIT_THRESHOLD` and +`VINCI_QWEN_CIRCUIT_OPEN_MS` are bounded overrides. + +Concurrency starts at one. The qualification schema allows an explicit increase up to eight after +burn-in; excess requests are rejected as backpressure, never queued without a bound. Transport +retries are capped at two. High available token volume does not admit work: each digest handoff +still needs bounded resources and acceptance criteria, and the worker refuses an invalid or +over-broad WorkOrder before inference. + +Qwen output gets a `vinci-qwen-output-label` session record marking it non-authoritative and +requiring independent checking. It is never permission, a Governor ruling, merge authorization, +spend/credential approval, or release authority. Existing deterministic leases, harness stops, +verification, review, and no-merge boundaries remain authoritative. There is no automatic provider +switching. OpenRouter fallback means a new, separately authorized attempt whose envelope explicitly +selects `openrouter` and whose operator allowlist permits it. + +Post-launch canary (readiness/auth GETs plus one bounded inference request; no deployment, GPU, +credential, or remote-state mutation): + +``` +VINCI_QWEN_BASE_URL=https://operator-endpoint.example \ +VINCI_QWEN_SECRET_REF=file:/run/secrets/vinci-qwen-token \ +node --experimental-strip-types vinci/extensions/lib/qwen-runtime.ts --canary +``` + +The canary requires the endpoint's models response (or headers) to expose the served revision and +runtime tuple. It requests one streaming `report_ready` tool call, checks that the assembled +arguments are exactly `{ "status": "ready" }`, and prints JSON to stdout. It does not write or +admit a qualification record. + +After independently reviewing that report, generate the exact per-WorkOrder qualification bytes +locally. The command writes JSON to stdout only; redirect it to an operator-owned file, make that +file non-writable, and pin its SHA-256 in the service environment. Required non-secret inputs are +the endpoint, prompt file, ordered tool list, served revision, runtime engine/version, runtime +artifact and arguments digests, context/output limits, and four estimated per-million-token rates. +Defaults are timeout 120 seconds, one retry with a 5-second cap, and concurrency 1. Admission is an +explicit operator act (`VINCI_QWEN_ADMIT=qualified`), never an inference result: + +``` +VINCI_QWEN_ADMIT=qualified \ +VINCI_QWEN_BASE_URL=https://operator-endpoint.example \ +VINCI_QWEN_QUALIFICATION_PROMPT_FILE=/absolute/work-order-prompt.txt \ +VINCI_QWEN_QUALIFICATION_TOOLS='["read","grep","find","ls","bash","edit","write"]' \ +VINCI_QWEN_SERVED_REVISION=<40-or-64-hex> \ +VINCI_QWEN_RUNTIME_ENGINE=vllm \ +VINCI_QWEN_RUNTIME_VERSION= \ +VINCI_QWEN_RUNTIME_ARTIFACT_SHA256=<64hex> \ +VINCI_QWEN_RUNTIME_ARGUMENTS_SHA256=<64hex> \ +VINCI_QWEN_CONTEXT_WINDOW= VINCI_QWEN_MAX_TOKENS= \ +VINCI_QWEN_INPUT_PER_MILLION_USD= \ +VINCI_QWEN_OUTPUT_PER_MILLION_USD= \ +VINCI_QWEN_CACHE_READ_PER_MILLION_USD= \ +VINCI_QWEN_CACHE_WRITE_PER_MILLION_USD= \ +node --experimental-strip-types vinci/extensions/lib/qwen-runtime.ts --qualification-template +``` + +Per-attempt telemetry remains in the existing economics summary: `work_order_id`, `session_id` +(Run), `attempt_label`, `started_at`/`finished_at` (wall latency), terminal `local_result`, and the +per-provider/model roll-up of calls, input/cache/output/reasoning tokens and estimated micro-USD. +The route is `single-provider-no-automatic-fallback` and names Qwen when inference occurred. + +Burn-in is deliberately small and ordered: one canary; one WorkOrder at concurrency 1; three +sequential WorkOrders; then a 15-minute soak of batches of at most four WorkOrders with concurrency +still 1. Review token totals, wall latency (`started_at`/`finished_at`), terminal outcome, estimated +cost, readiness failures, and circuit state after each step. Raise qualified concurrency to 2 only +after every acceptance criterion passes; repeat the soak before any further explicit increase. +Hundreds of millions of available tokens are capacity, not an acceptance signal. + ### Base checkout (`baseRef` / `baseCommit`) The task branch (`targetBranch`) is created FROM the pinned `baseCommit`, never continued from @@ -542,6 +653,8 @@ configured nothing changes (no downgrade), so soak boxes may run without it. - `VINCI_DECLARATION_REFRESH_S`: how often (seconds) a governed daemon re-posts its capability declaration; default `21600` (6h), and anything that is not a positive number falls back to the default. It must stay comfortably below the Governor's `VGC_DECLARATION_MAX_AGE_S` (default 86400), which is when a declaration expires and admission starts answering `eligible: false, reason: stale_declaration`. **The default is chosen against row retention, not liveness** (gpu-control §32): the Governor's `worker_declarations` table is append-only with a DELETE trigger and every refresh writes an audit row, so the volume cannot be pruned later. 6h keeps four refreshes inside the 24h window — three consecutive failed re-posts can be absorbed before one goes stale — at a quarter the rows of hourly, which buys no liveness at all - `GH_TOKEN`: (optional) GitHub machine user token for cloning/pushing private repos and creating PRs - `OPENROUTER_API_KEY`: (or provider-specific key) via vinci's standard configuration +- `VINCI_QWEN_BASE_URL` + `VINCI_QWEN_SECRET_REF`: direct Qwen endpoint and credential reference; + never place the credential value in worker configuration Never hardcode. Use systemd SecureString parameters, AWS Secrets Manager, or similar. @@ -696,11 +809,13 @@ byte-for-byte what it was. **The child's environment (exact allowlist).** Copied verbatim from the daemon when set: `PATH`, `LANG`, `VINCI_ENV`, `VINCI_BASE_URL`, `VINCI_PLATFORM_URL`, `VINCI_NO_BOOTSTRAP_HEAL`, -`VINCI_TOOL_BOOTSTRAP`, `VINCI_SHOW_OTHER_PROVIDERS`, `VINCI_SOURCE_CLI` — the variables +`VINCI_TOOL_BOOTSTRAP`, `VINCI_SHOW_OTHER_PROVIDERS`, `VINCI_SOURCE_CLI`, and the Qwen +endpoint/secret-reference/qualification/circuit settings above — the variables `vinci/bin/vinci` and the install shim read to find the backend and the run mode. Plus **only** the key the envelope's `provider:` authenticates with: `OPENROUTER_API_KEY` for `openrouter`, `VINCI_API_KEY` for `vinci`, `VINCI_INTERNAL_DEEPINFRA_API_KEY` for `deepinfra` (an unknown provider -gets no key and the launcher refuses it, as today). Set by the daemon, never copied: `HOME` and +gets no key and the launcher refuses it, as today). `qwen-h200` carries no credential value through +the allowlist; its extension resolves the narrow reference during bootstrap. Set by the daemon, never copied: `HOME` and `TMPDIR` (per attempt), `VINCI_CODING_AGENT_DIR` and `PI_CODING_AGENT_DIR` (both spellings, both `.home/agent` — the daemon's own slot is **not** passed through: it holds `auth.json` for every provider, every prior session and `bin/`), `VINCI_HOME` (the launcher's install root — the diff --git a/vinci/worker/cleanroom.mjs b/vinci/worker/cleanroom.mjs index 62fca0cb9..d1a756cd8 100644 --- a/vinci/worker/cleanroom.mjs +++ b/vinci/worker/cleanroom.mjs @@ -98,12 +98,21 @@ export const CLEAN_ROOM_ENV_ALLOWLIST = Object.freeze([ "VINCI_SOURCE_CLI", ]); -// ONLY the key the envelope's provider authenticates with (vinci/bin/vinci reads exactly these). -// An unknown provider gets no key at all and the launcher refuses it, as it does today. +// ONLY the authentication and provider-routing environment selected by the envelope. An unknown +// provider gets none and the launcher refuses it, as it does today. Qwen receives references and +// non-secret pins/settings here; the resolved bearer value is never a worker config value. export const PROVIDER_KEY_ENV = Object.freeze({ openrouter: ["OPENROUTER_API_KEY"], vinci: ["VINCI_API_KEY"], deepinfra: ["VINCI_INTERNAL_DEEPINFRA_API_KEY"], + "qwen-h200": [ + "VINCI_QWEN_BASE_URL", + "VINCI_QWEN_SECRET_REF", + "VINCI_QWEN_QUALIFICATION_FILE", + "VINCI_QWEN_QUALIFICATION_SHA256", + "VINCI_QWEN_CIRCUIT_THRESHOLD", + "VINCI_QWEN_CIRCUIT_OPEN_MS", + ], }); // Every provider credential and authentication-routing value the bundled coding agent knows how @@ -113,6 +122,12 @@ export const PROVIDER_KEY_ENV = Object.freeze({ // inventory was fail-open — an OpenRouter child still received Anthropic, OpenAI and public // DeepInfra credentials. export const PROVIDER_CREDENTIAL_ENV = Object.freeze([ + "VINCI_QWEN_BASE_URL", + "VINCI_QWEN_SECRET_REF", + "VINCI_QWEN_QUALIFICATION_FILE", + "VINCI_QWEN_QUALIFICATION_SHA256", + "VINCI_QWEN_CIRCUIT_THRESHOLD", + "VINCI_QWEN_CIRCUIT_OPEN_MS", "AI_GATEWAY_API_KEY", "ANTHROPIC_API_KEY", "ANTHROPIC_OAUTH_TOKEN", @@ -161,6 +176,15 @@ export const PROVIDER_CREDENTIAL_ENV = Object.freeze([ "ZAI_CODING_CN_API_KEY", ]); +const QWEN_ENV_SECRET_REFERENCE = /^env:([A-Z][A-Z0-9_]{0,127})$/; + +function qwenSecretEnvName(base) { + const match = typeof base.VINCI_QWEN_SECRET_REF === "string" + ? base.VINCI_QWEN_SECRET_REF.match(QWEN_ENV_SECRET_REFERENCE) + : null; + return match?.[1]; +} + // The provider boundary for the NORMAL (non-clean-room) path. // // PROVIDER_KEY_ENV above promises a child gets ONLY the key its envelope's provider @@ -188,6 +212,8 @@ export function providerScopedEnv({ base = process.env, provider, agentDir }) { const keep = new Set(Object.hasOwn(PROVIDER_KEY_ENV, provider) ? PROVIDER_KEY_ENV[provider] : []); const env = { ...base }; for (const key of PROVIDER_CREDENTIAL_ENV) if (!keep.has(key)) delete env[key]; + const referencedQwenSecret = qwenSecretEnvName(base); + if (provider !== "qwen-h200" && referencedQwenSecret) delete env[referencedQwenSecret]; // Do not let normal mode's provider selection be bypassed by the daemon's shared auth.json. // This is a resolution boundary, not uid isolation: a same-uid child can still deliberately // read the daemon's files. Both launchers resolve stored credentials from this isolated slot. @@ -221,6 +247,10 @@ export function cleanRoomEnv({ base = process.env, provider, homeDir, tmpDir }) // Same prototype-chain hazard as providerScopedEnv below. const providerKeys = Object.hasOwn(PROVIDER_KEY_ENV, provider) ? PROVIDER_KEY_ENV[provider] : []; for (const key of providerKeys) if (base[key] !== undefined) env[key] = base[key]; + const referencedQwenSecret = qwenSecretEnvName(base); + if (provider === "qwen-h200" && referencedQwenSecret && base[referencedQwenSecret] !== undefined) { + env[referencedQwenSecret] = base[referencedQwenSecret]; + } const vinciHome = base.VINCI_HOME ?? (base.HOME ? join(base.HOME, ".vinci-code") : undefined); if (vinciHome !== undefined) env.VINCI_HOME = vinciHome; env.HOME = homeDir; diff --git a/vinci/worker/economics.mjs b/vinci/worker/economics.mjs index fb98d210d..2ad08888a 100644 --- a/vinci/worker/economics.mjs +++ b/vinci/worker/economics.mjs @@ -254,7 +254,15 @@ export function buildEconomicsSummary(input = {}) { summary.finished_at = finishedAt; if (work !== null) summary.work = work; if (usage.length > 0) summary.usage = usage; - summary.route = { policy_id: "none", initial_provider: null, initial_model: null, escalations: [] }; + // The worker runs exactly one provider/model per attempt. Recording the first usage row makes + // that route attributable without inventing a provider when no inference occurred. There is + // deliberately no in-attempt fallback: an OpenRouter fallback is a separate authorized attempt. + summary.route = { + policy_id: "single-provider-no-automatic-fallback", + initial_provider: usage[0]?.provider ?? null, + initial_model: usage[0]?.model ?? null, + escalations: [], + }; summary.assets_consumed = []; summary.compactions = 0; summary.human_interventions = []; diff --git a/vinci/worker/run.mjs b/vinci/worker/run.mjs index c2f7ed7fb..d1053559e 100644 --- a/vinci/worker/run.mjs +++ b/vinci/worker/run.mjs @@ -1210,6 +1210,47 @@ export function runVinci({ envelope, repoDir, stateDir, taskId, sessionId, env, const tools = Array.isArray(envelope.tools) && envelope.tools.length > 0 ? envelope.tools.join(",") : "read,grep,find,ls,bash,edit,write"; const taskEnvironment = applyEnvDelta(env ?? process.env, envDelta); taskEnvironment.VINCI_UPDATE_DISABLED = "1"; + // The direct H200 lane is one exact, pre-qualified provider. These values are derived by the + // worker, not accepted from the model or repository, and bind the provider extension to the + // WorkOrder/Run/Attempt plus the exact prompt and tool surface for this invocation. + if (envelope.provider === "qwen-h200") { + if (envelope.model !== "Qwen/Qwen3.8-27B") { + throw blocked("qwen_model_mismatch", "qwen_model_mismatch: qwen-h200 may serve only Qwen/Qwen3.8-27B"); + } + const workOrderId = envelope.work_order_id; + if (typeof workOrderId !== "string" || !workOrderId) { + throw blocked( + "qwen_attribution_missing", + "qwen_attribution_missing: qwen-h200 requires a validated digest WorkOrder identity and acceptance criteria", + ); + } + let attempt; + try { + attempt = JSON.parse(readFileSync(join(stateDir, "tasks", `${taskId}.json`), "utf8")).attempt; + } catch { + throw blocked("qwen_attribution_missing", "qwen_attribution_missing: task attempt state is unavailable"); + } + if (!Number.isSafeInteger(attempt) || attempt < 1) { + throw blocked("qwen_attribution_missing", "qwen_attribution_missing: task attempt is invalid"); + } + taskEnvironment.VINCI_QWEN_SELECTED = "1"; + taskEnvironment.VINCI_QWEN_WORK_ORDER_ID = workOrderId; + taskEnvironment.VINCI_QWEN_RUN_ID = sessionId; + taskEnvironment.VINCI_QWEN_ATTEMPT_ID = `${taskId}/${attempt}`; + taskEnvironment.VINCI_QWEN_PROMPT_SHA256 = sha256(envelope.spec); + taskEnvironment.VINCI_QWEN_TOOLS_SHA256 = sha256(canonicalize(Array.isArray(envelope.tools) && envelope.tools.length > 0 ? envelope.tools : tools.split(","))); + taskEnvironment.VINCI_QWEN_CIRCUIT_FILE = join(stateDir, "qwen-h200", "circuit.json"); + } else { + for (const name of [ + "VINCI_QWEN_SELECTED", + "VINCI_QWEN_WORK_ORDER_ID", + "VINCI_QWEN_RUN_ID", + "VINCI_QWEN_ATTEMPT_ID", + "VINCI_QWEN_PROMPT_SHA256", + "VINCI_QWEN_TOOLS_SHA256", + "VINCI_QWEN_CIRCUIT_FILE", + ]) delete taskEnvironment[name]; + } for (const name of [ "VINCI_WORKER_DEBRIS_ROOT_ANCHOR", "VINCI_WORKER_DEBRIS_ROOT_ANCHOR_SHA256", diff --git a/vinci/worker/task.mjs b/vinci/worker/task.mjs index b224db5ff..3b5c39ae0 100644 --- a/vinci/worker/task.mjs +++ b/vinci/worker/task.mjs @@ -423,6 +423,7 @@ export function materializeEnvelope(triple, registry, opts = {}) { return { envelope: { + work_order_id: triple.work_order_id, repo, evidence, provider, From 1ccc7d4afa6958442a307f49260b08cb018c8e42 Mon Sep 17 00:00:00 2001 From: George Pu <19347973+thegeorgepu@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:42:58 -0400 Subject: [PATCH 2/9] fix(worker): harden qualified qwen lane --- packages/ai/CHANGELOG.md | 1 + packages/ai/src/api/openai-completions.ts | 19 +- packages/ai/src/index.ts | 2 +- packages/ai/src/legacy-api-aliases.ts | 4 +- .../openai-completions-empty-tools.test.ts | 16 + packages/coding-agent/CHANGELOG.md | 2 +- vinci/bin/vinci | 12 + vinci/extensions/lib/qwen-runtime.ts | 1543 +++++++++++++---- vinci/extensions/vinci-provider.ts | 151 -- vinci/extensions/vinci-qwen-provider.ts | 230 +++ vinci/test/fixtures/qwen-loader-probe.ts | 25 + vinci/test/worker-qwen-loader-startup.mjs | 100 ++ vinci/test/worker-qwen-provider.mjs | 866 ++++++--- vinci/worker/README.md | 140 +- vinci/worker/cleanroom.mjs | 21 +- vinci/worker/run.mjs | 88 +- vinci/worker/task.mjs | 10 +- 17 files changed, 2449 insertions(+), 781 deletions(-) create mode 100644 vinci/extensions/vinci-qwen-provider.ts create mode 100644 vinci/test/fixtures/qwen-loader-probe.ts create mode 100644 vinci/test/worker-qwen-loader-startup.mjs diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 00a179985..3b3a2df69 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -18,6 +18,7 @@ ### Added +- Added an optional request-scoped `fetch` transport to OpenAI-compatible Completions so governed providers can enforce their own connection pinning, deadlines, body bounds, and retry policy without bypassing the shared response parser. - Refreshed generated model catalogs from models.dev, adding newly listed models including Kimi K2.7 Code for GitHub Copilot and Fable 5 to several providers ([#6256](https://github.com/earendil-works/pi/issues/6256)). - Added Claude Sonnet 5 to the GitHub Copilot model catalog ([#6200](https://github.com/earendil-works/pi/issues/6200)). - Added zstd request-body compression for the OpenAI Codex Responses SSE transport. Requests are sent with `Content-Encoding: zstd` when Node/Bun zstd support is available; the WebSocket transport is unchanged. diff --git a/packages/ai/src/api/openai-completions.ts b/packages/ai/src/api/openai-completions.ts index 898595bef..c4e7647bc 100644 --- a/packages/ai/src/api/openai-completions.ts +++ b/packages/ai/src/api/openai-completions.ts @@ -110,6 +110,14 @@ function isEncryptedReasoningDetail(detail: unknown): detail is OpenAIEncryptedR export interface OpenAICompletionsOptions extends StreamOptions { toolChoice?: "auto" | "none" | "required" | { type: "function"; function: { name: string } }; reasoningEffort?: "minimal" | "low" | "medium" | "high" | "xhigh"; + /** Provider-scoped transport override for endpoint pinning and bounded response streams. */ + fetch?: typeof globalThis.fetch; +} + +export interface OpenAICompletionsSimpleOptions extends SimpleStreamOptions { + toolChoice?: OpenAICompletionsOptions["toolChoice"]; + /** Provider-scoped transport override for endpoint pinning and bounded response streams. */ + fetch?: typeof globalThis.fetch; } interface OpenAICompatCacheControl { @@ -180,7 +188,7 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio const compat = getCompat(model); const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env); const cacheSessionId = cacheRetention === "none" ? undefined : options?.sessionId; - const client = createClient(model, context, apiKey, options?.headers, cacheSessionId, compat); + const client = createClient(model, context, apiKey, options?.headers, cacheSessionId, compat, options?.fetch); let params = buildParams(model, context, options, compat, cacheRetention); const nextParams = await options?.onPayload?.(params, model); if (nextParams !== undefined) { @@ -487,22 +495,23 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio return stream; }; -export const streamSimple: StreamFunction<"openai-completions", SimpleStreamOptions> = ( +export const streamSimple: StreamFunction<"openai-completions", OpenAICompletionsSimpleOptions> = ( model: Model<"openai-completions">, context: Context, - options?: SimpleStreamOptions, + options?: OpenAICompletionsSimpleOptions, ): AssistantMessageEventStream => { getClientApiKey(model.provider, options?.apiKey, options?.headers); const base = buildBaseOptions(model, context, options, options?.apiKey); const clampedReasoning = options?.reasoning ? clampThinkingLevel(model, options.reasoning) : undefined; const reasoningEffort = clampedReasoning === "off" ? undefined : clampedReasoning; - const toolChoice = (options as OpenAICompletionsOptions | undefined)?.toolChoice; + const toolChoice = options?.toolChoice; return stream(model, context, { ...base, reasoningEffort, toolChoice, + fetch: options?.fetch, } satisfies OpenAICompletionsOptions); }; @@ -513,6 +522,7 @@ function createClient( optionsHeaders?: ProviderHeaders, sessionId?: string, compat: ResolvedOpenAICompletionsCompat = getCompat(model), + requestFetch?: typeof globalThis.fetch, ) { const headers: ProviderHeaders = { ...model.headers }; if (model.provider === "github-copilot") { @@ -540,6 +550,7 @@ function createClient( baseURL: model.baseUrl, dangerouslyAllowBrowser: true, defaultHeaders: headers, + fetch: requestFetch, }); } diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 646c6ec0b..3acfb2fb7 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -15,7 +15,7 @@ export type { GoogleVertexOptions } from "./api/google-vertex.ts"; export * from "./api/lazy.ts"; export type { MistralOptions } from "./api/mistral-conversations.ts"; export type { OpenAICodexResponsesOptions, OpenAICodexWebSocketDebugStats } from "./api/openai-codex-responses.ts"; -export type { OpenAICompletionsOptions } from "./api/openai-completions.ts"; +export type { OpenAICompletionsOptions, OpenAICompletionsSimpleOptions } from "./api/openai-completions.ts"; export type { OpenAIResponsesOptions } from "./api/openai-responses.ts"; export * from "./auth/context.ts"; export * from "./auth/credential-store.ts"; diff --git a/packages/ai/src/legacy-api-aliases.ts b/packages/ai/src/legacy-api-aliases.ts index b49c199cc..25a74ffde 100644 --- a/packages/ai/src/legacy-api-aliases.ts +++ b/packages/ai/src/legacy-api-aliases.ts @@ -11,7 +11,7 @@ import type { MistralOptions } from "./api/mistral-conversations.ts"; import { openAICodexResponsesApi } from "./api/openai-codex-responses.lazy.ts"; import type { OpenAICodexResponsesOptions } from "./api/openai-codex-responses.ts"; import { openAICompletionsApi } from "./api/openai-completions.lazy.ts"; -import type { OpenAICompletionsOptions } from "./api/openai-completions.ts"; +import type { OpenAICompletionsOptions, OpenAICompletionsSimpleOptions } from "./api/openai-completions.ts"; import { openAIResponsesApi } from "./api/openai-responses.lazy.ts"; import type { OpenAIResponsesOptions } from "./api/openai-responses.ts"; import type { SimpleStreamOptions, StreamFunction } from "./types.ts"; @@ -93,7 +93,7 @@ export const streamOpenAICompletions = openAICompletionsStreams.stream as Stream /** @deprecated Use `streamSimple` from `@earendil-works/pi-ai/api/openai-completions` or `openAICompletionsApi().streamSimple`. */ export const streamSimpleOpenAICompletions = openAICompletionsStreams.streamSimple as StreamFunction< "openai-completions", - SimpleStreamOptions + OpenAICompletionsSimpleOptions >; /** @deprecated Use `stream` from `@earendil-works/pi-ai/api/openai-responses` or `openAIResponsesApi().stream`. */ diff --git a/packages/ai/test/openai-completions-empty-tools.test.ts b/packages/ai/test/openai-completions-empty-tools.test.ts index 9fd9fc5f8..2cde335e5 100644 --- a/packages/ai/test/openai-completions-empty-tools.test.ts +++ b/packages/ai/test/openai-completions-empty-tools.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import { streamSimple as streamSimpleOpenAICompletions } from "../src/api/openai-completions.ts"; import { getModel, streamSimple } from "../src/compat.ts"; // Empty tools arrays must NOT be serialized as `tools: []` — some OpenAI-compatible @@ -92,6 +93,21 @@ describe("openai-completions empty tools handling", () => { expect("tools" in (params as object)).toBe(false); }); + it("passes a request-scoped fetch transport to the OpenAI client", async () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; + const model = { ...baseModel, api: "openai-completions" } as const; + const requestFetch = vi.fn() as unknown as typeof globalThis.fetch; + + await streamSimpleOpenAICompletions( + model, + { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }, + { apiKey: "test", fetch: requestFetch }, + ).result(); + + const clientOptions = mockState.lastClientOptions as { fetch?: typeof globalThis.fetch }; + expect(clientOptions.fetch).toBe(requestFetch); + }); + it("sends default maxTokens", async () => { const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini")!; const model = { ...baseModel, api: "openai-completions" } as const; diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 62a2f9e38..d3786b542 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -13,7 +13,7 @@ - Added public SDK exports for CLI-equivalent model and scoped-model resolution ([#6201](https://github.com/earendil-works/pi/issues/6201)). - Added extension entry renderers for persisted display-only session entries that are rendered in interactive mode without being sent to the model context. - Added optional terminal and process-handler injection to `InteractiveMode` for deterministic embedded and UI-test environments. -- Added a fail-closed, qualified Vinci Worker lane for the exact non-authoritative `Qwen/Qwen3.8-27B` H200 endpoint. +- Added a fail-closed Vinci Worker lane for the exact non-authoritative `Qwen/Qwen3.8-27B` H200 endpoint, gated by independently signed exact-build qualification, bounded pinned transport, per-attempt accounting, and the 1→2→4→8→16→24→32 burn-in schema (runtime concurrency remains 1 until a fleet permit authority exists). ### Changed diff --git a/vinci/bin/vinci b/vinci/bin/vinci index e670a0f60..369d9c035 100755 --- a/vinci/bin/vinci +++ b/vinci/bin/vinci @@ -436,6 +436,18 @@ case "${VINCI_PROVIDER}" in } export VINCI_DEEPINFRA_QUALIFICATION=1 ;; + qwen-h200) + [ "${VINCI_QWEN_SELECTED:-0}" = "1" ] || { + echo "✗ qwen-h200 is a Worker-only qualified provider" >&2 + exit 2 + } + [ "${VINCI_MODEL}" = "Qwen/Qwen3.8-27B" ] || { + echo "✗ The Qwen H200 lane is pinned to Qwen/Qwen3.8-27B" >&2 + exit 2 + } + VINCI_QWEN_EXTENSION="${VINCI}/extensions/vinci-qwen-provider.ts" + set -- --extension "${VINCI_QWEN_EXTENSION}" "$@" + ;; *) # Offline process-level regressions inject a faux provider without reopening public CLI flags. [ "${VINCI_INTERNAL_PROVIDER_TEST:-0}" = "1" ] || { diff --git a/vinci/extensions/lib/qwen-runtime.ts b/vinci/extensions/lib/qwen-runtime.ts index cb1836db6..84ae39459 100644 --- a/vinci/extensions/lib/qwen-runtime.ts +++ b/vinci/extensions/lib/qwen-runtime.ts @@ -1,27 +1,73 @@ -import { createHash } from "node:crypto"; +import { createHash, createPublicKey, randomBytes, verify } from "node:crypto"; +import { lookup as dnsLookup } from "node:dns/promises"; import { + closeSync, + constants, existsSync, + fstatSync, lstatSync, mkdirSync, + openSync, readFileSync, renameSync, + rmdirSync, writeFileSync, } from "node:fs"; +import { isIP } from "node:net"; import { dirname, isAbsolute } from "node:path"; import { pathToFileURL } from "node:url"; +import type { Context, Model, ProviderHeaders } from "@earendil-works/pi-ai"; +import { Agent, fetch as undiciFetch } from "undici"; export const QWEN_PROVIDER = "qwen-h200"; export const QWEN_MODEL = "Qwen/Qwen3.8-27B"; export const QWEN_API = "vinci-qwen-openai-completions"; -const QUALIFICATION_SCHEMA = "vinci.qwen-worker-qualification.v1"; -const CIRCUIT_SCHEMA = "vinci.qwen-worker-circuit.v1"; -const MAX_RESPONSE_BYTES = 256 * 1024; -const FALLBACK_POLICY = "explicit-openrouter-separate-attempt-only"; +const QUALIFICATION_SCHEMA = "vinci.qwen-worker-qualification.v2"; +const QUALIFICATION_ENVELOPE_SCHEMA = "vinci.qwen-worker-qualification-envelope.v2"; +const QUALIFICATION_REQUEST_SCHEMA = "vinci.qwen-worker-qualification-request.v2"; +const CIRCUIT_SCHEMA = "vinci.qwen-worker-circuit.v2"; +const CANARY_SCHEMA = "vinci.qwen-worker-canary.v2"; const AUTHORITY_ROLE = "non-authoritative-evidence-and-proposals-only"; +const FALLBACK_POLICY = "explicit-openrouter-separate-attempt-only"; +const QUALIFICATION_AUTHORITY = "independent-never-builder-review"; +const MAX_QUALIFICATION_BYTES = 512 * 1024; +const MAX_CANARY_BYTES = 256 * 1024; +const MAX_BURN_IN_BYTES = 256 * 1024; const HEX64 = /^[0-9a-f]{64}$/; const IMMUTABLE_REVISION = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/; -const ENV_NAME = /^[A-Z][A-Z0-9_]{0,127}$/; +const IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,255}$/; +const UTC_TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/; +const MAX_QUALIFICATION_LIFETIME_MS = 7 * 24 * 60 * 60 * 1000; +const MAX_CANARY_AGE_AT_ISSUE_MS = 24 * 60 * 60 * 1000; +const LOCK_WAIT_MS = 1_000; +const LOCK_POLL_MS = 10; +const SLEEP_CELL = new Int32Array(new SharedArrayBuffer(4)); + +export const QWEN_REQUEST_ENCODING = Object.freeze({ + schema: "vinci.qwen-openai-chat-request.v1", + api: "openai-completions", + path: "/v1/chat/completions", + transport: "sse", + thinking_format: "qwen-chat-template", + tool_output: "tool-arguments-json", + redirects: "refused", + retries: "client-bounded", +}); + +export const QWEN_REQUALIFICATION_CONDITIONS = Object.freeze([ + "canary-expired-or-failed", + "capabilities-or-limits-changed", + "client-build-changed", + "endpoint-identity-or-address-policy-changed", + "model-or-revision-changed", + "outbound-request-encoding-changed", + "price-basis-changed", + "runtime-artifact-or-arguments-changed", + "system-prompt-changed", + "tool-schema-or-policy-changed", +]); +export const QWEN_CONCURRENCY_LADDER = Object.freeze([1, 2, 4, 8, 16, 24, 32]); type RuntimeTuple = { engine: string; @@ -35,31 +81,113 @@ type Qualification = { status: string; authority_role: string; fallback_policy: string; - model: string; - revision: string; - runtime: RuntimeTuple; - endpoint_sha256: string; - prompt_sha256: string; - tools_sha256: string; + safe_resume: boolean; + provenance: { + issuer: string; + authority: string; + issued_at: string; + expires_at: string; + review_message_id: string; + review_body_sha256: string; + burn_in_report_sha256: string; + canary: { + schema: string; + report_sha256: string; + observed_at: string; + }; + }; + burn_in: { + schema: string; + previous_concurrency: number; + target_concurrency: number; + observed_hours: number; + work_orders: number; + acceptance_pass_rate: number; + usage_coverage_rate: number; + transport_error_rate: number; + identity_failures: number; + verification_failures: number; + circuit_opens: number; + resource_alarms: number; + governor_stops: number; + }; + bindings: { + model: string; + revision: string; + runtime: RuntimeTuple; + endpoint_sha256: string; + endpoint_identity_sha256: string; + work_order_prompt_sha256: string; + system_prompt_sha256: string; + tool_names_sha256: string; + tool_schemas_sha256: string; + tool_policy_sha256: string; + client_build_sha256: string; + extension_build_sha256: string; + request_encoding_sha256: string; + }; capabilities: { streaming_sse: boolean; tool_calls: boolean; structured_output: string; + usage_chunk: boolean; }; limits: { - timeout_ms: number; + total_timeout_ms: number; max_retries: number; max_retry_delay_ms: number; max_concurrency: number; + advertised_max_concurrency: number; context_window: number; max_tokens: number; + max_request_bytes: number; + max_response_bytes: number; + max_error_bytes: number; }; pricing: { + currency: string; + basis: string; input_per_million_usd: number; output_per_million_usd: number; cache_read_per_million_usd: number; cache_write_per_million_usd: number; }; + requalification_conditions: string[]; +}; + +type QualificationEnvelope = { + schema: string; + qualification: Qualification; + signature: { + algorithm: string; + key_id: string; + signature_base64: string; + }; +}; + +type CircuitState = { + schema: string; + failures: number; + open_until_ms: number; + last_reason: string | null; + sequence: number; +}; + +type LookupAddress = { address: string; family: number }; +export type QwenLookup = (hostname: string) => Promise; +export type QwenFetch = (input: string | URL | Request, init?: RequestInit) => Promise; + +export type QwenAttemptRecord = { + request_id: string; + transport_attempt: number; + started_at: string; + finished_at: string; + latency_ms: number; + outcome: string; + status: number | null; + cost_usd: number; + input_tokens: number; + output_tokens: number; }; export type QwenRuntimeConfig = { @@ -67,8 +195,10 @@ export type QwenRuntimeConfig = { healthUrl: string; modelsUrl: string; chatUrl: string; + endpointHostname: string; + endpointLoopback: boolean; + endpointAddresses: string[]; secret: string; - secretRef: string; qualification: Qualification; qualificationSha256: string; circuitFile: string; @@ -81,13 +211,6 @@ export type QwenRuntimeConfig = { }; }; -type CircuitState = { - schema: string; - failures: number; - open_until_ms: number; - last_reason: string | null; -}; - export class QwenReadinessError extends Error { code: string; @@ -102,35 +225,35 @@ function fail(code: string, message: string): never { throw new QwenReadinessError(code, message); } -function sha256(value: string | Buffer): string { +export function qwenSha256(value: string | Buffer): string { return createHash("sha256").update(value).digest("hex"); } -function canonical(value: unknown): string { +export function qwenCanonical(value: unknown): string { if (value === null) return "null"; - if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; + if (Array.isArray(value)) return `[${value.map(qwenCanonical).join(",")}]`; if (typeof value === "object") { const record = value as Record; return `{${Object.keys(record) .sort() - .map((key) => `${JSON.stringify(key)}:${canonical(record[key])}`) + .map((key) => `${JSON.stringify(key)}:${qwenCanonical(record[key])}`) .join(",")}}`; } - return JSON.stringify(value); + return JSON.stringify(value) ?? "null"; } function exactKeys(value: unknown, expected: string[], label: string): asserts value is Record { if (!value || typeof value !== "object" || Array.isArray(value)) fail("qualification_invalid", `${label} must be an object`); const actual = Object.keys(value).sort(); const wanted = [...expected].sort(); - if (canonical(actual) !== canonical(wanted)) fail("qualification_invalid", `${label} has unexpected or missing fields`); + if (qwenCanonical(actual) !== qwenCanonical(wanted)) fail("qualification_invalid", `${label} has unexpected or missing fields`); } function boundedInteger(value: unknown, minimum: number, maximum: number, label: string): number { - if (!Number.isSafeInteger(value) || (value as number) < minimum || (value as number) > maximum) { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < minimum || value > maximum) { fail("qualification_invalid", `${label} must be an integer in [${minimum}, ${maximum}]`); } - return value as number; + return value; } function nonNegativeNumber(value: unknown, label: string): number { @@ -140,81 +263,34 @@ function nonNegativeNumber(value: unknown, label: string): number { return value; } -export function normalizeQwenBaseUrl(raw: string | undefined): { - baseUrl: string; - healthUrl: string; - modelsUrl: string; - chatUrl: string; -} { - if (!raw) fail("config_missing", "VINCI_QWEN_BASE_URL is required"); - let parsed: URL; - try { - parsed = new URL(raw); - } catch { - fail("config_invalid", "VINCI_QWEN_BASE_URL must be an absolute URL"); - } - if (parsed.username || parsed.password || parsed.search || parsed.hash) { - fail("config_invalid", "VINCI_QWEN_BASE_URL may not contain credentials, a query, or a fragment"); +function nonEmptyString(value: unknown, label: string, maximum = 512): string { + if (typeof value !== "string" || value.length < 1 || value.length > maximum || /[\r\n\0]/.test(value)) { + fail("qualification_invalid", `${label} must be a bounded single-line string`); } - const loopback = parsed.hostname === "127.0.0.1" || parsed.hostname === "localhost" || parsed.hostname === "[::1]"; - if (parsed.protocol !== "https:" && !(parsed.protocol === "http:" && loopback)) { - fail("config_invalid", "VINCI_QWEN_BASE_URL must use HTTPS (HTTP is allowed only on loopback)"); - } - const withoutSlashes = parsed.toString().replace(/\/+$/, ""); - const root = withoutSlashes.endsWith("/v1") ? withoutSlashes.slice(0, -3) : withoutSlashes; - return { - baseUrl: `${root}/v1`, - healthUrl: `${root}/health`, - modelsUrl: `${root}/v1/models`, - chatUrl: `${root}/v1/chat/completions`, - }; + return value; } -function readSecretReference(reference: string | undefined, env: NodeJS.ProcessEnv): string { - if (!reference) fail("config_missing", "VINCI_QWEN_SECRET_REF is required"); - let secret: string; - if (reference.startsWith("env:")) { - const name = reference.slice(4); - if (!ENV_NAME.test(name)) fail("config_invalid", "VINCI_QWEN_SECRET_REF env name is invalid"); - secret = env[name] ?? ""; - // Keep the resolved value only in the provider closure. Repository tools inherit this process - // environment, so leaving a dynamically named credential here would bypass the static key - // inventory even though the reference itself is scrubbed later. - delete env[name]; - } else if (reference.startsWith("file:")) { - const path = reference.slice(5); - if (!isAbsolute(path)) fail("config_invalid", "VINCI_QWEN_SECRET_REF file path must be absolute"); - let stat; - try { - stat = lstatSync(path); - } catch { - fail("credential_unavailable", "the referenced Qwen credential file is unavailable"); - } - if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || (stat.mode & 0o077) !== 0) { - fail("credential_unsafe", "the referenced Qwen credential must be a private regular file"); - } - secret = readFileSync(path, "utf8").trim(); - } else { - fail("config_invalid", "VINCI_QWEN_SECRET_REF must use env:NAME or file:/absolute/path"); - } - if (!secret || secret.length > 16_384 || /\s/.test(secret)) { - fail("credential_invalid", "the referenced Qwen credential is empty, oversized, or contains whitespace"); - } - return secret; +function timestamp(value: unknown, label: string): { text: string; time: number } { + if (typeof value !== "string" || !UTC_TIMESTAMP.test(value)) fail("qualification_invalid", `${label} must be an ISO-8601 UTC timestamp`); + const time = Date.parse(value); + if (!Number.isFinite(time)) fail("qualification_invalid", `${label} is not a valid timestamp`); + return { text: value, time }; } function validateRuntime(value: unknown): RuntimeTuple { exactKeys(value, ["engine", "version", "artifact_sha256", "arguments_sha256"], "runtime"); - for (const key of ["engine", "version"] as const) { - if (typeof value[key] !== "string" || !value[key]) fail("qualification_invalid", `runtime.${key} must be a non-empty string`); + const engine = nonEmptyString(value.engine, "runtime.engine"); + const version = nonEmptyString(value.version, "runtime.version"); + if (typeof value.artifact_sha256 !== "string" || !HEX64.test(value.artifact_sha256)) { + fail("qualification_invalid", "runtime.artifact_sha256 must be lowercase SHA-256"); } - for (const key of ["artifact_sha256", "arguments_sha256"] as const) { - if (typeof value[key] !== "string" || !HEX64.test(value[key])) fail("qualification_invalid", `runtime.${key} must be lowercase SHA-256`); + if (typeof value.arguments_sha256 !== "string" || !HEX64.test(value.arguments_sha256)) { + fail("qualification_invalid", "runtime.arguments_sha256 must be lowercase SHA-256"); } - return value as RuntimeTuple; + return { engine, version, artifact_sha256: value.artifact_sha256, arguments_sha256: value.arguments_sha256 }; } -function validateQualification(raw: unknown): Qualification { +function validateQualification(raw: unknown, expectedIssuer: string, nowMs: number): Qualification { exactKeys( raw, [ @@ -222,98 +298,411 @@ function validateQualification(raw: unknown): Qualification { "status", "authority_role", "fallback_policy", - "model", - "revision", - "runtime", - "endpoint_sha256", - "prompt_sha256", - "tools_sha256", + "safe_resume", + "provenance", + "burn_in", + "bindings", "capabilities", "limits", "pricing", + "requalification_conditions", ], "qualification", ); - if (raw.schema !== QUALIFICATION_SCHEMA || raw.status !== "qualified") fail("qualification_invalid", "qualification is not an admitted v1 qualified record"); + if (raw.schema !== QUALIFICATION_SCHEMA || raw.status !== "qualified") { + fail("qualification_invalid", "qualification is not an admitted v2 qualified record"); + } if (raw.authority_role !== AUTHORITY_ROLE) fail("authority_forbidden", "Qwen must remain non-authoritative"); if (raw.fallback_policy !== FALLBACK_POLICY) fail("fallback_forbidden", "fallback must be a separately authorized OpenRouter attempt"); - if (raw.model !== QWEN_MODEL) fail("model_mismatch", `qualification must name ${QWEN_MODEL}`); - if (typeof raw.revision !== "string" || !IMMUTABLE_REVISION.test(raw.revision)) { - fail("qualification_invalid", "revision must be an immutable lowercase 40- or 64-hex commit/digest"); + if (raw.safe_resume !== false) fail("safe_resume_forbidden", "safeResume remains false until independently requalified"); + + exactKeys(raw.provenance, ["issuer", "authority", "issued_at", "expires_at", "review_message_id", "review_body_sha256", "burn_in_report_sha256", "canary"], "provenance"); + if (raw.provenance.issuer !== expectedIssuer) fail("issuer_mismatch", "qualification issuer differs from the process-pinned issuer"); + if (raw.provenance.authority !== QUALIFICATION_AUTHORITY) fail("authority_forbidden", "qualification authority must be independent review"); + const issued = timestamp(raw.provenance.issued_at, "provenance.issued_at"); + const expires = timestamp(raw.provenance.expires_at, "provenance.expires_at"); + if (issued.time > nowMs + 5 * 60 * 1000 || expires.time <= nowMs || expires.time <= issued.time) { + fail("qualification_expired", "qualification issue/expiry interval is not currently valid"); + } + if (expires.time - issued.time > MAX_QUALIFICATION_LIFETIME_MS) { + fail("qualification_invalid", "qualification lifetime exceeds seven days"); } - const runtime = validateRuntime(raw.runtime); - for (const key of ["endpoint_sha256", "prompt_sha256", "tools_sha256"] as const) { - if (typeof raw[key] !== "string" || !HEX64.test(raw[key])) fail("qualification_invalid", `${key} must be lowercase SHA-256`); + nonEmptyString(raw.provenance.review_message_id, "provenance.review_message_id"); + if (typeof raw.provenance.review_body_sha256 !== "string" || !HEX64.test(raw.provenance.review_body_sha256)) { + fail("qualification_invalid", "provenance.review_body_sha256 must be lowercase SHA-256"); + } + if (typeof raw.provenance.burn_in_report_sha256 !== "string" || !HEX64.test(raw.provenance.burn_in_report_sha256)) { + fail("qualification_invalid", "provenance.burn_in_report_sha256 must be lowercase SHA-256"); + } + exactKeys(raw.provenance.canary, ["schema", "report_sha256", "observed_at"], "provenance.canary"); + if (raw.provenance.canary.schema !== CANARY_SCHEMA) fail("qualification_invalid", "qualification cites the wrong canary schema"); + if (typeof raw.provenance.canary.report_sha256 !== "string" || !HEX64.test(raw.provenance.canary.report_sha256)) { + fail("qualification_invalid", "provenance.canary.report_sha256 must be lowercase SHA-256"); + } + const canaryObserved = timestamp(raw.provenance.canary.observed_at, "provenance.canary.observed_at"); + if (canaryObserved.time > issued.time || issued.time - canaryObserved.time > MAX_CANARY_AGE_AT_ISSUE_MS) { + fail("qualification_invalid", "canary evidence must precede issuance by no more than 24 hours"); } - exactKeys(raw.capabilities, ["streaming_sse", "tool_calls", "structured_output"], "capabilities"); - if (raw.capabilities.streaming_sse !== true || raw.capabilities.tool_calls !== true) { - fail("capability_missing", "streaming SSE and structured tool calls must both be qualified"); + exactKeys( + raw.bindings, + [ + "model", + "revision", + "runtime", + "endpoint_sha256", + "endpoint_identity_sha256", + "work_order_prompt_sha256", + "system_prompt_sha256", + "tool_names_sha256", + "tool_schemas_sha256", + "tool_policy_sha256", + "client_build_sha256", + "extension_build_sha256", + "request_encoding_sha256", + ], + "bindings", + ); + if (raw.bindings.model !== QWEN_MODEL) fail("model_mismatch", `qualification must name ${QWEN_MODEL}`); + if (typeof raw.bindings.revision !== "string" || !IMMUTABLE_REVISION.test(raw.bindings.revision)) { + fail("qualification_invalid", "bindings.revision must be an immutable lowercase 40- or 64-hex digest"); + } + const runtime = validateRuntime(raw.bindings.runtime); + for (const key of [ + "endpoint_sha256", + "endpoint_identity_sha256", + "work_order_prompt_sha256", + "system_prompt_sha256", + "tool_names_sha256", + "tool_schemas_sha256", + "tool_policy_sha256", + "client_build_sha256", + "extension_build_sha256", + "request_encoding_sha256", + ] as const) { + if (typeof raw.bindings[key] !== "string" || !HEX64.test(raw.bindings[key])) { + fail("qualification_invalid", `bindings.${key} must be lowercase SHA-256`); + } + } + + exactKeys(raw.capabilities, ["streaming_sse", "tool_calls", "structured_output", "usage_chunk"], "capabilities"); + if (raw.capabilities.streaming_sse !== true || raw.capabilities.tool_calls !== true || raw.capabilities.usage_chunk !== true) { + fail("capability_missing", "streaming SSE, tool calls, and usage chunks must be independently qualified"); } if (raw.capabilities.structured_output !== "tool-arguments-json") { - fail("capability_missing", "the worker-required structured output is tool-arguments JSON"); + fail("capability_missing", "structured output must be tool-arguments JSON"); } - exactKeys(raw.limits, ["timeout_ms", "max_retries", "max_retry_delay_ms", "max_concurrency", "context_window", "max_tokens"], "limits"); + exactKeys( + raw.limits, + [ + "total_timeout_ms", + "max_retries", + "max_retry_delay_ms", + "max_concurrency", + "advertised_max_concurrency", + "context_window", + "max_tokens", + "max_request_bytes", + "max_response_bytes", + "max_error_bytes", + ], + "limits", + ); const limits = { - timeout_ms: boundedInteger(raw.limits.timeout_ms, 1_000, 300_000, "limits.timeout_ms"), + total_timeout_ms: boundedInteger(raw.limits.total_timeout_ms, 1_000, 300_000, "limits.total_timeout_ms"), max_retries: boundedInteger(raw.limits.max_retries, 0, 2, "limits.max_retries"), max_retry_delay_ms: boundedInteger(raw.limits.max_retry_delay_ms, 0, 30_000, "limits.max_retry_delay_ms"), - max_concurrency: boundedInteger(raw.limits.max_concurrency, 1, 8, "limits.max_concurrency"), + max_concurrency: boundedInteger(raw.limits.max_concurrency, 1, 32, "limits.max_concurrency"), + advertised_max_concurrency: boundedInteger(raw.limits.advertised_max_concurrency, 1, 32, "limits.advertised_max_concurrency"), context_window: boundedInteger(raw.limits.context_window, 8_192, 2_000_000, "limits.context_window"), max_tokens: boundedInteger(raw.limits.max_tokens, 256, 131_072, "limits.max_tokens"), + max_request_bytes: boundedInteger(raw.limits.max_request_bytes, 1_024, 16 * 1024 * 1024, "limits.max_request_bytes"), + max_response_bytes: boundedInteger(raw.limits.max_response_bytes, 1_024, 64 * 1024 * 1024, "limits.max_response_bytes"), + max_error_bytes: boundedInteger(raw.limits.max_error_bytes, 256, 256 * 1024, "limits.max_error_bytes"), }; if (limits.max_tokens > limits.context_window) fail("qualification_invalid", "limits.max_tokens exceeds limits.context_window"); + if (limits.max_concurrency > limits.advertised_max_concurrency) { + fail("qualification_invalid", "qualified concurrency exceeds Ayush's advertised ceiling"); + } + if (!QWEN_CONCURRENCY_LADDER.includes(limits.max_concurrency)) { + fail("qualification_invalid", "qualified concurrency is not on the closed 1→2→4→8→16→24→32 ladder"); + } + + exactKeys( + raw.burn_in, + [ + "schema", + "previous_concurrency", + "target_concurrency", + "observed_hours", + "work_orders", + "acceptance_pass_rate", + "usage_coverage_rate", + "transport_error_rate", + "identity_failures", + "verification_failures", + "circuit_opens", + "resource_alarms", + "governor_stops", + ], + "burn_in", + ); + if (raw.burn_in.schema !== "vinci.qwen-worker-burn-in.v1") fail("qualification_invalid", "burn-in report schema is not v1"); + const ladderIndex = QWEN_CONCURRENCY_LADDER.indexOf(limits.max_concurrency); + const burnIn = { + schema: raw.burn_in.schema, + previous_concurrency: boundedInteger(raw.burn_in.previous_concurrency, 0, 32, "burn_in.previous_concurrency"), + target_concurrency: boundedInteger(raw.burn_in.target_concurrency, 1, 32, "burn_in.target_concurrency"), + observed_hours: nonNegativeNumber(raw.burn_in.observed_hours, "burn_in.observed_hours"), + work_orders: boundedInteger(raw.burn_in.work_orders, 0, Number.MAX_SAFE_INTEGER, "burn_in.work_orders"), + acceptance_pass_rate: nonNegativeNumber(raw.burn_in.acceptance_pass_rate, "burn_in.acceptance_pass_rate"), + usage_coverage_rate: nonNegativeNumber(raw.burn_in.usage_coverage_rate, "burn_in.usage_coverage_rate"), + transport_error_rate: nonNegativeNumber(raw.burn_in.transport_error_rate, "burn_in.transport_error_rate"), + identity_failures: boundedInteger(raw.burn_in.identity_failures, 0, Number.MAX_SAFE_INTEGER, "burn_in.identity_failures"), + verification_failures: boundedInteger(raw.burn_in.verification_failures, 0, Number.MAX_SAFE_INTEGER, "burn_in.verification_failures"), + circuit_opens: boundedInteger(raw.burn_in.circuit_opens, 0, Number.MAX_SAFE_INTEGER, "burn_in.circuit_opens"), + resource_alarms: boundedInteger(raw.burn_in.resource_alarms, 0, Number.MAX_SAFE_INTEGER, "burn_in.resource_alarms"), + governor_stops: boundedInteger(raw.burn_in.governor_stops, 0, Number.MAX_SAFE_INTEGER, "burn_in.governor_stops"), + }; + if (burnIn.target_concurrency !== limits.max_concurrency) fail("qualification_invalid", "burn-in target differs from qualified concurrency"); + if (ladderIndex === 0) { + if (burnIn.previous_concurrency !== 0 || burnIn.observed_hours !== 0 || burnIn.work_orders !== 0) { + fail("qualification_invalid", "concurrency 1 is the zero-history entry stage"); + } + } else if ( + burnIn.previous_concurrency !== QWEN_CONCURRENCY_LADDER[ladderIndex - 1] || + burnIn.observed_hours < 168 || + burnIn.work_orders < 1_000 || + burnIn.acceptance_pass_rate !== 1 || + burnIn.usage_coverage_rate !== 1 || + burnIn.transport_error_rate > 0.005 || + burnIn.identity_failures !== 0 || + burnIn.verification_failures !== 0 || + burnIn.circuit_opens !== 0 || + burnIn.resource_alarms !== 0 || + burnIn.governor_stops !== 0 + ) { + fail("burn_in_gate_failed", "promotion requires the immediately prior stage, 168 hours, 1,000 WorkOrders, complete acceptance/usage, ≤0.5% transport errors, and zero stop triggers"); + } + if (burnIn.acceptance_pass_rate > 1 || burnIn.usage_coverage_rate > 1 || burnIn.transport_error_rate > 1) { + fail("qualification_invalid", "burn-in rates must be in [0, 1]"); + } - exactKeys(raw.pricing, ["input_per_million_usd", "output_per_million_usd", "cache_read_per_million_usd", "cache_write_per_million_usd"], "pricing"); + exactKeys( + raw.pricing, + ["currency", "basis", "input_per_million_usd", "output_per_million_usd", "cache_read_per_million_usd", "cache_write_per_million_usd"], + "pricing", + ); + if (raw.pricing.currency !== "USD") fail("qualification_invalid", "pricing.currency must be USD"); const pricing = { + currency: "USD", + basis: nonEmptyString(raw.pricing.basis, "pricing.basis", 1_024), input_per_million_usd: nonNegativeNumber(raw.pricing.input_per_million_usd, "pricing.input_per_million_usd"), output_per_million_usd: nonNegativeNumber(raw.pricing.output_per_million_usd, "pricing.output_per_million_usd"), cache_read_per_million_usd: nonNegativeNumber(raw.pricing.cache_read_per_million_usd, "pricing.cache_read_per_million_usd"), cache_write_per_million_usd: nonNegativeNumber(raw.pricing.cache_write_per_million_usd, "pricing.cache_write_per_million_usd"), }; - return { ...(raw as unknown as Qualification), runtime, limits, pricing }; + if (!Array.isArray(raw.requalification_conditions) || qwenCanonical(raw.requalification_conditions) !== qwenCanonical(QWEN_REQUALIFICATION_CONDITIONS)) { + fail("qualification_invalid", "requalification conditions must match the closed v2 contract"); + } + return { + ...(raw as unknown as Qualification), + burn_in: burnIn, + bindings: { ...(raw.bindings as Qualification["bindings"]), runtime }, + limits, + pricing, + requalification_conditions: [...QWEN_REQUALIFICATION_CONDITIONS], + }; } -function readQualification(env: NodeJS.ProcessEnv): { qualification: Qualification; digest: string } { - const path = env.VINCI_QWEN_QUALIFICATION_FILE; - const expectedDigest = env.VINCI_QWEN_QUALIFICATION_SHA256; - if (!path || !isAbsolute(path)) fail("config_missing", "VINCI_QWEN_QUALIFICATION_FILE must be an absolute path"); - if (!expectedDigest || !HEX64.test(expectedDigest)) fail("config_missing", "VINCI_QWEN_QUALIFICATION_SHA256 must pin the qualification bytes"); +function secureRegularFile(path: string, label: string, maximumBytes: number): Buffer { + if (!isAbsolute(path)) fail("config_invalid", `${label} must be an absolute path`); let stat; let bytes: Buffer; try { stat = lstatSync(path); bytes = readFileSync(path); } catch { - fail("qualification_unavailable", "the pinned qualification artifact is unavailable"); + fail("config_unavailable", `${label} is unavailable`); } if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || (stat.mode & 0o022) !== 0) { - fail("qualification_unsafe", "the qualification artifact must be a non-writable regular file"); + fail("config_unsafe", `${label} must be a non-writable regular file`); + } + if (bytes.length < 1 || bytes.length > maximumBytes) fail("config_invalid", `${label} has an invalid byte length`); + return bytes; +} + +function readSecretDescriptor(env: NodeJS.ProcessEnv): string { + const raw = env.VINCI_QWEN_SECRET_FD; + delete env.VINCI_QWEN_SECRET_FD; + const fd = Number(raw); + if (!Number.isSafeInteger(fd) || fd < 3 || fd > 64) fail("config_missing", "VINCI_QWEN_SECRET_FD must name a scoped inherited descriptor"); + let bytes: Buffer; + try { + const stat = fstatSync(fd); + if (!stat.isFile() || stat.nlink !== 1 || (stat.mode & 0o077) !== 0) fail("credential_unsafe", "Qwen credential descriptor is not a private regular file"); + bytes = readFileSync(fd); + } catch (error) { + if (error instanceof QwenReadinessError) throw error; + fail("credential_unavailable", "Qwen credential descriptor is unreadable"); + } finally { + try { + closeSync(fd); + } catch {} + } + if (bytes.length < 1 || bytes.length > 16_384) fail("credential_invalid", "Qwen credential has an invalid byte length"); + const secret = bytes.toString("utf8").trim(); + bytes.fill(0); + if (!secret || /\s/.test(secret)) fail("credential_invalid", "Qwen credential is empty or contains whitespace"); + return secret; +} + +function readCanarySecret(reference: string | undefined): string { + if (!reference?.startsWith("file:")) fail("config_invalid", "VINCI_QWEN_SECRET_REF must use file:/absolute/private/path"); + const bytes = secureRegularFile(reference.slice(5), "Qwen canary credential", 16_384); + const secret = bytes.toString("utf8").trim(); + bytes.fill(0); + if (!secret || /\s/.test(secret)) fail("credential_invalid", "Qwen canary credential is empty or contains whitespace"); + return secret; +} + +export function normalizeQwenBaseUrl(raw: string | undefined): { + baseUrl: string; + healthUrl: string; + modelsUrl: string; + chatUrl: string; + endpointHostname: string; + endpointLoopback: boolean; +} { + if (!raw) fail("config_missing", "VINCI_QWEN_BASE_URL is required"); + let parsed: URL; + try { + parsed = new URL(raw); + } catch { + fail("config_invalid", "VINCI_QWEN_BASE_URL must be an absolute URL"); + } + if (parsed.username || parsed.password || parsed.search || parsed.hash) { + fail("config_invalid", "VINCI_QWEN_BASE_URL may not contain credentials, a query, or a fragment"); + } + const endpointLoopback = parsed.hostname === "127.0.0.1" || parsed.hostname === "localhost" || parsed.hostname === "[::1]"; + if (parsed.protocol !== "https:" && !(parsed.protocol === "http:" && endpointLoopback)) { + fail("config_invalid", "VINCI_QWEN_BASE_URL must use HTTPS (HTTP is allowed only on loopback)"); + } + const withoutSlashes = parsed.toString().replace(/\/+$/, ""); + const root = withoutSlashes.endsWith("/v1") ? withoutSlashes.slice(0, -3) : withoutSlashes; + return { + baseUrl: `${root}/v1`, + healthUrl: `${root}/health`, + modelsUrl: `${root}/v1/models`, + chatUrl: `${root}/v1/chat/completions`, + endpointHostname: parsed.hostname.replace(/^\[|\]$/g, ""), + endpointLoopback, + }; +} + +function publicIpv4(address: string): boolean { + const parts = address.split(".").map(Number); + if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return false; + const [a, b, c] = parts; + if (a === 0 || a === 10 || a === 127 || a >= 224) return false; + if (a === 100 && b >= 64 && b <= 127) return false; + if (a === 169 && b === 254) return false; + if (a === 172 && b >= 16 && b <= 31) return false; + if (a === 192 && (b === 168 || (b === 0 && (c === 0 || c === 2)) || (b === 88 && c === 99) || (b === 175 && c === 48))) return false; + if (a === 198 && (b === 18 || b === 19 || (b === 51 && c === 100))) return false; + if (a === 203 && b === 0 && c === 113) return false; + return true; +} + +function loopbackIpv4(address: string): boolean { + return /^127\./.test(address); +} + +export async function pinQwenEndpoint(config: QwenRuntimeConfig, lookupImpl: QwenLookup = async (hostname) => dnsLookup(hostname, { all: true, verbatim: true })): Promise { + let answers: LookupAddress[]; + const family = isIP(config.endpointHostname); + if (family !== 0) answers = [{ address: config.endpointHostname, family }]; + else { + try { + answers = await lookupImpl(config.endpointHostname); + } catch { + fail("dns_unavailable", "endpoint hostname could not be resolved"); + } + } + if (answers.length < 1) fail("dns_unavailable", "endpoint hostname resolved to no addresses"); + if (answers.some((answer) => answer.family !== 4)) fail("ssrf_forbidden", "endpoint must resolve only to pinned IPv4 addresses"); + const addresses = [...new Set(answers.map((answer) => answer.address))].sort(); + if (config.endpointLoopback) { + if (addresses.some((address) => !loopbackIpv4(address))) fail("dns_rebinding", "loopback endpoint resolved outside loopback"); + } else if (addresses.some((address) => !publicIpv4(address))) { + fail("ssrf_forbidden", "endpoint resolved to a private, local, reserved, or non-public address"); } - if (sha256(bytes) !== expectedDigest) fail("qualification_digest_mismatch", "qualification bytes do not match the process pin"); + config.endpointAddresses = addresses; +} + +function readQualification(env: NodeJS.ProcessEnv, nowMs: number): { qualification: Qualification; digest: string } { + const path = env.VINCI_QWEN_QUALIFICATION_FILE; + const expectedDigest = env.VINCI_QWEN_QUALIFICATION_SHA256; + const publicKeyPath = env.VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_FILE; + const publicKeyDigest = env.VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_SHA256; + const expectedIssuer = env.VINCI_QWEN_QUALIFICATION_ISSUER; + if (!path || !expectedDigest || !HEX64.test(expectedDigest)) fail("config_missing", "qualification file and byte digest pin are required"); + if (!publicKeyPath || !publicKeyDigest || !HEX64.test(publicKeyDigest)) fail("config_missing", "qualification public key file and digest pin are required"); + if (!expectedIssuer || !IDENTIFIER.test(expectedIssuer)) fail("config_missing", "a bounded qualification issuer pin is required"); + const bytes = secureRegularFile(path, "Qwen qualification artifact", MAX_QUALIFICATION_BYTES); + if (qwenSha256(bytes) !== expectedDigest) fail("qualification_digest_mismatch", "qualification bytes do not match the process pin"); + const publicKeyBytes = secureRegularFile(publicKeyPath, "Qwen qualification public key", 64 * 1024); + if (qwenSha256(publicKeyBytes) !== publicKeyDigest) fail("qualification_key_mismatch", "qualification public key bytes do not match the process pin"); let parsed: unknown; try { parsed = JSON.parse(bytes.toString("utf8")); } catch { - fail("qualification_invalid", "qualification is not JSON"); + fail("qualification_invalid", "qualification artifact is not JSON"); + } + exactKeys(parsed, ["schema", "qualification", "signature"], "qualification envelope"); + if (parsed.schema !== QUALIFICATION_ENVELOPE_SCHEMA) fail("qualification_invalid", "qualification envelope schema is not v2"); + exactKeys(parsed.signature, ["algorithm", "key_id", "signature_base64"], "qualification signature"); + if (parsed.signature.algorithm !== "Ed25519") fail("qualification_invalid", "qualification signature algorithm must be Ed25519"); + nonEmptyString(parsed.signature.key_id, "qualification signature key_id"); + if (typeof parsed.signature.signature_base64 !== "string" || !/^[A-Za-z0-9+/]{86}==$/.test(parsed.signature.signature_base64)) { + fail("qualification_invalid", "qualification signature is not canonical base64 Ed25519"); + } + let publicKey; + try { + publicKey = createPublicKey(publicKeyBytes); + } catch { + fail("qualification_key_invalid", "qualification trust key is not valid SPKI/PEM"); } - return { qualification: validateQualification(parsed), digest: expectedDigest }; + if (publicKey.asymmetricKeyType !== "ed25519") fail("qualification_key_invalid", "qualification trust key must be Ed25519"); + const signedBytes = Buffer.from(qwenCanonical(parsed.qualification)); + if (!verify(null, signedBytes, publicKey, Buffer.from(parsed.signature.signature_base64, "base64"))) { + fail("qualification_signature_invalid", "qualification signature does not verify under the pinned trust key"); + } + return { qualification: validateQualification(parsed.qualification, expectedIssuer, nowMs), digest: expectedDigest }; } -export function loadQwenRuntimeConfig(env: NodeJS.ProcessEnv = process.env): QwenRuntimeConfig { +export function loadQwenRuntimeConfig(env: NodeJS.ProcessEnv = process.env, nowMs = Date.now()): QwenRuntimeConfig { const urls = normalizeQwenBaseUrl(env.VINCI_QWEN_BASE_URL); - const secretRef = env.VINCI_QWEN_SECRET_REF; - const secret = readSecretReference(secretRef, env); - const admitted = readQualification(env); + const secret = readSecretDescriptor(env); + const admitted = readQualification(env, nowMs); const qualification = admitted.qualification; - const expectedEndpoint = sha256(urls.baseUrl); - if (qualification.endpoint_sha256 !== expectedEndpoint) fail("endpoint_mismatch", "qualification is bound to a different base URL"); - if (qualification.prompt_sha256 !== env.VINCI_QWEN_PROMPT_SHA256) fail("prompt_mismatch", "task prompt is not the qualified prompt"); - if (qualification.tools_sha256 !== env.VINCI_QWEN_TOOLS_SHA256) fail("tools_mismatch", "task tools are not the qualified tools"); + const bindings = qualification.bindings; + if (bindings.endpoint_sha256 !== qwenSha256(urls.baseUrl)) fail("endpoint_mismatch", "qualification is bound to a different base URL"); + if (bindings.work_order_prompt_sha256 !== env.VINCI_QWEN_PROMPT_SHA256) fail("prompt_mismatch", "WorkOrder prompt is not the qualified prompt"); + if (bindings.tool_names_sha256 !== env.VINCI_QWEN_TOOLS_SHA256) fail("tools_mismatch", "ordered task tools are not qualified"); + if (bindings.tool_policy_sha256 !== env.VINCI_QWEN_TOOL_POLICY_SHA256) fail("tool_policy_mismatch", "task tool policy is not qualified"); + if (bindings.client_build_sha256 !== env.VINCI_QWEN_CLIENT_BUILD_SHA256) fail("client_build_mismatch", "executed client build is not qualified"); + if (bindings.extension_build_sha256 !== env.VINCI_QWEN_EXTENSION_BUILD_SHA256) fail("extension_build_mismatch", "executed Qwen extension build is not qualified"); + if (bindings.request_encoding_sha256 !== qwenSha256(qwenCanonical(QWEN_REQUEST_ENCODING))) { + fail("request_encoding_mismatch", "outbound OpenAI request encoding is not qualified"); + } if (env.VINCI_UNATTENDED_POLICY !== "governed" || !env.VINCI_UNATTENDED_LEASE) { - fail("authority_forbidden", "Qwen worker runs require a deterministic Governor lease"); + fail("authority_forbidden", "Qwen Worker runs require a deterministic Governor lease"); + } + if (qualification.limits.max_concurrency > 1) { + fail("fleet_permit_authority_missing", "concurrency above one requires the future fleet-wide permit authority interface"); } const workOrderId = env.VINCI_QWEN_WORK_ORDER_ID; const runId = env.VINCI_QWEN_RUN_ID; @@ -323,8 +712,8 @@ export function loadQwenRuntimeConfig(env: NodeJS.ProcessEnv = process.env): Qwe if (!circuitFile || !isAbsolute(circuitFile)) fail("config_missing", "VINCI_QWEN_CIRCUIT_FILE must be an absolute path"); return { ...urls, + endpointAddresses: [], secret, - secretRef: secretRef as string, qualification, qualificationSha256: admitted.digest, circuitFile, @@ -334,53 +723,90 @@ export function loadQwenRuntimeConfig(env: NodeJS.ProcessEnv = process.env): Qwe }; } -function emptyCircuit(): CircuitState { - return { schema: CIRCUIT_SCHEMA, failures: 0, open_until_ms: 0, last_reason: null }; +function emptyCircuit(sequence = 0): CircuitState { + return { schema: CIRCUIT_SCHEMA, failures: 0, open_until_ms: 0, last_reason: null, sequence }; } function readCircuit(path: string): CircuitState { if (!existsSync(path)) return emptyCircuit(); + let stat; + let value: unknown; try { - const stat = lstatSync(path); - const value = JSON.parse(readFileSync(path, "utf8")) as CircuitState; - if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || value.schema !== CIRCUIT_SCHEMA || !Number.isSafeInteger(value.failures) || value.failures < 0 || !Number.isSafeInteger(value.open_until_ms) || value.open_until_ms < 0) { - fail("circuit_invalid", "circuit state is malformed or unsafe"); - } - return value; - } catch (error) { - if (error instanceof QwenReadinessError) throw error; + stat = lstatSync(path); + value = JSON.parse(readFileSync(path, "utf8")); + } catch { fail("circuit_invalid", "circuit state is unreadable"); } + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || (stat.mode & 0o077) !== 0) { + fail("circuit_invalid", "circuit state is not a private regular file"); + } + exactKeys(value, ["schema", "failures", "open_until_ms", "last_reason", "sequence"], "circuit state"); + if (value.schema !== CIRCUIT_SCHEMA) fail("circuit_invalid", "circuit state has the wrong schema"); + const failures = boundedInteger(value.failures, 0, Number.MAX_SAFE_INTEGER, "circuit failures"); + const openUntil = boundedInteger(value.open_until_ms, 0, Number.MAX_SAFE_INTEGER, "circuit open_until_ms"); + const sequence = boundedInteger(value.sequence, 0, Number.MAX_SAFE_INTEGER, "circuit sequence"); + if (value.last_reason !== null && (typeof value.last_reason !== "string" || value.last_reason.length > 128)) { + fail("circuit_invalid", "circuit last_reason is malformed"); + } + return { schema: CIRCUIT_SCHEMA, failures, open_until_ms: openUntil, last_reason: value.last_reason as string | null, sequence }; } function writeCircuit(path: string, state: CircuitState): void { mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); - const temporary = `${path}.tmp-${process.pid}`; - writeFileSync(temporary, `${canonical(state)}\n`, { mode: 0o600 }); + const temporary = `${path}.tmp-${process.pid}-${randomBytes(8).toString("hex")}`; + writeFileSync(temporary, `${qwenCanonical(state)}\n`, { mode: 0o600, flag: "wx" }); renameSync(temporary, path); } +function withCircuitLock(path: string, operation: () => T): T { + const lock = `${path}.lock`; + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const deadline = Date.now() + LOCK_WAIT_MS; + while (true) { + try { + mkdirSync(lock, { mode: 0o700 }); + break; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") fail("circuit_lock_failed", "circuit lock cannot be acquired"); + if (Date.now() >= deadline) fail("circuit_busy", "concurrent circuit update did not complete within one second"); + Atomics.wait(SLEEP_CELL, 0, 0, LOCK_POLL_MS); + } + } + try { + return operation(); + } finally { + try { + rmdirSync(lock); + } catch { + fail("circuit_lock_failed", "circuit lock could not be released"); + } + } +} + export function assertQwenCircuitClosed(config: QwenRuntimeConfig, nowMs = Date.now()): void { const state = readCircuit(config.circuitFile); if (state.open_until_ms > nowMs) fail("circuit_open", `endpoint circuit is open until ${new Date(state.open_until_ms).toISOString()}`); } export function recordQwenCircuitOutcome(config: QwenRuntimeConfig, ok: boolean, reason: string, nowMs = Date.now()): void { - if (ok) { - writeCircuit(config.circuitFile, emptyCircuit()); - return; - } - const current = readCircuit(config.circuitFile); - const failures = current.failures + 1; - writeCircuit(config.circuitFile, { - schema: CIRCUIT_SCHEMA, - failures, - open_until_ms: failures >= config.circuitThreshold ? nowMs + config.circuitOpenMs : 0, - last_reason: reason.slice(0, 128), + withCircuitLock(config.circuitFile, () => { + const current = readCircuit(config.circuitFile); + if (ok) { + writeCircuit(config.circuitFile, emptyCircuit(current.sequence + 1)); + return; + } + const failures = current.failures + 1; + writeCircuit(config.circuitFile, { + schema: CIRCUIT_SCHEMA, + failures, + open_until_ms: failures >= config.circuitThreshold ? nowMs + config.circuitOpenMs : 0, + last_reason: reason.slice(0, 128), + sequence: current.sequence + 1, + }); }); } -async function readBoundedText(response: Response): Promise { +async function readBoundedText(response: Response, maximumBytes: number, abort?: AbortController): Promise { if (!response.body) return ""; const reader = response.body.getReader(); const chunks: Uint8Array[] = []; @@ -389,7 +815,10 @@ async function readBoundedText(response: Response): Promise { const next = await reader.read(); if (next.done) break; total += next.value.length; - if (total > MAX_RESPONSE_BYTES) fail("response_oversized", "endpoint response exceeded 256 KiB"); + if (total > maximumBytes) { + abort?.abort("response_oversized"); + fail("response_oversized", `endpoint response exceeded ${maximumBytes} bytes`); + } chunks.push(next.value); } const bytes = new Uint8Array(total); @@ -398,37 +827,11 @@ async function readBoundedText(response: Response): Promise { bytes.set(chunk, offset); offset += chunk.length; } - return new TextDecoder("utf-8", { fatal: true }).decode(bytes); -} - -async function request( - url: string, - init: RequestInit, - timeoutMs: number, - retries: number, - fetchImpl: typeof fetch, - signal?: AbortSignal, -): Promise { - let lastError: unknown; - for (let attempt = 0; attempt <= retries; attempt += 1) { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort("timeout"), timeoutMs); - const abort = () => controller.abort(signal?.reason ?? "cancelled"); - if (signal?.aborted) abort(); - else signal?.addEventListener("abort", abort, { once: true }); - try { - return await fetchImpl(url, { ...init, redirect: "error", signal: controller.signal }); - } catch (error) { - lastError = error; - if (signal?.aborted) fail("cancelled", "readiness probe was cancelled"); - if (attempt === retries) break; - } finally { - clearTimeout(timeout); - signal?.removeEventListener("abort", abort); - } + try { + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + fail("response_invalid", "endpoint response is not valid UTF-8"); } - const suffix = lastError instanceof Error && lastError.name === "AbortError" ? "timed out" : "failed"; - fail("endpoint_unavailable", `request ${suffix} after ${retries + 1} bounded attempt(s)`); } function authHeaders(config: Pick): Record { @@ -442,15 +845,69 @@ function authHeaders(config: Pick): }; } -function servedIdentity(response: Response, payload: unknown): { revision: string; runtime: RuntimeTuple } { +function pinnedAgent(config: QwenRuntimeConfig, addressIndex: number): Agent { + if (config.endpointAddresses.length < 1) fail("dns_unpinned", "endpoint DNS must be validated and pinned before transport"); + const address = config.endpointAddresses[addressIndex % config.endpointAddresses.length]; + return new Agent({ + connect: { + lookup: (_hostname, _options, callback) => callback(null, address, 4), + }, + }); +} + +function realPinnedFetch(config: QwenRuntimeConfig, addressIndex: number): { fetchImpl: QwenFetch; close: () => void } { + const agent = pinnedAgent(config, addressIndex); + return { + fetchImpl: async (input, init) => { + const target = new URL(typeof input === "string" || input instanceof URL ? input : input.url); + const requestInit = { ...(init as unknown as NonNullable[1]>), dispatcher: agent }; + return undiciFetch(target, requestInit) as unknown as Response; + }, + close: () => { + void agent.close(); + }, + }; +} + +async function boundedRequest( + config: QwenRuntimeConfig, + url: string, + init: RequestInit, + maximumBytes: number, + options: { fetchImpl?: QwenFetch; signal?: AbortSignal } = {}, +): Promise<{ response: Response; text: string }> { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort("total_timeout"), config.qualification.limits.total_timeout_ms); + const abort = () => controller.abort(options.signal?.reason ?? "cancelled"); + if (options.signal?.aborted) abort(); + else options.signal?.addEventListener("abort", abort, { once: true }); + const real = options.fetchImpl ? null : realPinnedFetch(config, 0); + try { + const response = await (options.fetchImpl ?? real!.fetchImpl)(url, { ...init, redirect: "error", signal: controller.signal }); + if (response.status >= 300 && response.status < 400) fail("redirect_forbidden", "endpoint redirects are refused"); + const text = await readBoundedText(response, maximumBytes, controller); + return { response, text }; + } catch (error) { + if (error instanceof QwenReadinessError) throw error; + if (options.signal?.aborted) fail("cancelled", "endpoint request was cancelled"); + if (controller.signal.aborted) fail("request_timeout", "endpoint request exceeded its total deadline"); + fail("endpoint_unavailable", "endpoint request failed"); + } finally { + clearTimeout(timeout); + options.signal?.removeEventListener("abort", abort); + real?.close(); + } + throw new Error("unreachable"); +} + +function servedIdentity(response: Response, payload: unknown): { revision: string; runtime: RuntimeTuple; endpointIdentity: string } { exactKeys(payload, ["object", "data"], "models response"); if (payload.object !== "list" || !Array.isArray(payload.data)) fail("models_invalid", "/v1/models must return an OpenAI list"); const matches = payload.data.filter((entry) => entry && typeof entry === "object" && (entry as Record).id === QWEN_MODEL); if (matches.length !== 1) fail("model_mismatch", `/v1/models must expose exactly one ${QWEN_MODEL}`); const model = matches[0] as Record; - const runtimeValue = model.runtime; - const runtime = runtimeValue && typeof runtimeValue === "object" - ? validateRuntime(runtimeValue) + const runtime = model.runtime && typeof model.runtime === "object" + ? validateRuntime(model.runtime) : validateRuntime({ engine: response.headers.get("x-vinci-runtime-engine"), version: response.headers.get("x-vinci-runtime-version"), @@ -459,16 +916,19 @@ function servedIdentity(response: Response, payload: unknown): { revision: strin }); const revision = typeof model.revision === "string" ? model.revision : response.headers.get("x-vinci-model-revision"); if (!revision || !IMMUTABLE_REVISION.test(revision)) fail("identity_missing", "/v1/models omitted the immutable served revision"); - return { revision, runtime }; + const endpointIdentity = typeof model.endpoint_identity_sha256 === "string" + ? model.endpoint_identity_sha256 + : response.headers.get("x-vinci-endpoint-identity-sha256"); + if (!endpointIdentity || !HEX64.test(endpointIdentity)) fail("identity_missing", "/v1/models omitted endpoint identity"); + return { revision, runtime, endpointIdentity }; } -async function validateHealthResponse(response: Response): Promise { +async function validateHealthResponse(response: Response, text: string): Promise { if (!response.ok) fail("health_failed", `authenticated /health returned ${response.status}`); - const healthText = await readBoundedText(response); - if (!healthText) fail("health_invalid", "/health returned an empty response"); + if (!text) fail("health_invalid", "/health returned an empty response"); let healthBody: unknown; try { - healthBody = JSON.parse(healthText); + healthBody = JSON.parse(text); } catch { fail("health_invalid", "/health returned non-JSON content"); } @@ -478,35 +938,35 @@ async function validateHealthResponse(response: Response): Promise { export async function probeQwenReadiness( config: QwenRuntimeConfig, - options: { fetchImpl?: typeof fetch; signal?: AbortSignal; nowMs?: number } = {}, -): Promise<{ revision: string; runtime: RuntimeTuple }> { - const fetchImpl = options.fetchImpl ?? fetch; + options: { fetchImpl?: QwenFetch; signal?: AbortSignal; nowMs?: number; lookupImpl?: QwenLookup } = {}, +): Promise<{ revision: string; runtime: RuntimeTuple; endpointIdentity: string }> { const nowMs = options.nowMs ?? Date.now(); assertQwenCircuitClosed(config, nowMs); - const timeoutMs = config.qualification.limits.timeout_ms; - const retries = config.qualification.limits.max_retries; + if (config.endpointAddresses.length < 1) await pinQwenEndpoint(config, options.lookupImpl); + const maximum = config.qualification.limits.max_response_bytes; try { - const health = await request(config.healthUrl, { headers: authHeaders(config) }, timeoutMs, retries, fetchImpl, options.signal); - await validateHealthResponse(health); - - const models = await request(config.modelsUrl, { headers: authHeaders(config) }, timeoutMs, retries, fetchImpl, options.signal); - if (!models.ok) fail("models_failed", `authenticated /v1/models returned ${models.status}`); - const modelsText = await readBoundedText(models); + const health = await boundedRequest(config, config.healthUrl, { headers: authHeaders(config) }, maximum, options); + await validateHealthResponse(health.response, health.text); + const models = await boundedRequest(config, config.modelsUrl, { headers: authHeaders(config) }, maximum, options); + if (!models.response.ok) fail("models_failed", `authenticated /v1/models returned ${models.response.status}`); let modelsBody: unknown; try { - modelsBody = JSON.parse(modelsText); + modelsBody = JSON.parse(models.text); } catch { fail("models_invalid", "/v1/models returned non-JSON content"); } - const identity = servedIdentity(models, modelsBody); - if (identity.revision !== config.qualification.revision || canonical(identity.runtime) !== canonical(config.qualification.runtime)) { - fail("runtime_mismatch", "served model revision/runtime differs from the qualification tuple"); + const identity = servedIdentity(models.response, modelsBody); + const bindings = config.qualification.bindings; + if ( + identity.revision !== bindings.revision || + qwenCanonical(identity.runtime) !== qwenCanonical(bindings.runtime) || + identity.endpointIdentity !== bindings.endpoint_identity_sha256 + ) { + fail("runtime_mismatch", "served endpoint/model/runtime differs from the signed qualification"); } - for (const [url, path] of [[config.healthUrl, "/health"], [config.modelsUrl, "/v1/models"]] as const) { - const anonymous = await request(url, { headers: { accept: "application/json" } }, timeoutMs, 0, fetchImpl, options.signal); - await readBoundedText(anonymous); - if (anonymous.status !== 401 && anonymous.status !== 403) { + const anonymous = await boundedRequest(config, url, { headers: { accept: "application/json" } }, maximum, options); + if (anonymous.response.status !== 401 && anonymous.response.status !== 403) { fail("auth_not_enforced", `unauthenticated ${path} was not refused`); } } @@ -522,42 +982,455 @@ export async function probeQwenReadiness( export async function ensureQwenReady( env: NodeJS.ProcessEnv = process.env, - options: { fetchImpl?: typeof fetch; signal?: AbortSignal; nowMs?: number } = {}, + options: { fetchImpl?: QwenFetch; signal?: AbortSignal; nowMs?: number; lookupImpl?: QwenLookup } = {}, ): Promise { - const config = loadQwenRuntimeConfig(env); + const config = loadQwenRuntimeConfig(env, options.nowMs); + await pinQwenEndpoint(config, options.lookupImpl); await probeQwenReadiness(config, options); return config; } -function canaryEndpointConfig(env: NodeJS.ProcessEnv): Pick { +function responseIdentityHeaders(response: Response, config: QwenRuntimeConfig): void { + const bindings = config.qualification.bindings; + const observed = { + model: response.headers.get("x-vinci-model-id"), + revision: response.headers.get("x-vinci-model-revision"), + endpoint: response.headers.get("x-vinci-endpoint-identity-sha256"), + runtime: { + engine: response.headers.get("x-vinci-runtime-engine"), + version: response.headers.get("x-vinci-runtime-version"), + artifact_sha256: response.headers.get("x-vinci-runtime-artifact-sha256"), + arguments_sha256: response.headers.get("x-vinci-runtime-arguments-sha256"), + }, + }; + if ( + observed.model !== QWEN_MODEL || + observed.revision !== bindings.revision || + observed.endpoint !== bindings.endpoint_identity_sha256 || + qwenCanonical(observed.runtime) !== qwenCanonical(bindings.runtime) + ) { + fail("response_identity_mismatch", "inference response headers do not match the signed model/runtime/endpoint identity"); + } +} + +function validateUsage(value: unknown): { input: number; output: number } { + exactKeys(value, ["prompt_tokens", "completion_tokens", "total_tokens", "prompt_tokens_details", "completion_tokens_details"], "stream usage"); + const prompt = boundedInteger(value.prompt_tokens, 0, Number.MAX_SAFE_INTEGER, "stream usage prompt_tokens"); + const completion = boundedInteger(value.completion_tokens, 0, Number.MAX_SAFE_INTEGER, "stream usage completion_tokens"); + const total = boundedInteger(value.total_tokens, 0, Number.MAX_SAFE_INTEGER, "stream usage total_tokens"); + if (total !== prompt + completion) fail("usage_invalid", "stream usage total_tokens does not equal prompt plus completion"); + if (value.prompt_tokens_details !== null && typeof value.prompt_tokens_details !== "object") fail("usage_invalid", "prompt token details are malformed"); + if (value.completion_tokens_details !== null && typeof value.completion_tokens_details !== "object") fail("usage_invalid", "completion token details are malformed"); + return { input: prompt, output: completion }; +} + +function validateSseData(data: string): { done: boolean; usage?: { input: number; output: number } } { + if (data === "[DONE]") return { done: true }; + let chunk: unknown; + try { + chunk = JSON.parse(data); + } catch { + fail("stream_invalid", "inference stream contains malformed JSON"); + } + if (!chunk || typeof chunk !== "object" || Array.isArray(chunk)) fail("stream_invalid", "inference stream chunk is not an object"); + const record = chunk as Record; + const allowed = new Set(["id", "object", "created", "model", "choices", "usage", "system_fingerprint", "service_tier"]); + if (Object.keys(record).some((key) => !allowed.has(key))) fail("stream_invalid", "inference stream chunk has an unexpected field"); + if (record.object !== "chat.completion.chunk" || record.model !== QWEN_MODEL || !Array.isArray(record.choices)) { + fail("response_identity_mismatch", "inference stream chunk does not identify the exact qualified model/object"); + } + if (record.usage !== undefined && record.usage !== null) { + return { done: false, usage: validateUsage(record.usage) }; + } + return { done: false }; +} + +function retryDelayMs(response: Response, maximum: number): number { + const raw = response.headers.get("retry-after"); + if (!raw) return Math.min(250, maximum); + const seconds = Number(raw); + const requested = Number.isFinite(seconds) && seconds >= 0 ? seconds * 1000 : Date.parse(raw) - Date.now(); + if (!Number.isFinite(requested) || requested < 0 || requested > maximum) { + fail("retry_delay_exceeded", "provider retry delay is invalid or exceeds the qualified cap"); + } + return requested; +} + +async function cancellableDelay(delayMs: number, signal: AbortSignal): Promise { + if (delayMs <= 0) return; + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal.removeEventListener("abort", abort); + resolve(); + }, delayMs); + const abort = () => { + clearTimeout(timer); + reject(new QwenReadinessError("cancelled", "retry delay was cancelled")); + }; + if (signal.aborted) abort(); + else signal.addEventListener("abort", abort, { once: true }); + }); +} + +function bodyByteLength(body: RequestInit["body"]): number { + if (body === undefined || body === null) return 0; + if (typeof body === "string") return Buffer.byteLength(body); + if (body instanceof URLSearchParams) return Buffer.byteLength(body.toString()); + if (body instanceof ArrayBuffer) return body.byteLength; + if (ArrayBuffer.isView(body)) return body.byteLength; + fail("request_invalid", "Qwen request body must be a bounded in-memory encoding"); +} + +function inferenceBody( + response: Response, + config: QwenRuntimeConfig, + abort: AbortController, + finish: (outcome: string, status: number | null, inputTokens?: number, outputTokens?: number) => void, +): ReadableStream { + if (!response.body) fail("stream_invalid", "successful inference response has no body"); + const reader = response.body.getReader(); + const decoder = new TextDecoder("utf-8", { fatal: true }); + let pending = ""; + let bytes = 0; + let doneSeen = false; + let usageSeen = false; + let inputTokens = 0; + let outputTokens = 0; + let settled = false; + const settle = (outcome: string, status: number | null) => { + if (settled) return; + settled = true; + finish(outcome, status); + }; + const inspect = (text: string, final: boolean) => { + pending += text; + const lines = pending.split(/\r?\n/); + const trailing = lines.pop() ?? ""; + pending = final ? "" : trailing; + for (const line of lines) { + if (!line || line.startsWith(":")) continue; + if (!line.startsWith("data:")) fail("stream_invalid", "inference stream contains a non-SSE field"); + if (doneSeen) fail("stream_invalid", "inference stream contains data after [DONE]"); + const result = validateSseData(line.slice(5).trim()); + if (result.usage && usageSeen) fail("usage_invalid", "inference stream contains more than one usage object"); + if (result.done && !usageSeen) fail("stream_invalid", "inference stream ended before its strict usage object"); + doneSeen = result.done; + if (result.usage) { + usageSeen = true; + inputTokens = result.usage.input; + outputTokens = result.usage.output; + } + } + if (final && trailing.trim()) fail("stream_invalid", "inference stream ended with a partial SSE line"); + }; + return new ReadableStream({ + async pull(controller) { + try { + const next = await reader.read(); + if (next.done) { + inspect(decoder.decode(), true); + if (!doneSeen || !usageSeen) fail("stream_invalid", "inference stream omitted [DONE] or its strict usage object"); + if (!settled) { + settled = true; + finish("success", response.status, inputTokens, outputTokens); + } + controller.close(); + return; + } + bytes += next.value.length; + if (bytes > config.qualification.limits.max_response_bytes) { + abort.abort("response_oversized"); + fail("response_oversized", "inference stream exceeded the signed response-byte bound"); + } + inspect(decoder.decode(next.value, { stream: true }), false); + controller.enqueue(next.value); + } catch (error) { + settle(error instanceof QwenReadinessError ? error.code : "stream_error", response.status); + controller.error(error); + } + }, + cancel() { + abort.abort("cancelled"); + settle("cancelled", response.status); + void reader.cancel(); + }, + }); +} + +export function createQwenInferenceFetch( + config: QwenRuntimeConfig, + requestId: string, + onAttempt: (record: QwenAttemptRecord) => void, + injectedFetch?: QwenFetch, +): QwenFetch { + return async (input, init = {}) => { + assertQwenCircuitClosed(config); + const target = new URL(typeof input === "string" || input instanceof URL ? input : input.url); + if (target.href !== config.chatUrl || (init.method ?? "GET").toUpperCase() !== "POST") { + fail("ssrf_forbidden", "inference transport may call only the exact qualified chat-completions URL"); + } + if (bodyByteLength(init.body) > config.qualification.limits.max_request_bytes) { + fail("request_oversized", "inference request exceeded the signed request-byte bound"); + } + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort("total_timeout"), config.qualification.limits.total_timeout_ms); + const externalSignal = init.signal; + const abort = () => controller.abort(externalSignal?.reason ?? "cancelled"); + if (externalSignal?.aborted) abort(); + else externalSignal?.addEventListener("abort", abort, { once: true }); + let timerOwnedByBody = false; + try { + for (let transportAttempt = 0; transportAttempt <= config.qualification.limits.max_retries; transportAttempt += 1) { + assertQwenCircuitClosed(config); + const started = Date.now(); + const startedAt = new Date(started).toISOString(); + const real = injectedFetch ? null : realPinnedFetch(config, transportAttempt); + const headers = new Headers(init.headers); + headers.set("x-vinci-idempotency-key", `${requestId}/${transportAttempt}`); + let attemptReported = false; + const finishAttempt = (outcome: string, status: number | null, inputTokens = 0, outputTokens = 0) => { + if (attemptReported) return; + attemptReported = true; + real?.close(); + const finished = Date.now(); + if (outcome === "success") recordQwenCircuitOutcome(config, true, "success", finished); + else if (outcome !== "cancelled") recordQwenCircuitOutcome(config, false, outcome, finished); + onAttempt({ + request_id: requestId, + transport_attempt: transportAttempt, + started_at: startedAt, + finished_at: new Date(finished).toISOString(), + latency_ms: finished - started, + outcome, + status, + cost_usd: outcome === "success" + ? (inputTokens * config.qualification.pricing.input_per_million_usd + outputTokens * config.qualification.pricing.output_per_million_usd) / 1_000_000 + : 0, + input_tokens: inputTokens, + output_tokens: outputTokens, + }); + }; + let response: Response; + try { + response = await (injectedFetch ?? real!.fetchImpl)(target, { + ...init, + headers, + redirect: "error", + signal: controller.signal, + }); + } catch (error) { + const outcome = externalSignal?.aborted ? "cancelled" : controller.signal.aborted ? "request_timeout" : "transport_error"; + finishAttempt(outcome, null); + if (externalSignal?.aborted) fail("cancelled", "inference request was cancelled"); + if (controller.signal.aborted) fail("request_timeout", "inference request exceeded its total deadline"); + if (transportAttempt === config.qualification.limits.max_retries) throw error; + await cancellableDelay(Math.min(250, config.qualification.limits.max_retry_delay_ms), controller.signal); + continue; + } + if (response.status >= 300 && response.status < 400) { + finishAttempt("redirect_forbidden", response.status); + fail("redirect_forbidden", "inference redirects are refused"); + } + if (!response.ok) { + try { + await readBoundedText(response, config.qualification.limits.max_error_bytes, controller); + } catch (error) { + finishAttempt(error instanceof QwenReadinessError ? error.code : "error_body_invalid", response.status); + throw error; + } + finishAttempt(`http_${response.status}`, response.status); + const retryable = response.status === 408 || response.status === 409 || response.status === 429 || response.status >= 500; + if (!retryable || transportAttempt === config.qualification.limits.max_retries) { + fail("http_status", `inference returned HTTP ${response.status}`); + } + await cancellableDelay(retryDelayMs(response, config.qualification.limits.max_retry_delay_ms), controller.signal); + continue; + } + try { + responseIdentityHeaders(response, config); + } catch (error) { + finishAttempt(error instanceof QwenReadinessError ? error.code : "response_identity_mismatch", response.status); + throw error; + } + if (!response.headers.get("content-type")?.toLowerCase().startsWith("text/event-stream")) { + finishAttempt("content_type_invalid", response.status); + fail("stream_invalid", "inference response is not text/event-stream"); + } + const finish = (outcome: string, status: number | null, inputTokens = 0, outputTokens = 0) => { + clearTimeout(timeout); + externalSignal?.removeEventListener("abort", abort); + finishAttempt(outcome, status, inputTokens, outputTokens); + }; + const body = inferenceBody(response, config, controller, finish); + timerOwnedByBody = true; + return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers }); + } + fail("endpoint_unavailable", "inference transport exhausted its bounded attempts"); + } finally { + if (!timerOwnedByBody) { + clearTimeout(timeout); + externalSignal?.removeEventListener("abort", abort); + } + } + }; +} + +export function assertQwenContextBindings(config: QwenRuntimeConfig, context: Context): void { + const bindings = config.qualification.bindings; + const workOrderMessage = context.messages.find((message) => message.role === "user"); + if (!workOrderMessage || typeof workOrderMessage.content !== "string" || qwenSha256(workOrderMessage.content) !== bindings.work_order_prompt_sha256) { + fail("prompt_mismatch", "first runtime user message is not the exact qualified WorkOrder prompt"); + } + if (qwenSha256(context.systemPrompt ?? "") !== bindings.system_prompt_sha256) { + fail("system_prompt_mismatch", "full assembled system prompt is not independently qualified"); + } + const tools = context.tools ?? []; + if (qwenSha256(qwenCanonical(tools.map((tool) => tool.name))) !== bindings.tool_names_sha256) { + fail("tools_mismatch", "runtime tool order differs from the signed qualification"); + } + if (qwenSha256(qwenCanonical(tools)) !== bindings.tool_schemas_sha256) { + fail("tool_schema_mismatch", "runtime tool schemas differ from the signed qualification"); + } +} + +export function validateQwenOutboundPayload(config: QwenRuntimeConfig, payload: unknown): void { + if (!payload || typeof payload !== "object" || Array.isArray(payload)) fail("request_invalid", "outbound request payload must be an object"); + const value = payload as Record; + const allowed = new Set([ + "model", + "messages", + "tools", + "tool_choice", + "stream", + "stream_options", + "temperature", + "max_tokens", + "reasoning_effort", + "chat_template_kwargs", + ]); + if (Object.entries(value).some(([key, field]) => field !== undefined && !allowed.has(key))) { + fail("request_invalid", "outbound request payload has an unqualified field"); + } + if (value.model !== QWEN_MODEL || value.stream !== true || !Array.isArray(value.messages)) { + fail("request_invalid", "outbound request must stream the exact qualified model and messages"); + } + if (!Array.isArray(value.tools)) fail("request_invalid", "outbound request must carry the qualified tool schemas"); + if (typeof value.max_tokens !== "number" || !Number.isSafeInteger(value.max_tokens) || value.max_tokens < 1 || value.max_tokens > config.qualification.limits.max_tokens) { + fail("request_invalid", "outbound max_tokens exceeds the signed bound"); + } + const streamOptions = value.stream_options; + if (!streamOptions || typeof streamOptions !== "object" || (streamOptions as Record).include_usage !== true) { + fail("request_invalid", "outbound stream must request usage telemetry"); + } +} + +function canaryConfig(env: NodeJS.ProcessEnv): QwenRuntimeConfig { const urls = normalizeQwenBaseUrl(env.VINCI_QWEN_BASE_URL); + const secret = readCanarySecret(env.VINCI_QWEN_SECRET_REF); return { ...urls, - secret: readSecretReference(env.VINCI_QWEN_SECRET_REF, env), - attribution: { - workOrderId: "canary-read-only", - runId: "canary-read-only", - attemptId: "canary-read-only/1", + endpointAddresses: [], + secret, + qualification: { + schema: QUALIFICATION_SCHEMA, + status: "qualified", + authority_role: AUTHORITY_ROLE, + fallback_policy: FALLBACK_POLICY, + safe_resume: false, + provenance: { + issuer: "canary-only", + authority: QUALIFICATION_AUTHORITY, + issued_at: new Date().toISOString(), + expires_at: new Date(Date.now() + 1_000).toISOString(), + review_message_id: "canary-only", + review_body_sha256: "0".repeat(64), + burn_in_report_sha256: "0".repeat(64), + canary: { schema: CANARY_SCHEMA, report_sha256: "0".repeat(64), observed_at: new Date().toISOString() }, + }, + burn_in: { + schema: "vinci.qwen-worker-burn-in.v1", + previous_concurrency: 0, + target_concurrency: 1, + observed_hours: 0, + work_orders: 0, + acceptance_pass_rate: 1, + usage_coverage_rate: 1, + transport_error_rate: 0, + identity_failures: 0, + verification_failures: 0, + circuit_opens: 0, + resource_alarms: 0, + governor_stops: 0, + }, + bindings: { + model: QWEN_MODEL, + revision: "0".repeat(40), + runtime: { engine: "canary", version: "canary", artifact_sha256: "0".repeat(64), arguments_sha256: "0".repeat(64) }, + endpoint_sha256: qwenSha256(urls.baseUrl), + endpoint_identity_sha256: "0".repeat(64), + work_order_prompt_sha256: "0".repeat(64), + system_prompt_sha256: "0".repeat(64), + tool_names_sha256: "0".repeat(64), + tool_schemas_sha256: "0".repeat(64), + tool_policy_sha256: "0".repeat(64), + client_build_sha256: "0".repeat(64), + extension_build_sha256: "0".repeat(64), + request_encoding_sha256: qwenSha256(qwenCanonical(QWEN_REQUEST_ENCODING)), + }, + capabilities: { streaming_sse: true, tool_calls: true, structured_output: "tool-arguments-json", usage_chunk: true }, + limits: { + total_timeout_ms: boundedInteger(Number(env.VINCI_QWEN_CANARY_TIMEOUT_MS ?? "30000"), 1_000, 300_000, "VINCI_QWEN_CANARY_TIMEOUT_MS"), + max_retries: 0, + max_retry_delay_ms: 0, + max_concurrency: 1, + advertised_max_concurrency: 1, + context_window: 8_192, + max_tokens: 64, + max_request_bytes: 64 * 1024, + max_response_bytes: MAX_CANARY_BYTES, + max_error_bytes: 64 * 1024, + }, + pricing: { + currency: "USD", + basis: "canary-only", + input_per_million_usd: 0, + output_per_million_usd: 0, + cache_read_per_million_usd: 0, + cache_write_per_million_usd: 0, + }, + requalification_conditions: [...QWEN_REQUALIFICATION_CONDITIONS], }, + qualificationSha256: "0".repeat(64), + circuitFile: "/canary/unused", + circuitThreshold: 1, + circuitOpenMs: 1_000, + attribution: { workOrderId: "canary-read-only", runId: "canary-read-only", attemptId: "canary-read-only/1" }, }; } -export async function runQwenCanary(env: NodeJS.ProcessEnv = process.env, fetchImpl: typeof fetch = fetch): Promise> { - const config = canaryEndpointConfig(env); - const timeoutMs = boundedInteger(Number(env.VINCI_QWEN_CANARY_TIMEOUT_MS ?? "30000"), 1_000, 300_000, "VINCI_QWEN_CANARY_TIMEOUT_MS"); +export async function runQwenCanary( + env: NodeJS.ProcessEnv = process.env, + fetchImpl?: QwenFetch, + lookupImpl?: QwenLookup, +): Promise> { + const config = canaryConfig(env); + await pinQwenEndpoint(config, lookupImpl); + const maximum = config.qualification.limits.max_error_bytes; const started = Date.now(); - const health = await request(config.healthUrl, { headers: authHeaders(config) }, timeoutMs, 0, fetchImpl); - await validateHealthResponse(health); - const models = await request(config.modelsUrl, { headers: authHeaders(config) }, timeoutMs, 0, fetchImpl); - if (!models.ok) fail("models_failed", `authenticated /v1/models returned ${models.status}`); - const identity = servedIdentity(models, JSON.parse(await readBoundedText(models))); + const health = await boundedRequest(config, config.healthUrl, { headers: authHeaders(config) }, maximum, { fetchImpl }); + await validateHealthResponse(health.response, health.text); + const models = await boundedRequest(config, config.modelsUrl, { headers: authHeaders(config) }, maximum, { fetchImpl }); + if (!models.response.ok) fail("models_failed", `authenticated /v1/models returned ${models.response.status}`); + const identity = servedIdentity(models.response, JSON.parse(models.text)); + config.qualification.bindings.revision = identity.revision; + config.qualification.bindings.runtime = identity.runtime; + config.qualification.bindings.endpoint_identity_sha256 = identity.endpointIdentity; for (const [url, path] of [[config.healthUrl, "/health"], [config.modelsUrl, "/v1/models"]] as const) { - const anonymous = await request(url, { headers: { accept: "application/json" } }, timeoutMs, 0, fetchImpl); - await readBoundedText(anonymous); - if (anonymous.status !== 401 && anonymous.status !== 403) fail("auth_not_enforced", `unauthenticated ${path} was not refused`); + const anonymous = await boundedRequest(config, url, { headers: { accept: "application/json" } }, maximum, { fetchImpl }); + if (anonymous.response.status !== 401 && anonymous.response.status !== 403) fail("auth_not_enforced", `unauthenticated ${path} was not refused`); } - - const response = await request( + const response = await boundedRequest( + config, config.chatUrl, { method: "POST", @@ -589,29 +1462,33 @@ export async function runQwenCanary(env: NodeJS.ProcessEnv = process.env, fetchI tool_choice: { type: "function", function: { name: "report_ready" } }, }), }, - timeoutMs, - 0, - fetchImpl, + MAX_CANARY_BYTES, + { fetchImpl }, ); - if (!response.ok) fail("canary_failed", `streaming tool-call inference returned ${response.status}`); - if (!response.headers.get("content-type")?.toLowerCase().startsWith("text/event-stream")) { + if (!response.response.ok) fail("canary_failed", `streaming tool-call inference returned ${response.response.status}`); + responseIdentityHeaders(response.response, config); + if (!response.response.headers.get("content-type")?.toLowerCase().startsWith("text/event-stream")) { fail("canary_invalid", "streaming tool-call inference did not return text/event-stream"); } - const stream = await readBoundedText(response); let toolName = ""; let argumentsText = ""; let usageSeen = false; - for (const line of stream.split(/\r?\n/)) { + let doneSeen = false; + for (const line of response.text.split(/\r?\n/)) { if (!line.startsWith("data:")) continue; const data = line.slice(5).trim(); - if (!data || data === "[DONE]") continue; + if (!data) continue; + if (doneSeen) fail("canary_invalid", "stream contained data after [DONE]"); + const validation = validateSseData(data); + doneSeen = validation.done; + usageSeen ||= validation.usage !== undefined; + if (validation.done) continue; let chunk: Record; try { chunk = JSON.parse(data) as Record; } catch { fail("canary_invalid", "stream contained malformed JSON"); } - if (chunk.usage && typeof chunk.usage === "object") usageSeen = true; const choices = Array.isArray(chunk.choices) ? chunk.choices : []; for (const choice of choices) { const delta = choice && typeof choice === "object" ? (choice as Record).delta : null; @@ -634,21 +1511,25 @@ export async function runQwenCanary(env: NodeJS.ProcessEnv = process.env, fetchI } catch { fail("canary_invalid", "tool-call arguments were not complete JSON"); } - if (toolName !== "report_ready" || canonical(argumentsValue) !== canonical({ status: "ready" })) { - fail("canary_invalid", "stream did not return the required structured tool call"); + if (toolName !== "report_ready" || qwenCanonical(argumentsValue) !== qwenCanonical({ status: "ready" }) || !usageSeen || !doneSeen) { + fail("canary_invalid", "stream omitted the required structured tool call, strict usage, or [DONE]"); } - if (!usageSeen) fail("canary_invalid", "stream omitted the usage chunk required for token telemetry"); return { - schema: "vinci.qwen-worker-canary.v1", + schema: CANARY_SCHEMA, + observed_at: new Date().toISOString(), + endpoint_sha256: qwenSha256(config.baseUrl), + endpoint_identity_sha256: identity.endpointIdentity, + pinned_addresses_sha256: qwenSha256(qwenCanonical(config.endpointAddresses)), model: QWEN_MODEL, revision: identity.revision, runtime: identity.runtime, authenticated: true, anonymous_refused: true, - capabilities: { streaming_sse: true, tool_calls: true, structured_output: "tool-arguments-json", usage_chunk: usageSeen }, + capabilities: { streaming_sse: true, tool_calls: true, structured_output: "tool-arguments-json", usage_chunk: true }, latency_ms: Date.now() - started, authority_role: AUTHORITY_ROLE, fallback_policy: FALLBACK_POLICY, + safe_resume: false, }; } @@ -658,18 +1539,31 @@ function requiredEnv(env: NodeJS.ProcessEnv, name: string): string { return value; } -export function buildQwenQualificationTemplate(env: NodeJS.ProcessEnv = process.env): Qualification { - if (env.VINCI_QWEN_ADMIT !== "qualified") { - fail("qualification_not_admitted", "VINCI_QWEN_ADMIT=qualified is required after independent canary review"); - } +export function buildQwenQualificationRequest(env: NodeJS.ProcessEnv = process.env): Record { const urls = normalizeQwenBaseUrl(env.VINCI_QWEN_BASE_URL); - const promptFile = requiredEnv(env, "VINCI_QWEN_QUALIFICATION_PROMPT_FILE"); - if (!isAbsolute(promptFile)) fail("config_invalid", "VINCI_QWEN_QUALIFICATION_PROMPT_FILE must be absolute"); - let prompt: string; + const readInput = (name: string, maximum: number) => secureRegularFile(requiredEnv(env, name), name, maximum); + const prompt = readInput("VINCI_QWEN_QUALIFICATION_PROMPT_FILE", 1024 * 1024); + const systemPrompt = readInput("VINCI_QWEN_QUALIFICATION_SYSTEM_PROMPT_FILE", 4 * 1024 * 1024); + const toolSchemas = readInput("VINCI_QWEN_QUALIFICATION_TOOL_SCHEMAS_FILE", 4 * 1024 * 1024); + const canaryBytes = readInput("VINCI_QWEN_CANARY_REPORT_FILE", MAX_CANARY_BYTES); + const burnInBytes = readInput("VINCI_QWEN_BURN_IN_REPORT_FILE", MAX_BURN_IN_BYTES); + let canary: unknown; + try { + canary = JSON.parse(canaryBytes.toString("utf8")); + } catch { + fail("config_invalid", "canary report is not JSON"); + } + if (!canary || typeof canary !== "object" || (canary as Record).schema !== CANARY_SCHEMA || (canary as Record).model !== QWEN_MODEL) { + fail("config_invalid", "canary report does not describe the v2 exact-model canary"); + } + let burnIn: unknown; try { - prompt = readFileSync(promptFile, "utf8"); + burnIn = JSON.parse(burnInBytes.toString("utf8")); } catch { - fail("config_invalid", "qualification prompt file is unreadable"); + fail("config_invalid", "burn-in report is not JSON"); + } + if (!burnIn || typeof burnIn !== "object" || (burnIn as Record).schema !== "vinci.qwen-worker-burn-in.v1") { + fail("config_invalid", "burn-in report does not describe the v1 numeric gate"); } let tools: unknown; try { @@ -677,12 +1571,14 @@ export function buildQwenQualificationTemplate(env: NodeJS.ProcessEnv = process. } catch { fail("config_invalid", "VINCI_QWEN_QUALIFICATION_TOOLS must be a JSON array"); } - if (!Array.isArray(tools) || tools.length === 0 || !tools.every((tool) => typeof tool === "string" && tool.length > 0)) { - fail("config_invalid", "VINCI_QWEN_QUALIFICATION_TOOLS must be a non-empty array of tool names"); + if (!Array.isArray(tools) || tools.length < 1 || !tools.every((tool) => typeof tool === "string" && tool.length > 0)) { + fail("config_invalid", "VINCI_QWEN_QUALIFICATION_TOOLS must be a non-empty ordered string array"); } - const revision = requiredEnv(env, "VINCI_QWEN_SERVED_REVISION"); - if (!IMMUTABLE_REVISION.test(revision)) { - fail("config_invalid", "VINCI_QWEN_SERVED_REVISION must be an immutable lowercase 40- or 64-hex commit/digest"); + let parsedToolSchemas: unknown; + try { + parsedToolSchemas = JSON.parse(toolSchemas.toString("utf8")); + } catch { + fail("config_invalid", "VINCI_QWEN_QUALIFICATION_TOOL_SCHEMAS_FILE must contain JSON"); } const runtime = validateRuntime({ engine: requiredEnv(env, "VINCI_QWEN_RUNTIME_ENGINE"), @@ -690,56 +1586,101 @@ export function buildQwenQualificationTemplate(env: NodeJS.ProcessEnv = process. artifact_sha256: requiredEnv(env, "VINCI_QWEN_RUNTIME_ARTIFACT_SHA256"), arguments_sha256: requiredEnv(env, "VINCI_QWEN_RUNTIME_ARGUMENTS_SHA256"), }); + const revision = requiredEnv(env, "VINCI_QWEN_SERVED_REVISION"); + if (!IMMUTABLE_REVISION.test(revision)) fail("config_invalid", "served revision must be immutable 40- or 64-hex"); const numberEnv = (name: string) => Number(requiredEnv(env, name)); - return validateQualification({ - schema: QUALIFICATION_SCHEMA, - status: "qualified", - authority_role: AUTHORITY_ROLE, - fallback_policy: FALLBACK_POLICY, - model: QWEN_MODEL, - revision, - runtime, - endpoint_sha256: sha256(urls.baseUrl), - prompt_sha256: sha256(prompt), - tools_sha256: sha256(canonical(tools)), - capabilities: { - streaming_sse: true, - tool_calls: true, - structured_output: "tool-arguments-json", - }, - limits: { - timeout_ms: Number(env.VINCI_QWEN_TIMEOUT_MS ?? "120000"), - max_retries: Number(env.VINCI_QWEN_MAX_RETRIES ?? "1"), - max_retry_delay_ms: Number(env.VINCI_QWEN_MAX_RETRY_DELAY_MS ?? "5000"), - max_concurrency: Number(env.VINCI_QWEN_MAX_CONCURRENCY ?? "1"), - context_window: numberEnv("VINCI_QWEN_CONTEXT_WINDOW"), - max_tokens: numberEnv("VINCI_QWEN_MAX_TOKENS"), - }, - pricing: { - input_per_million_usd: numberEnv("VINCI_QWEN_INPUT_PER_MILLION_USD"), - output_per_million_usd: numberEnv("VINCI_QWEN_OUTPUT_PER_MILLION_USD"), - cache_read_per_million_usd: numberEnv("VINCI_QWEN_CACHE_READ_PER_MILLION_USD"), - cache_write_per_million_usd: numberEnv("VINCI_QWEN_CACHE_WRITE_PER_MILLION_USD"), + const toolPolicy = { + ordered_tools: tools, + unattended_policy: "governed", + authority: "Governor", + safe_resume: false, + }; + return { + schema: QUALIFICATION_REQUEST_SCHEMA, + candidate: { + model: QWEN_MODEL, + revision, + runtime, + endpoint_sha256: qwenSha256(urls.baseUrl), + endpoint_identity_sha256: requiredEnv(env, "VINCI_QWEN_ENDPOINT_IDENTITY_SHA256"), + work_order_prompt_sha256: qwenSha256(prompt), + system_prompt_sha256: qwenSha256(systemPrompt), + tool_names_sha256: qwenSha256(qwenCanonical(tools)), + tool_schemas_sha256: qwenSha256(qwenCanonical(parsedToolSchemas)), + tool_policy_sha256: qwenSha256(qwenCanonical(toolPolicy)), + client_build_sha256: requiredEnv(env, "VINCI_QWEN_CLIENT_BUILD_SHA256"), + extension_build_sha256: requiredEnv(env, "VINCI_QWEN_EXTENSION_BUILD_SHA256"), + request_encoding_sha256: qwenSha256(qwenCanonical(QWEN_REQUEST_ENCODING)), + canary_report_sha256: qwenSha256(canaryBytes), + canary_observed_at: (canary as Record).observed_at, + burn_in_report_sha256: qwenSha256(burnInBytes), + burn_in: burnIn, + capabilities: (canary as Record).capabilities, + limits: { + total_timeout_ms: Number(env.VINCI_QWEN_TOTAL_TIMEOUT_MS ?? "120000"), + max_retries: Number(env.VINCI_QWEN_MAX_RETRIES ?? "1"), + max_retry_delay_ms: Number(env.VINCI_QWEN_MAX_RETRY_DELAY_MS ?? "5000"), + max_concurrency: Number(env.VINCI_QWEN_MAX_CONCURRENCY ?? "1"), + advertised_max_concurrency: numberEnv("VINCI_QWEN_ADVERTISED_MAX_CONCURRENCY"), + context_window: numberEnv("VINCI_QWEN_CONTEXT_WINDOW"), + max_tokens: numberEnv("VINCI_QWEN_MAX_TOKENS"), + max_request_bytes: Number(env.VINCI_QWEN_MAX_REQUEST_BYTES ?? String(4 * 1024 * 1024)), + max_response_bytes: Number(env.VINCI_QWEN_MAX_RESPONSE_BYTES ?? String(16 * 1024 * 1024)), + max_error_bytes: Number(env.VINCI_QWEN_MAX_ERROR_BYTES ?? String(64 * 1024)), + }, + pricing: { + currency: "USD", + basis: requiredEnv(env, "VINCI_QWEN_PRICE_BASIS"), + input_per_million_usd: numberEnv("VINCI_QWEN_INPUT_PER_MILLION_USD"), + output_per_million_usd: numberEnv("VINCI_QWEN_OUTPUT_PER_MILLION_USD"), + cache_read_per_million_usd: numberEnv("VINCI_QWEN_CACHE_READ_PER_MILLION_USD"), + cache_write_per_million_usd: numberEnv("VINCI_QWEN_CACHE_WRITE_PER_MILLION_USD"), + }, + requalification_conditions: QWEN_REQUALIFICATION_CONDITIONS, + safe_resume: false, + authority_role: AUTHORITY_ROLE, + fallback_policy: FALLBACK_POLICY, }, - }); + }; } export function scrubQwenBootstrapEnvironment(env: NodeJS.ProcessEnv = process.env): void { - for (const name of ["VINCI_QWEN_SECRET_REF", "VINCI_QWEN_QUALIFICATION_FILE", "VINCI_QWEN_QUALIFICATION_SHA256"]) delete env[name]; + for (const name of [ + "VINCI_QWEN_QUALIFICATION_FILE", + "VINCI_QWEN_QUALIFICATION_SHA256", + "VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_FILE", + "VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_SHA256", + "VINCI_QWEN_QUALIFICATION_ISSUER", + ]) delete env[name]; +} + +export function qwenProviderHeaders(config: QwenRuntimeConfig, requestId: string): ProviderHeaders { + return { + "x-vinci-work-order-id": config.attribution.workOrderId, + "x-vinci-run-id": config.attribution.runId, + "x-vinci-attempt-id": config.attribution.attemptId, + "x-vinci-qwen-request-id": requestId, + "x-vinci-qwen-output-authority": "non-authoritative", + "x-vinci-qwen-qualification-sha256": config.qualificationSha256, + }; +} + +export function qwenModelWithOpenAiApi(model: Model): Model<"openai-completions"> { + return { ...model, api: "openai-completions" } as Model<"openai-completions">; } const invokedPath = process.argv[1] ? pathToFileURL(process.argv[1]).href : ""; if (import.meta.url === invokedPath) { if (process.argv.includes("--canary")) { runQwenCanary() - .then((report) => process.stdout.write(`${canonical(report)}\n`)) + .then((report) => process.stdout.write(`${qwenCanonical(report)}\n`)) .catch((error) => { process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); process.exitCode = 1; }); - } else if (process.argv.includes("--qualification-template")) { + } else if (process.argv.includes("--qualification-request")) { try { - process.stdout.write(`${canonical(buildQwenQualificationTemplate())}\n`); + process.stdout.write(`${qwenCanonical(buildQwenQualificationRequest())}\n`); } catch (error) { process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); process.exitCode = 1; diff --git a/vinci/extensions/vinci-provider.ts b/vinci/extensions/vinci-provider.ts index 9750e697e..c282c2e5d 100644 --- a/vinci/extensions/vinci-provider.ts +++ b/vinci/extensions/vinci-provider.ts @@ -1,23 +1,5 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; -import { - type Api, - createAssistantMessageEventStream, - type Context, - type Model, - type SimpleStreamOptions, -} from "@earendil-works/pi-ai"; -import { streamSimple as streamOpenAICompletions } from "@earendil-works/pi-ai/api/openai-completions"; import type { OAuthCredentials, OAuthLoginCallbacks } from "@earendil-works/pi-ai/compat"; -import { - assertQwenCircuitClosed, - ensureQwenReady, - QWEN_API, - QWEN_MODEL, - QWEN_PROVIDER, - recordQwenCircuitOutcome, - scrubQwenBootstrapEnvironment, - type QwenRuntimeConfig, -} from "./lib/qwen-runtime.ts"; import { setVinciConnection } from "./lib/ui-state.ts"; import { VINCI_BILLING_URL, VINCI_GATEWAY_BASE_URL, VINCI_PLATFORM_BASE_URL } from "./vinci-links.ts"; @@ -152,101 +134,6 @@ function vinciClassModel(id: string, name: string) { }; } -function qwenProviderConfig(runtime: QwenRuntimeConfig) { - let inFlight = 0; - return { - name: "Qwen 3.8 27B (Vinci H200, non-authoritative)", - baseUrl: runtime.baseUrl, - // A non-secret sentinel satisfies provider registration. The resolved credential is held only - // in this closure and replaces this value at the actual OpenAI-compatible request boundary. - apiKey: "runtime-resolved-secret-reference", - api: QWEN_API, - streamSimple(model: Model, context: Context, options?: SimpleStreamOptions) { - assertQwenCircuitClosed(runtime); - if (inFlight >= runtime.qualification.limits.max_concurrency) { - throw new Error("qwen_concurrency_exceeded: the qualified single-request bound is already in use"); - } - inFlight += 1; - let source; - try { - source = streamOpenAICompletions( - { ...model, api: "openai-completions" } as Model<"openai-completions">, - context, - { - ...options, - apiKey: runtime.secret, - timeoutMs: runtime.qualification.limits.timeout_ms, - maxRetries: runtime.qualification.limits.max_retries, - maxRetryDelayMs: runtime.qualification.limits.max_retry_delay_ms, - }, - ); - } catch (error) { - inFlight -= 1; - throw error; - } - const bounded = createAssistantMessageEventStream(); - void (async () => { - try { - for await (const event of source) bounded.push(event); - } catch (error) { - recordQwenCircuitOutcome(runtime, false, "stream_error"); - const message = { - role: "assistant" as const, - content: [], - api: model.api, - provider: model.provider, - model: model.id, - usage: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 0, - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, - }, - stopReason: "error" as const, - errorMessage: error instanceof Error ? error.message : String(error), - timestamp: Date.now(), - }; - bounded.push({ type: "error", reason: "error", error: message }); - bounded.end(message); - } finally { - inFlight -= 1; - } - })(); - return bounded; - }, - models: [ - { - id: QWEN_MODEL, - name: "Qwen 3.8 27B (qualified, non-authoritative)", - reasoning: true, - thinkingLevelMap: { off: "off", minimal: "low", low: "low", medium: "medium", high: "high", xhigh: "high" }, - input: ["text"] as Array<"text">, - contextWindow: runtime.qualification.limits.context_window, - maxTokens: runtime.qualification.limits.max_tokens, - cost: { - input: runtime.qualification.pricing.input_per_million_usd, - output: runtime.qualification.pricing.output_per_million_usd, - cacheRead: runtime.qualification.pricing.cache_read_per_million_usd, - cacheWrite: runtime.qualification.pricing.cache_write_per_million_usd, - }, - compat: { - supportsStore: false, - supportsDeveloperRole: false, - supportsReasoningEffort: false, - supportsUsageInStreaming: true, - maxTokensField: "max_tokens" as const, - requiresToolResultName: true, - supportsStrictMode: false, - supportsLongCacheRetention: false, - thinkingFormat: "qwen-chat-template" as const, - }, - }, - ], - }; -} - /** * Compose a terminal message for a Vinci billing refusal. * Accepts either a full error body (for structured codes) or just the error message text (for backward compat). @@ -431,44 +318,6 @@ export default async function (pi: ExtensionAPI) { }); } - let qwenRuntime: QwenRuntimeConfig | undefined; - if (process.env.VINCI_QWEN_SELECTED === "1") { - qwenRuntime = await ensureQwenReady(); - pi.registerProvider(QWEN_PROVIDER, qwenProviderConfig(qwenRuntime)); - scrubQwenBootstrapEnvironment(); - - pi.on("before_provider_headers", (event, ctx) => { - if (ctx.model?.provider !== QWEN_PROVIDER || !qwenRuntime) return; - event.headers["x-vinci-work-order-id"] = qwenRuntime.attribution.workOrderId; - event.headers["x-vinci-run-id"] = qwenRuntime.attribution.runId; - event.headers["x-vinci-attempt-id"] = qwenRuntime.attribution.attemptId; - event.headers["x-vinci-qwen-output-authority"] = "non-authoritative"; - event.headers["x-vinci-qwen-qualification-sha256"] = qwenRuntime.qualificationSha256; - }); - - pi.on("after_provider_response", (event, ctx) => { - if (ctx.model?.provider !== QWEN_PROVIDER || !qwenRuntime) return; - recordQwenCircuitOutcome(qwenRuntime, event.status >= 200 && event.status < 300, `http_${event.status}`); - }); - - pi.on("message_end", (event, ctx) => { - if (event.message.role !== "assistant" || event.message.provider !== QWEN_PROVIDER || !qwenRuntime) return; - pi.appendEntry("vinci-qwen-output-label", { - authority: "non-authoritative", - independent_check_required: true, - model: QWEN_MODEL, - revision: qwenRuntime.qualification.revision, - runtime: qwenRuntime.qualification.runtime, - qualification_sha256: qwenRuntime.qualificationSha256, - work_order_id: qwenRuntime.attribution.workOrderId, - run_id: qwenRuntime.attribution.runId, - attempt_id: qwenRuntime.attribution.attemptId, - outcome: event.message.stopReason, - session_id: ctx.sessionManager.getSessionId(), - }); - }); - } - pi.on("after_provider_response", (event, ctx) => { if (ctx.model?.provider !== "vinci") return; if (event.status >= 200 && event.status < 300) { diff --git a/vinci/extensions/vinci-qwen-provider.ts b/vinci/extensions/vinci-qwen-provider.ts new file mode 100644 index 000000000..51a7b6012 --- /dev/null +++ b/vinci/extensions/vinci-qwen-provider.ts @@ -0,0 +1,230 @@ +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { + type Api, + createAssistantMessageEventStream, + type Context, + type Model, + type SimpleStreamOptions, +} from "@earendil-works/pi-ai"; +import { streamSimpleOpenAICompletions } from "@earendil-works/pi-ai/compat"; +import { + assertQwenCircuitClosed, + assertQwenContextBindings, + createQwenInferenceFetch, + ensureQwenReady, + QWEN_API, + QWEN_MODEL, + QWEN_PROVIDER, + qwenModelWithOpenAiApi, + qwenProviderHeaders, + qwenSha256, + scrubQwenBootstrapEnvironment, + type QwenAttemptRecord, + type QwenRuntimeConfig, + validateQwenOutboundPayload, +} from "./lib/qwen-runtime.ts"; + +function validUsage(message: { usage?: unknown }): boolean { + if (!message.usage || typeof message.usage !== "object") return false; + const usage = message.usage as Record; + const allowedUsage = new Set(["input", "output", "cacheRead", "cacheWrite", "cacheWrite1h", "reasoning", "totalTokens", "cost"]); + if (Object.keys(usage).some((key) => !allowedUsage.has(key))) return false; + for (const key of ["input", "output", "cacheRead", "cacheWrite", "totalTokens"] as const) { + if (typeof usage[key] !== "number" || !Number.isSafeInteger(usage[key]) || usage[key] < 0) return false; + } + for (const key of ["cacheWrite1h", "reasoning"] as const) { + if (usage[key] !== undefined && (typeof usage[key] !== "number" || !Number.isSafeInteger(usage[key]) || usage[key] < 0)) return false; + } + const countedTokens = (usage.input as number) + (usage.output as number) + (usage.cacheRead as number) + (usage.cacheWrite as number); + if (usage.totalTokens !== countedTokens) return false; + if (typeof usage.reasoning === "number" && usage.reasoning > (usage.output as number)) return false; + if (!usage.cost || typeof usage.cost !== "object") return false; + const cost = usage.cost as Record; + if (Object.keys(cost).sort().join("\0") !== ["cacheRead", "cacheWrite", "input", "output", "total"].join("\0")) return false; + for (const value of Object.values(cost)) { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return false; + } + const costParts = (cost.input as number) + (cost.output as number) + (cost.cacheRead as number) + (cost.cacheWrite as number); + return Math.abs((cost.total as number) - costParts) <= Number.EPSILON * Math.max(1, costParts) * 8; +} + +export function qwenProviderConfig( + runtime: QwenRuntimeConfig, + streamOpenAI = streamSimpleOpenAICompletions, + onAttempt: (record: QwenAttemptRecord) => void = () => {}, +) { + let inFlight = 0; + let requestOrdinal = 0; + return { + name: "Qwen 3.8 27B (Vinci H200, non-authoritative)", + baseUrl: runtime.baseUrl, + // This sentinel is not a credential. The descriptor-resolved secret remains in this closure + // and replaces the sentinel only at the governed OpenAI-compatible request boundary. + apiKey: "runtime-resolved-secret-descriptor", + api: QWEN_API, + streamSimple(model: Model, context: Context, options?: SimpleStreamOptions) { + assertQwenCircuitClosed(runtime); + assertQwenContextBindings(runtime, context); + if (inFlight >= runtime.qualification.limits.max_concurrency) { + throw new Error("qwen_concurrency_exceeded: the qualified single-request bound is already in use"); + } + inFlight += 1; + let released = false; + const release = () => { + if (released) return; + released = true; + inFlight -= 1; + }; + requestOrdinal += 1; + const requestId = qwenSha256(`${runtime.attribution.workOrderId}\0${runtime.attribution.runId}\0${runtime.attribution.attemptId}\0${requestOrdinal}`); + let source; + try { + source = streamOpenAI( + qwenModelWithOpenAiApi(model), + context, + { + ...options, + apiKey: runtime.secret, + headers: { ...options?.headers, ...qwenProviderHeaders(runtime, requestId) }, + timeoutMs: runtime.qualification.limits.total_timeout_ms, + maxRetries: 0, + fetch: createQwenInferenceFetch(runtime, requestId, onAttempt), + onPayload: async (payload, requestModel) => { + const candidate = await options?.onPayload?.(payload, requestModel); + const finalPayload = candidate ?? payload; + validateQwenOutboundPayload(runtime, finalPayload); + return finalPayload; + }, + } as SimpleStreamOptions & { fetch: typeof globalThis.fetch }, + ); + } catch (error) { + release(); + throw error; + } + const bounded = createAssistantMessageEventStream(); + void (async () => { + let terminalSeen = false; + try { + for await (const event of source) { + if (event.type === "done" || event.type === "error") { + terminalSeen = true; + const message = event.type === "done" ? event.message : event.error; + if (message.provider !== QWEN_PROVIDER || message.model !== QWEN_MODEL || !validUsage(message)) { + throw new Error("qwen_usage_invalid: terminal response lacks exact model identity or strict usage/cost telemetry"); + } + // Release before the terminal event can resolve result() or become observable to a + // caller that immediately starts the next qualified request. + release(); + } + bounded.push(event); + } + if (!terminalSeen) throw new Error("qwen_stream_truncated: provider stream ended without a terminal event"); + } catch (error) { + release(); + const message = { + role: "assistant" as const, + content: [], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "error" as const, + errorMessage: (error instanceof Error ? error.message : String(error)).slice(0, 4096), + timestamp: Date.now(), + }; + bounded.push({ type: "error", reason: "error", error: message }); + bounded.end(message); + } finally { + release(); + } + })(); + return bounded; + }, + models: [ + { + id: QWEN_MODEL, + name: "Qwen 3.8 27B (qualified, non-authoritative)", + reasoning: true, + thinkingLevelMap: { off: "off", minimal: "low", low: "low", medium: "medium", high: "high", xhigh: "high" }, + input: ["text"] as Array<"text">, + contextWindow: runtime.qualification.limits.context_window, + maxTokens: runtime.qualification.limits.max_tokens, + cost: { + input: runtime.qualification.pricing.input_per_million_usd, + output: runtime.qualification.pricing.output_per_million_usd, + cacheRead: runtime.qualification.pricing.cache_read_per_million_usd, + cacheWrite: runtime.qualification.pricing.cache_write_per_million_usd, + }, + compat: { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + supportsUsageInStreaming: true, + maxTokensField: "max_tokens" as const, + requiresToolResultName: true, + supportsStrictMode: false, + supportsLongCacheRetention: false, + thinkingFormat: "qwen-chat-template" as const, + }, + }, + ], + }; +} + +export default async function (pi: ExtensionAPI) { + if (process.env.VINCI_QWEN_SELECTED !== "1") { + throw new Error("qwen_not_selected: the Qwen extension may load only for a Worker-selected Qwen attempt"); + } + let runtime: QwenRuntimeConfig; + try { + runtime = await ensureQwenReady(); + } finally { + scrubQwenBootstrapEnvironment(); + } + pi.registerProvider( + QWEN_PROVIDER, + qwenProviderConfig(runtime, streamSimpleOpenAICompletions, (record) => { + pi.appendEntry("vinci-qwen-transport-attempt", { + ...record, + work_order_id: runtime.attribution.workOrderId, + run_id: runtime.attribution.runId, + attempt_id: runtime.attribution.attemptId, + qualification_sha256: runtime.qualificationSha256, + }); + }), + ); + + pi.on("before_provider_headers", (event, ctx) => { + if (ctx.model?.provider !== QWEN_PROVIDER) return; + event.headers["x-vinci-work-order-id"] = runtime.attribution.workOrderId; + event.headers["x-vinci-run-id"] = runtime.attribution.runId; + event.headers["x-vinci-attempt-id"] = runtime.attribution.attemptId; + event.headers["x-vinci-qwen-output-authority"] = "non-authoritative"; + event.headers["x-vinci-qwen-qualification-sha256"] = runtime.qualificationSha256; + }); + + pi.on("message_end", (event, ctx) => { + if (event.message.role !== "assistant" || event.message.provider !== QWEN_PROVIDER) return; + pi.appendEntry("vinci-qwen-output-label", { + authority: "non-authoritative", + independent_check_required: true, + model: QWEN_MODEL, + revision: runtime.qualification.bindings.revision, + runtime: runtime.qualification.bindings.runtime, + qualification_sha256: runtime.qualificationSha256, + work_order_id: runtime.attribution.workOrderId, + run_id: runtime.attribution.runId, + attempt_id: runtime.attribution.attemptId, + outcome: event.message.stopReason, + usage: event.message.usage, + session_id: ctx.sessionManager.getSessionId(), + }); + }); +} diff --git a/vinci/test/fixtures/qwen-loader-probe.ts b/vinci/test/fixtures/qwen-loader-probe.ts new file mode 100644 index 000000000..cc862cffc --- /dev/null +++ b/vinci/test/fixtures/qwen-loader-probe.ts @@ -0,0 +1,25 @@ +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { registerFauxProvider } from "@earendil-works/pi-ai/compat"; +import { qwenProviderConfig } from "../../extensions/vinci-qwen-provider.ts"; + +const registration = registerFauxProvider({ tokensPerSecond: 1_000 }); + +export default function (pi: ExtensionAPI) { + if (typeof qwenProviderConfig !== "function") throw new Error("qwen_loader_probe_missing_export"); + pi.registerProvider("qwen-loader-probe", { + name: "Qwen loader probe", + baseUrl: "http://localhost:0", + apiKey: "loader-probe-not-a-secret", + api: registration.api, + models: registration.models.map((model) => ({ + id: model.id, + name: model.name, + reasoning: model.reasoning, + input: model.input, + cost: model.cost, + contextWindow: model.contextWindow, + maxTokens: model.maxTokens, + })), + }); + pi.on("session_shutdown", () => registration.unregister()); +} diff --git a/vinci/test/worker-qwen-loader-startup.mjs b/vinci/test/worker-qwen-loader-startup.mjs new file mode 100644 index 000000000..98327400f --- /dev/null +++ b/vinci/test/worker-qwen-loader-startup.mjs @@ -0,0 +1,100 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +const root = resolve(import.meta.dirname, "../.."); +const launcher = join(root, "vinci/bin/vinci"); +const providerExtension = join(root, "vinci/test/fixtures/checkpoint-faux-provider.ts"); +const qwenLoaderProbe = join(root, "vinci/test/fixtures/qwen-loader-probe.ts"); +const launcherSource = readFileSync(launcher, "utf8"); +assert.match(launcherSource, /qwen-h200\)[\s\S]*VINCI_QWEN_SELECTED[\s\S]*vinci-qwen-provider\.ts/); +assert.doesNotMatch(readFileSync(join(root, "vinci/extensions/vinci-provider.ts"), "utf8"), /qwen-runtime|vinci-qwen-provider/); + +function runInPty(args, options) { + return new Promise((resolveRun, rejectRun) => { + const child = spawn("/usr/bin/script", ["-q", "/dev/null", "bash", launcher, ...args], { + cwd: options.cwd, + env: options.env, + stdio: ["ignore", "pipe", "pipe"], + }); + let output = ""; + const append = (chunk) => { + output += chunk; + if (output.length > 1_000_000) output = output.slice(-1_000_000); + }; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", append); + child.stderr.on("data", append); + const timer = setTimeout(() => { + child.kill("SIGKILL"); + rejectRun(new Error(`PTY loader regression timed out:\n${output.slice(-4_000)}`)); + }, 15_000); + child.once("error", (error) => { + clearTimeout(timer); + rejectRun(error); + }); + child.once("exit", (code, signal) => { + clearTimeout(timer); + resolveRun({ code, signal, output }); + }); + }); +} + +const temp = mkdtempSync(join(tmpdir(), "vinci-qwen-loader-")); +try { + const run = await runInPty( + [ + "--extension", + providerExtension, + "--list-models", + "faux", + ], + { + cwd: temp, + env: { + ...process.env, + HOME: join(temp, "home"), + VINCI_CHECKPOINT_KILL_MARKER: "", + VINCI_INTERNAL_PROVIDER_TEST: "1", + VINCI_MODEL: "faux-1", + VINCI_NO_BOOTSTRAP_HEAL: "1", + VINCI_NO_RESUME: "1", + VINCI_NO_VERIFY: "1", + VINCI_PROVIDER: "faux", + VINCI_TOOL_BOOTSTRAP: "0", + }, + }, + ); + assert.equal(run.signal, null, run.output.slice(-4_000)); + assert.equal(run.code, 0, run.output.slice(-4_000)); + assert.doesNotMatch(run.output, /ERR_MODULE_NOT_FOUND|Package subpath .* is not defined by "exports"/); + assert.match(run.output, /faux-1/); + + const qwenLoader = await runInPty( + ["--extension", qwenLoaderProbe, "--list-models", "qwen-loader-probe"], + { + cwd: temp, + env: { + ...process.env, + HOME: join(temp, "home-qwen-loader"), + VINCI_INTERNAL_PROVIDER_TEST: "1", + VINCI_MODEL: "faux-1", + VINCI_NO_BOOTSTRAP_HEAL: "1", + VINCI_NO_RESUME: "1", + VINCI_NO_VERIFY: "1", + VINCI_PROVIDER: "faux", + VINCI_TOOL_BOOTSTRAP: "0", + }, + }, + ); + assert.doesNotMatch(qwenLoader.output, /ERR_MODULE_NOT_FOUND|Package subpath .* is not defined by "exports"/); + assert.match(qwenLoader.output, /qwen-loader-probe/); + +} finally { + rmSync(temp, { recursive: true, force: true }); +} + +process.stdout.write("PASS worker-qwen-loader-startup real loader, checkpoint, and PTY startup\n"); diff --git a/vinci/test/worker-qwen-provider.mjs b/vinci/test/worker-qwen-provider.mjs index 407be13c9..aea3e92cf 100644 --- a/vinci/test/worker-qwen-provider.mjs +++ b/vinci/test/worker-qwen-provider.mjs @@ -1,45 +1,233 @@ import assert from "node:assert/strict"; -import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { spawn } from "node:child_process"; +import { generateKeyPairSync, sign } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, openSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; -import { createHash } from "node:crypto"; +import { pathToFileURL } from "node:url"; +import { createAssistantMessageEventStream } from "@earendil-works/pi-ai"; import * as runtime from "../extensions/lib/qwen-runtime.ts"; -import providerExtension from "../extensions/vinci-provider.ts"; +import { qwenProviderConfig } from "../extensions/vinci-qwen-provider.ts"; import * as cleanroom from "../worker/cleanroom.mjs"; import * as digest from "../worker/contracts/digest.mjs"; import * as economics from "../worker/economics.mjs"; -import * as workerRun from "../worker/run.mjs"; +import { runVinci } from "../worker/run.mjs"; +import { TaskLifecycle } from "../worker/task.mjs"; const root = resolve(import.meta.dirname, "../.."); - const temp = mkdtempSync(join(tmpdir(), "vinci-qwen-test-")); const secretFile = join(temp, "secret"); const promptFile = join(temp, "prompt.txt"); +const systemPromptFile = join(temp, "system.txt"); +const toolSchemasFile = join(temp, "tools.json"); +const canaryFile = join(temp, "canary.json"); +const burnInFile = join(temp, "burn-in.json"); const qualificationFile = join(temp, "qualification.json"); -writeFileSync(secretFile, "test-secret\n", { mode: 0o600 }); -writeFileSync(promptFile, "inspect the bounded fixture\n", { mode: 0o600 }); +const publicKeyFile = join(temp, "qualification-key.pem"); +const endpointIdentity = "12".repeat(32); +const revision = "ab".repeat(20); +const runtimeTuple = { + engine: "vllm", + version: "0.10.2", + artifact_sha256: "cd".repeat(32), + arguments_sha256: "ef".repeat(32), +}; +const systemPrompt = "bounded governed system prompt"; +const workOrderPrompt = "inspect the bounded fixture"; +const tools = [ + { name: "read", description: "Read a file", parameters: { type: "object", properties: { path: { type: "string" } }, required: ["path"] } }, + { name: "grep", description: "Search text", parameters: { type: "object", properties: { pattern: { type: "string" } }, required: ["pattern"] } }, +]; +const nowMs = Date.parse("2026-09-04T18:00:00.000Z"); + +async function exerciseConcurrentBreakerWrites(circuitFile, count) { + const workerFile = join(temp, "breaker-worker.mjs"); + const startFile = join(temp, "breaker-start"); + const runtimeUrl = pathToFileURL(join(root, "vinci/extensions/lib/qwen-runtime.ts")).href; + writeFileSync(workerFile, `import { existsSync, writeFileSync } from "node:fs"; +import { recordQwenCircuitOutcome } from ${JSON.stringify(runtimeUrl)}; +const [circuitFile, readyFile, startFile] = process.argv.slice(2); +writeFileSync(readyFile, "ready"); +const sleep = new Int32Array(new SharedArrayBuffer(4)); +while (!existsSync(startFile)) Atomics.wait(sleep, 0, 0, 5); +recordQwenCircuitOutcome({ circuitFile, circuitThreshold: 100, circuitOpenMs: 60_000 }, false, "concurrent_500"); +`, { mode: 0o600 }); + const children = Array.from({ length: count }, (_, index) => { + const readyFile = join(temp, `breaker-ready-${index}`); + const child = spawn(process.execPath, [...process.execArgv, workerFile, circuitFile, readyFile, startFile], { + stdio: ["ignore", "pipe", "pipe"], + }); + return { child, readyFile }; + }); + await new Promise((resolveReady, rejectReady) => { + const deadline = Date.now() + 5_000; + const poll = () => { + if (children.every(({ readyFile }) => existsSync(readyFile))) return resolveReady(); + if (Date.now() >= deadline) return rejectReady(new Error("breaker workers did not become ready")); + setTimeout(poll, 5); + }; + poll(); + }); + writeFileSync(startFile, "start", { mode: 0o600 }); + await Promise.all(children.map(({ child }) => new Promise((resolveExit, rejectExit) => { + let stderr = ""; + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk) => { stderr += chunk; }); + child.on("error", rejectExit); + child.on("exit", (code) => code === 0 ? resolveExit() : rejectExit(new Error(`breaker worker exited ${code}: ${stderr}`))); + }))); +} + +writeFileSync(secretFile, "synthetic-test-secret\n", { mode: 0o600 }); +writeFileSync(promptFile, workOrderPrompt, { mode: 0o400 }); +writeFileSync(systemPromptFile, systemPrompt, { mode: 0o400 }); +writeFileSync(toolSchemasFile, `${JSON.stringify(tools)}\n`, { mode: 0o400 }); +const canary = { + schema: "vinci.qwen-worker-canary.v2", + observed_at: "2026-09-04T16:00:00.000Z", + endpoint_sha256: runtime.qwenSha256("https://qwen.example.test/v1"), + endpoint_identity_sha256: endpointIdentity, + model: runtime.QWEN_MODEL, + revision, + runtime: runtimeTuple, + capabilities: { streaming_sse: true, tool_calls: true, structured_output: "tool-arguments-json", usage_chunk: true }, + safe_resume: false, +}; +writeFileSync(canaryFile, `${runtime.qwenCanonical(canary)}\n`, { mode: 0o400 }); +const entryBurnIn = { + schema: "vinci.qwen-worker-burn-in.v1", + previous_concurrency: 0, + target_concurrency: 1, + observed_hours: 0, + work_orders: 0, + acceptance_pass_rate: 1, + usage_coverage_rate: 1, + transport_error_rate: 0, + identity_failures: 0, + verification_failures: 0, + circuit_opens: 0, + resource_alarms: 0, + governor_stops: 0, +}; +writeFileSync(burnInFile, `${runtime.qwenCanonical(entryBurnIn)}\n`, { mode: 0o400 }); -const hex = (pair) => pair.repeat(32); -const sha256 = (value) => createHash("sha256").update(value).digest("hex"); -const baseEnv = { - VINCI_QWEN_ADMIT: "qualified", +const { privateKey, publicKey } = generateKeyPairSync("ed25519"); +const publicKeyBytes = publicKey.export({ type: "spki", format: "pem" }); +writeFileSync(publicKeyFile, publicKeyBytes, { mode: 0o400 }); + +const requestEnv = { VINCI_QWEN_BASE_URL: "https://qwen.example.test/v1", - VINCI_QWEN_SECRET_REF: `file:${secretFile}`, VINCI_QWEN_QUALIFICATION_PROMPT_FILE: promptFile, + VINCI_QWEN_QUALIFICATION_SYSTEM_PROMPT_FILE: systemPromptFile, + VINCI_QWEN_QUALIFICATION_TOOL_SCHEMAS_FILE: toolSchemasFile, VINCI_QWEN_QUALIFICATION_TOOLS: '["read","grep"]', - VINCI_QWEN_SERVED_REVISION: hex("ab"), - VINCI_QWEN_RUNTIME_ENGINE: "vllm", - VINCI_QWEN_RUNTIME_VERSION: "0.10.2", - VINCI_QWEN_RUNTIME_ARTIFACT_SHA256: hex("cd"), - VINCI_QWEN_RUNTIME_ARGUMENTS_SHA256: hex("ef"), + VINCI_QWEN_CANARY_REPORT_FILE: canaryFile, + VINCI_QWEN_BURN_IN_REPORT_FILE: burnInFile, + VINCI_QWEN_SERVED_REVISION: revision, + VINCI_QWEN_RUNTIME_ENGINE: runtimeTuple.engine, + VINCI_QWEN_RUNTIME_VERSION: runtimeTuple.version, + VINCI_QWEN_RUNTIME_ARTIFACT_SHA256: runtimeTuple.artifact_sha256, + VINCI_QWEN_RUNTIME_ARGUMENTS_SHA256: runtimeTuple.arguments_sha256, + VINCI_QWEN_ENDPOINT_IDENTITY_SHA256: endpointIdentity, + VINCI_QWEN_CLIENT_BUILD_SHA256: "21".repeat(32), + VINCI_QWEN_EXTENSION_BUILD_SHA256: "23".repeat(32), VINCI_QWEN_CONTEXT_WINDOW: "262144", VINCI_QWEN_MAX_TOKENS: "8192", + VINCI_QWEN_ADVERTISED_MAX_CONCURRENCY: "32", + VINCI_QWEN_MAX_CONCURRENCY: "1", + VINCI_QWEN_TOTAL_TIMEOUT_MS: "1000", + VINCI_QWEN_MAX_RETRIES: "1", + VINCI_QWEN_MAX_RETRY_DELAY_MS: "0", + VINCI_QWEN_MAX_REQUEST_BYTES: "4096", + VINCI_QWEN_MAX_RESPONSE_BYTES: "4096", + VINCI_QWEN_MAX_ERROR_BYTES: "256", + VINCI_QWEN_PRICE_BASIS: "operator-estimate:h200-amortized-2026-09-04", VINCI_QWEN_INPUT_PER_MILLION_USD: "0.25", VINCI_QWEN_OUTPUT_PER_MILLION_USD: "0.75", VINCI_QWEN_CACHE_READ_PER_MILLION_USD: "0.05", VINCI_QWEN_CACHE_WRITE_PER_MILLION_USD: "0.25", - VINCI_QWEN_PROMPT_SHA256: sha256(readFileSync(promptFile)), - VINCI_QWEN_TOOLS_SHA256: sha256('["read","grep"]'), +}; + +function qualificationFromRequest(overrides = {}, qualificationRequestEnv = requestEnv) { + const candidate = runtime.buildQwenQualificationRequest(qualificationRequestEnv).candidate; + return { + schema: "vinci.qwen-worker-qualification.v2", + status: "qualified", + authority_role: candidate.authority_role, + fallback_policy: candidate.fallback_policy, + safe_resume: false, + provenance: { + issuer: "reviewer:test", + authority: "independent-never-builder-review", + issued_at: "2026-09-04T17:00:00.000Z", + expires_at: "2026-09-05T17:00:00.000Z", + review_message_id: "msg_independent_test", + review_body_sha256: "45".repeat(32), + burn_in_report_sha256: candidate.burn_in_report_sha256, + canary: { + schema: "vinci.qwen-worker-canary.v2", + report_sha256: candidate.canary_report_sha256, + observed_at: candidate.canary_observed_at, + }, + }, + bindings: { + model: candidate.model, + revision: candidate.revision, + runtime: candidate.runtime, + endpoint_sha256: candidate.endpoint_sha256, + endpoint_identity_sha256: candidate.endpoint_identity_sha256, + work_order_prompt_sha256: candidate.work_order_prompt_sha256, + system_prompt_sha256: candidate.system_prompt_sha256, + tool_names_sha256: candidate.tool_names_sha256, + tool_schemas_sha256: candidate.tool_schemas_sha256, + tool_policy_sha256: candidate.tool_policy_sha256, + client_build_sha256: candidate.client_build_sha256, + extension_build_sha256: candidate.extension_build_sha256, + request_encoding_sha256: candidate.request_encoding_sha256, + }, + capabilities: candidate.capabilities, + limits: candidate.limits, + burn_in: candidate.burn_in, + pricing: candidate.pricing, + requalification_conditions: [...runtime.QWEN_REQUALIFICATION_CONDITIONS], + ...overrides, + }; +} + +function signedEnvelope(qualification) { + const signature = sign(null, Buffer.from(runtime.qwenCanonical(qualification)), privateKey).toString("base64"); + return { + schema: "vinci.qwen-worker-qualification-envelope.v2", + qualification, + signature: { algorithm: "Ed25519", key_id: "test-ed25519-1", signature_base64: signature }, + }; +} + +function writeQualification(value) { + if (typeof value === "object" && value !== null) value = `${runtime.qwenCanonical(value)}\n`; + try { + chmodSync(qualificationFile, 0o600); + } catch {} + writeFileSync(qualificationFile, value, { mode: 0o600 }); + chmodSync(qualificationFile, 0o400); + return runtime.qwenSha256(readFileSync(qualificationFile)); +} + +let qualification = qualificationFromRequest(); +let qualificationDigest = writeQualification(signedEnvelope(qualification)); + +const baseRuntimeEnv = { + VINCI_QWEN_BASE_URL: requestEnv.VINCI_QWEN_BASE_URL, + VINCI_QWEN_QUALIFICATION_FILE: qualificationFile, + VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest, + VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_FILE: publicKeyFile, + VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_SHA256: runtime.qwenSha256(publicKeyBytes), + VINCI_QWEN_QUALIFICATION_ISSUER: "reviewer:test", + VINCI_QWEN_PROMPT_SHA256: qualification.bindings.work_order_prompt_sha256, + VINCI_QWEN_TOOLS_SHA256: qualification.bindings.tool_names_sha256, + VINCI_QWEN_TOOL_POLICY_SHA256: qualification.bindings.tool_policy_sha256, + VINCI_QWEN_CLIENT_BUILD_SHA256: qualification.bindings.client_build_sha256, + VINCI_QWEN_EXTENSION_BUILD_SHA256: qualification.bindings.extension_build_sha256, VINCI_UNATTENDED_POLICY: "governed", VINCI_UNATTENDED_LEASE: "lease-test", VINCI_QWEN_WORK_ORDER_ID: "wo-test", @@ -50,257 +238,465 @@ const baseEnv = { VINCI_QWEN_CIRCUIT_OPEN_MS: "60000", }; +function runtimeEnv(overrides = {}) { + return { ...baseRuntimeEnv, VINCI_QWEN_SECRET_FD: String(openSync(secretFile, "r")), ...overrides }; +} + +function loadConfig(overrides = {}) { + return runtime.loadQwenRuntimeConfig(runtimeEnv(overrides), nowMs); +} + +function identityHeaders(overrides = {}) { + return { + "content-type": "text/event-stream", + "x-vinci-model-id": runtime.QWEN_MODEL, + "x-vinci-model-revision": revision, + "x-vinci-endpoint-identity-sha256": endpointIdentity, + "x-vinci-runtime-engine": runtimeTuple.engine, + "x-vinci-runtime-version": runtimeTuple.version, + "x-vinci-runtime-artifact-sha256": runtimeTuple.artifact_sha256, + "x-vinci-runtime-arguments-sha256": runtimeTuple.arguments_sha256, + ...overrides, + }; +} + +const usage = { + prompt_tokens: 10, + completion_tokens: 2, + total_tokens: 12, + prompt_tokens_details: null, + completion_tokens_details: null, +}; +const validSse = [ + `data: ${JSON.stringify({ id: "chunk-1", object: "chat.completion.chunk", created: 1, model: runtime.QWEN_MODEL, choices: [], usage })}`, + "data: [DONE]", + "", +].join("\n"); + try { + assert.equal(qualification.safe_resume, false); + assert.equal(qualification.limits.max_concurrency, 1); + assert.equal(qualification.limits.advertised_max_concurrency, 32); + assert.equal(qualification.bindings.request_encoding_sha256, runtime.qwenSha256(runtime.qwenCanonical(runtime.QWEN_REQUEST_ENCODING))); + const config = loadConfig(); + assert.equal(config.secret, "synthetic-test-secret"); + assert.equal(config.qualification.provenance.authority, "independent-never-builder-review"); assert.throws( - () => runtime.buildQwenQualificationTemplate({ ...baseEnv, VINCI_QWEN_ADMIT: undefined }), - /qualification_not_admitted/, + () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_CLIENT_BUILD_SHA256: "99".repeat(32) }), nowMs), + /client_build_mismatch/, ); - const qualification = runtime.buildQwenQualificationTemplate(baseEnv); - assert.equal(qualification.model, "Qwen/Qwen3.8-27B"); - assert.equal(qualification.limits.max_concurrency, 1, "concurrency must start conservative"); - assert.equal(qualification.limits.max_retries, 1); - assert.equal(qualification.authority_role, "non-authoritative-evidence-and-proposals-only"); - assert.equal(qualification.fallback_policy, "explicit-openrouter-separate-attempt-only"); - writeFileSync(qualificationFile, `${JSON.stringify(qualification)}\n`, { mode: 0o400 }); - chmodSync(qualificationFile, 0o400); - const env = { - ...baseEnv, - VINCI_QWEN_QUALIFICATION_FILE: qualificationFile, - VINCI_QWEN_QUALIFICATION_SHA256: sha256(readFileSync(qualificationFile)), - }; - const config = runtime.loadQwenRuntimeConfig(env); - assert.equal(config.baseUrl, "https://qwen.example.test/v1"); - assert.equal(config.secret, "test-secret"); - const runtimeTuple = qualification.runtime; + const invalidSignature = structuredClone(signedEnvelope(qualification)); + invalidSignature.qualification.bindings.model = "Qwen/Qwen3.8-27B-tampered"; + qualificationDigest = writeQualification(invalidSignature); + assert.throws( + () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest }), nowMs), + /qualification_signature_invalid/, + ); + qualificationDigest = writeQualification(signedEnvelope(qualification)); + baseRuntimeEnv.VINCI_QWEN_QUALIFICATION_SHA256 = qualificationDigest; + + qualificationDigest = writeQualification(qualification); + assert.throws( + () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest }), nowMs), + /qualification envelope.*missing fields|qualification envelope.*unexpected/, + ); + qualificationDigest = writeQualification(signedEnvelope(qualification)); + baseRuntimeEnv.VINCI_QWEN_QUALIFICATION_SHA256 = qualificationDigest; + + const expired = qualificationFromRequest({ + provenance: { ...qualification.provenance, issued_at: "2026-09-01T16:00:00.000Z", expires_at: "2026-09-02T16:00:00.000Z" }, + }); + qualificationDigest = writeQualification(signedEnvelope(expired)); + assert.throws( + () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest }), nowMs), + /qualification_expired/, + ); + qualificationDigest = writeQualification(signedEnvelope(qualification)); + baseRuntimeEnv.VINCI_QWEN_QUALIFICATION_SHA256 = qualificationDigest; + + const concurrency32 = qualificationFromRequest({ + limits: { ...qualification.limits, max_concurrency: 32 }, + burn_in: { + ...entryBurnIn, + previous_concurrency: 24, + target_concurrency: 32, + observed_hours: 168, + work_orders: 1_000, + transport_error_rate: 0.005, + }, + }); + qualificationDigest = writeQualification(signedEnvelope(concurrency32)); + assert.throws( + () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest }), nowMs), + /fleet_permit_authority_missing/, + ); + qualificationDigest = writeQualification(signedEnvelope(qualification)); + baseRuntimeEnv.VINCI_QWEN_QUALIFICATION_SHA256 = qualificationDigest; + + const skippedStage = qualificationFromRequest({ + limits: { ...qualification.limits, max_concurrency: 8 }, + burn_in: { ...entryBurnIn, previous_concurrency: 2, target_concurrency: 8, observed_hours: 168, work_orders: 1_000 }, + }); + qualificationDigest = writeQualification(signedEnvelope(skippedStage)); + assert.throws( + () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest }), nowMs), + /burn_in_gate_failed/, + ); + qualificationDigest = writeQualification(signedEnvelope(qualification)); + baseRuntimeEnv.VINCI_QWEN_QUALIFICATION_SHA256 = qualificationDigest; + const observed = []; const readyFetch = async (url, init = {}) => { - observed.push({ url: String(url), authorization: new Headers(init.headers).get("authorization"), method: init.method ?? "GET", body: init.body }); - if (String(url).endsWith("/health") && !new Headers(init.headers).has("authorization")) { - return Response.json({ error: "unauthorized" }, { status: 401 }); - } + const headers = new Headers(init.headers); + observed.push({ url: String(url), authorization: headers.get("authorization") }); + if (String(url).endsWith("/health") && !headers.has("authorization")) return Response.json({}, { status: 401 }); if (String(url).endsWith("/health")) return Response.json({ status: "ready" }); - if (String(url).endsWith("/v1/models") && !new Headers(init.headers).has("authorization")) { - return Response.json({ error: "unauthorized" }, { status: 401 }); - } + if (String(url).endsWith("/v1/models") && !headers.has("authorization")) return Response.json({}, { status: 401 }); if (String(url).endsWith("/v1/models")) { - return Response.json({ object: "list", data: [{ id: runtime.QWEN_MODEL, revision: qualification.revision, runtime: runtimeTuple }] }); + return Response.json({ + object: "list", + data: [{ id: runtime.QWEN_MODEL, revision, runtime: runtimeTuple, endpoint_identity_sha256: endpointIdentity }], + }); } - throw new Error(`unexpected URL ${url}`); + throw new Error(`unexpected fake URL ${url}`); }; - const identity = await runtime.probeQwenReadiness(config, { fetchImpl: readyFetch, nowMs: 1_000 }); - assert.deepEqual(identity, { revision: qualification.revision, runtime: runtimeTuple }); - assert.deepEqual(observed.map(({ authorization }) => authorization), ["Bearer test-secret", "Bearer test-secret", null, null]); - assert.doesNotMatch(JSON.stringify(observed), /VINCI_QWEN_SECRET_REF/); + const lookupPublic = async () => [{ address: "93.184.216.34", family: 4 }]; + const readyConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "ready-circuit.json") }); + const identity = await runtime.probeQwenReadiness(readyConfig, { fetchImpl: readyFetch, lookupImpl: lookupPublic, nowMs }); + assert.equal(identity.endpointIdentity, endpointIdentity); + assert.deepEqual(observed.map((entry) => entry.authorization), ["Bearer synthetic-test-secret", "Bearer synthetic-test-secret", null, null]); - const promptMismatch = { ...env, VINCI_QWEN_PROMPT_SHA256: hex("01") }; - assert.throws(() => runtime.loadQwenRuntimeConfig(promptMismatch), /qwen_prompt_mismatch/); - assert.throws( - () => runtime.loadQwenRuntimeConfig({ ...env, VINCI_UNATTENDED_POLICY: "off" }), - /qwen_authority_forbidden/, + const canarySse = [ + `data: ${JSON.stringify({ + id: "canary-tool", + object: "chat.completion.chunk", + created: 1, + model: runtime.QWEN_MODEL, + choices: [{ index: 0, delta: { tool_calls: [{ index: 0, function: { name: "report_ready", arguments: '{"status":"ready"}' } }] } }], + })}`, + `data: ${JSON.stringify({ id: "canary-usage", object: "chat.completion.chunk", created: 2, model: runtime.QWEN_MODEL, choices: [], usage })}`, + "data: [DONE]", + "", + ].join("\n"); + const canaryFetch = async (url, init = {}) => { + const target = String(url); + const authenticated = new Headers(init.headers).has("authorization"); + if (!authenticated && (target.endsWith("/health") || target.endsWith("/v1/models"))) return Response.json({}, { status: 401 }); + if (target.endsWith("/health")) return Response.json({ status: "ready" }); + if (target.endsWith("/v1/models")) { + return Response.json({ + object: "list", + data: [{ id: runtime.QWEN_MODEL, revision, runtime: runtimeTuple, endpoint_identity_sha256: endpointIdentity }], + }); + } + if (target.endsWith("/v1/chat/completions")) return new Response(canarySse, { headers: identityHeaders() }); + throw new Error(`unexpected canary URL ${target}`); + }; + const canaryReport = await runtime.runQwenCanary( + { + VINCI_QWEN_BASE_URL: requestEnv.VINCI_QWEN_BASE_URL, + VINCI_QWEN_SECRET_REF: `file:${secretFile}`, + VINCI_QWEN_CANARY_TIMEOUT_MS: "1000", + }, + canaryFetch, + lookupPublic, ); - assert.throws( - () => runtime.buildQwenQualificationTemplate({ ...baseEnv, VINCI_QWEN_MAX_CONCURRENCY: "9" }), - /max_concurrency/, + assert.equal(canaryReport.safe_resume, false); + assert.equal(canaryReport.endpoint_identity_sha256, endpointIdentity); + assert.equal(canaryReport.pinned_addresses_sha256, runtime.qwenSha256(runtime.qwenCanonical(["93.184.216.34"]))); + + const privateConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "private-circuit.json") }); + await assert.rejects(runtime.pinQwenEndpoint(privateConfig, async () => [{ address: "169.254.169.254", family: 4 }]), /ssrf_forbidden/); + + const records = []; + const breakerConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "breaker.json") }); + breakerConfig.endpointAddresses = ["93.184.216.34"]; + let failedCalls = 0; + const retryKeys = []; + const failingTransport = runtime.createQwenInferenceFetch( + breakerConfig, + "request-500", + (record) => records.push(record), + async (_url, init = {}) => { + failedCalls += 1; + retryKeys.push(new Headers(init.headers).get("x-vinci-idempotency-key")); + return new Response("failure", { status: 500 }); + }, + ); + await assert.rejects( + failingTransport(breakerConfig.chatUrl, { method: "POST", body: "{}" }), + /qwen_http_status/, ); + assert.equal(failedCalls, 2, "threshold two must count both real HTTP 500 responses"); + assert.equal(records.length, 2); + assert.deepEqual(retryKeys, ["request-500/0", "request-500/1"]); + assert.deepEqual(records.map((record) => record.transport_attempt), [0, 1]); + assert.ok(records.every((record) => record.cost_usd === 0 && record.input_tokens === 0 && record.output_tokens === 0)); + assert.throws(() => runtime.assertQwenCircuitClosed(breakerConfig), /qwen_circuit_open/); - const vectors = join(root, "vinci/test/fixtures/contract-vectors"); - const emptyCriteriaOrder = { - ...JSON.parse(readFileSync(join(vectors, "work-order-1-minimal/input.json"), "utf8")), - acceptanceCriteria: [], - }; - assert.throws( - () => digest.workOrderDigest(emptyCriteriaOrder), - /criteria_required/, - "the existing contract gate must reject a Qwen batch without acceptance criteria before materialization", + const concurrentCircuitFile = join(temp, "concurrent-breaker.json"); + await exerciseConcurrentBreakerWrites(concurrentCircuitFile, 8); + const concurrentState = JSON.parse(readFileSync(concurrentCircuitFile, "utf8")); + assert.equal(concurrentState.schema, "vinci.qwen-worker-circuit.v2"); + assert.equal(concurrentState.failures, 8, "atomic breaker updates must not lose concurrent failures"); + assert.equal(concurrentState.sequence, 8); + + const mismatchConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "mismatch.json") }); + mismatchConfig.endpointAddresses = ["93.184.216.34"]; + const mismatchTransport = runtime.createQwenInferenceFetch( + mismatchConfig, + "request-mismatch", + () => {}, + async () => new Response(validSse, { headers: identityHeaders({ "x-vinci-model-id": "Qwen/wrong" }) }), ); - assert.throws( - () => workerRun.runVinci({ - envelope: { provider: "qwen-h200", model: runtime.QWEN_MODEL, ref: "legacy-prose-ref", tools: ["read"], spec: "legacy prose" }, - repoDir: temp, - stateDir: temp, - taskId: "task-test", - sessionId: "run-test", - }), - /validated digest WorkOrder identity and acceptance criteria/, - "legacy prose cannot bypass the WorkOrder acceptance-criteria gate", + await assert.rejects(mismatchTransport(mismatchConfig.chatUrl, { method: "POST", body: "{}" }), /response_identity_mismatch/); + + const runtimeMismatchRecords = []; + const runtimeMismatchConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "runtime-mismatch.json") }); + runtimeMismatchConfig.endpointAddresses = ["93.184.216.34"]; + await assert.rejects( + runtime.createQwenInferenceFetch( + runtimeMismatchConfig, + "request-runtime-mismatch", + (record) => runtimeMismatchRecords.push(record), + async () => new Response(validSse, { headers: identityHeaders({ "x-vinci-runtime-version": "wrong" }) }), + )(runtimeMismatchConfig.chatUrl, { method: "POST", body: "{}" }), + /response_identity_mismatch/, ); + assert.equal(runtimeMismatchRecords[0].outcome, "response_identity_mismatch"); - const wrongModelConfig = { ...config, circuitFile: join(temp, "wrong-model-circuit.json") }; + const redirectConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "redirect.json") }); + redirectConfig.endpointAddresses = ["93.184.216.34"]; await assert.rejects( - runtime.probeQwenReadiness(wrongModelConfig, { - fetchImpl: async (url, init = {}) => { - if (String(url).endsWith("/health")) return Response.json({ status: "ready" }); - if (!new Headers(init.headers).has("authorization")) return Response.json({}, { status: 401 }); - return Response.json({ object: "list", data: [{ id: "Qwen/Qwen3.8-27B-alias", revision: qualification.revision, runtime: runtimeTuple }] }); - }, - nowMs: 2_000, - }), - /qwen_model_mismatch/, + runtime.createQwenInferenceFetch(redirectConfig, "request-redirect", () => {}, async () => new Response("", { status: 302 }))( + redirectConfig.chatUrl, + { method: "POST", body: "{}" }, + ), + /redirect_forbidden/, + ); + await assert.rejects( + runtime.createQwenInferenceFetch(redirectConfig, "request-ssrf", () => {}, async () => new Response(validSse, { headers: identityHeaders() }))( + "http://169.254.169.254/latest/meta-data", + { method: "POST", body: "{}" }, + ), + /ssrf_forbidden/, + ); + await assert.rejects( + runtime.createQwenInferenceFetch(redirectConfig, "request-too-large", () => {}, async () => new Response(validSse, { headers: identityHeaders() }))( + redirectConfig.chatUrl, + { method: "POST", body: "x".repeat(5_000) }, + ), + /request_oversized/, ); - const authOpenConfig = { ...config, circuitFile: join(temp, "auth-open-circuit.json") }; + const cancelledConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "cancelled-retry.json") }); + cancelledConfig.endpointAddresses = ["93.184.216.34"]; + cancelledConfig.qualification.limits.max_retry_delay_ms = 1_000; + const retryAbort = new AbortController(); + let cancelledCalls = 0; await assert.rejects( - runtime.probeQwenReadiness(authOpenConfig, { - fetchImpl: async (url) => String(url).endsWith("/health") - ? Response.json({ status: "ready" }) - : Response.json({ object: "list", data: [{ id: runtime.QWEN_MODEL, revision: qualification.revision, runtime: runtimeTuple }] }), - nowMs: 3_000, - }), - /qwen_auth_not_enforced/, + runtime.createQwenInferenceFetch(cancelledConfig, "request-cancelled", () => {}, async () => { + cancelledCalls += 1; + queueMicrotask(() => retryAbort.abort("operator_stop")); + return new Response("retry", { status: 429, headers: { "retry-after": "1" } }); + })(cancelledConfig.chatUrl, { method: "POST", body: "{}", signal: retryAbort.signal }), + /cancelled/, ); + assert.equal(cancelledCalls, 1, "cancellation during Retry-After must prevent the next transport attempt"); - const circuitConfig = { ...config, circuitFile: join(temp, "breaker.json") }; - let failedCalls = 0; - const unavailable = async () => { - failedCalls += 1; - throw new Error("offline fake"); - }; - await assert.rejects(runtime.probeQwenReadiness(circuitConfig, { fetchImpl: unavailable, nowMs: 10_000 }), /endpoint_unavailable/); - assert.equal(failedCalls, 2, "one retry means exactly two bounded attempts"); - await assert.rejects(runtime.probeQwenReadiness(circuitConfig, { fetchImpl: unavailable, nowMs: 11_000 }), /endpoint_unavailable/); - const callsAtOpen = failedCalls; - await assert.rejects(runtime.probeQwenReadiness(circuitConfig, { fetchImpl: unavailable, nowMs: 12_000 }), /qwen_circuit_open/); - assert.equal(failedCalls, callsAtOpen, "open circuit must make no endpoint call"); - - const cancelledConfig = { ...config, circuitFile: join(temp, "cancelled.json") }; - const controller = new AbortController(); - controller.abort("fixture cancellation"); + const oversizedConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "oversized.json") }); + oversizedConfig.endpointAddresses = ["93.184.216.34"]; await assert.rejects( - runtime.probeQwenReadiness(cancelledConfig, { - signal: controller.signal, - fetchImpl: async (_url, init = {}) => { - assert.equal(init.signal.aborted, true); - throw new DOMException("aborted", "AbortError"); - }, - }), - /qwen_cancelled/, + runtime.createQwenInferenceFetch(oversizedConfig, "request-oversized", () => {}, async () => new Response("x".repeat(300), { status: 500 }))( + oversizedConfig.chatUrl, + { method: "POST", body: "{}" }, + ), + /response_oversized/, ); - const sse = [ - 'data: {"choices":[{"delta":{"tool_calls":[{"function":{"name":"report_","arguments":"{\\"status\\":\\""}}]}}]}', - 'data: {"choices":[{"delta":{"tool_calls":[{"function":{"name":"ready","arguments":"ready\\"}"}}]}}]}', - 'data: {"choices":[],"usage":{"prompt_tokens":10,"completion_tokens":2}}', + const successRecords = []; + const successConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "success.json") }); + successConfig.endpointAddresses = ["93.184.216.34"]; + const successResponse = await runtime.createQwenInferenceFetch( + successConfig, + "request-success", + (record) => successRecords.push(record), + async () => new Response(validSse, { headers: identityHeaders() }), + )(successConfig.chatUrl, { method: "POST", body: "{}" }); + assert.equal(await successResponse.text(), validSse); + assert.equal(successRecords[0].outcome, "success"); + assert.equal(successRecords[0].input_tokens, 10); + assert.equal(successRecords[0].output_tokens, 2); + assert.equal(successRecords[0].cost_usd, 0.000004); + + const invalidUsage = { ...usage, total_tokens: 99 }; + const invalidUsageSse = [ + `data: ${JSON.stringify({ id: "chunk-bad-usage", object: "chat.completion.chunk", created: 1, model: runtime.QWEN_MODEL, choices: [], usage: invalidUsage })}`, "data: [DONE]", "", ].join("\n"); - const canaryCalls = []; - const canary = await runtime.runQwenCanary( - { ...baseEnv, VINCI_QWEN_CANARY_TIMEOUT_MS: "1000" }, - async (url, init = {}) => { - canaryCalls.push({ url: String(url), init }); - if (String(url).endsWith("/health") && !new Headers(init.headers).has("authorization")) return Response.json({}, { status: 401 }); - if (String(url).endsWith("/health")) return Response.json({ status: "ready" }); - if (String(url).endsWith("/v1/models") && !new Headers(init.headers).has("authorization")) return Response.json({}, { status: 401 }); - if (String(url).endsWith("/v1/models")) return Response.json({ object: "list", data: [{ id: runtime.QWEN_MODEL, revision: qualification.revision, runtime: runtimeTuple }] }); - if (String(url).endsWith("/v1/chat/completions")) return new Response(sse, { headers: { "content-type": "text/event-stream" } }); - throw new Error(`unexpected URL ${url}`); + const invalidUsageConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "invalid-usage.json") }); + invalidUsageConfig.endpointAddresses = ["93.184.216.34"]; + const invalidUsageResponse = await runtime.createQwenInferenceFetch( + invalidUsageConfig, + "request-invalid-usage", + () => {}, + async () => new Response(invalidUsageSse, { headers: identityHeaders() }), + )(invalidUsageConfig.chatUrl, { method: "POST", body: "{}" }); + await assert.rejects(invalidUsageResponse.text(), /usage_invalid/); + + const oversizedSuccessConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "oversized-success.json") }); + oversizedSuccessConfig.endpointAddresses = ["93.184.216.34"]; + const oversizedSuccessResponse = await runtime.createQwenInferenceFetch( + oversizedSuccessConfig, + "request-oversized-success", + () => {}, + async () => new Response(`data: ${"x".repeat(5_000)}\n`, { headers: identityHeaders() }), + )(oversizedSuccessConfig.chatUrl, { method: "POST", body: "{}" }); + await assert.rejects(oversizedSuccessResponse.text(), /response_oversized/); + + const timeoutConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "timeout.json") }); + timeoutConfig.endpointAddresses = ["93.184.216.34"]; + const timeoutResponse = await runtime.createQwenInferenceFetch( + timeoutConfig, + "request-timeout", + () => {}, + async (_url, init = {}) => new Response(new ReadableStream({ + start(controller) { + init.signal.addEventListener("abort", () => controller.error(new DOMException("aborted", "AbortError")), { once: true }); + }, + }), { headers: identityHeaders() }), + )(timeoutConfig.chatUrl, { method: "POST", body: "{}" }); + await assert.rejects(timeoutResponse.text(), /AbortError|aborted/); + + const context = { systemPrompt, messages: [{ role: "user", content: workOrderPrompt, timestamp: Date.now() }], tools }; + const terminalMessage = { + role: "assistant", + content: [], + api: "openai-completions", + provider: runtime.QWEN_PROVIDER, + model: runtime.QWEN_MODEL, + usage: { + input: 10, + output: 2, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 12, + cost: { input: 0.0000025, output: 0.0000015, cacheRead: 0, cacheWrite: 0, total: 0.000004 }, }, - ); - assert.equal(canary.capabilities.structured_output, "tool-arguments-json"); - assert.equal(canary.capabilities.usage_chunk, true); - const inference = canaryCalls.find(({ url }) => url.endsWith("/chat/completions")); - const payload = JSON.parse(inference.init.body); - assert.equal(payload.model, "Qwen/Qwen3.8-27B"); - assert.equal(payload.stream, true); - assert.equal(payload.tools[0].function.name, "report_ready"); - - const extensionEnvNames = [ - ...Object.keys(env), - "VINCI_QWEN_SELECTED", - ]; - const prior = new Map(extensionEnvNames.map((name) => [name, process.env[name]])); - const nativeFetch = globalThis.fetch; - try { - Object.assign(process.env, env, { VINCI_QWEN_SELECTED: "1", VINCI_QWEN_CIRCUIT_FILE: join(temp, "extension-circuit.json") }); - globalThis.fetch = readyFetch; - const registrations = []; - const handlers = {}; - const labels = []; - await providerExtension({ - registerProvider(name, providerConfig) { registrations.push({ name, providerConfig }); }, - on(name, handler) { (handlers[name] ??= []).push(handler); }, - appendEntry(name, value) { labels.push({ name, value }); }, + stopReason: "stop", + timestamp: Date.now(), + }; + const fakeStream = (model) => { + const stream = createAssistantMessageEventStream(); + queueMicrotask(() => { + stream.push({ type: "done", reason: "stop", message: { ...terminalMessage, api: model.api } }); + stream.end({ ...terminalMessage, api: model.api }); }); - assert.deepEqual(registrations.map(({ name }) => name), ["vinci", "qwen-h200"]); - const qwen = registrations[1].providerConfig; - assert.equal(qwen.api, "vinci-qwen-openai-completions"); - assert.equal(qwen.apiKey, "runtime-resolved-secret-reference", "provider config must not contain the secret value"); - assert.equal(qwen.models[0].id, "Qwen/Qwen3.8-27B"); - assert.equal(qwen.models[0].cost.input, 0.25); - assert.equal(process.env.VINCI_QWEN_SECRET_REF, undefined, "bootstrap secret reference must be scrubbed"); - - const headers = {}; - for (const handler of handlers.before_provider_headers ?? []) { - await handler({ headers }, { model: { provider: "qwen-h200" } }); - } - assert.equal(headers["x-vinci-work-order-id"], "wo-test"); - assert.equal(headers["x-vinci-run-id"], "run-test"); - assert.equal(headers["x-vinci-attempt-id"], "task-test/1"); - assert.equal(headers["x-vinci-qwen-output-authority"], "non-authoritative"); - - for (let index = 0; index < 2; index += 1) { - for (const handler of handlers.after_provider_response ?? []) { - await handler({ status: 401 }, { model: { provider: "qwen-h200" } }); - } - } - assert.throws( - () => qwen.streamSimple( - { ...qwen.models[0], provider: "qwen-h200", api: qwen.api }, - { messages: [] }, - ), - /qwen_circuit_open/, - "authentication failures must open the circuit before another inference call", - ); - - for (const handler of handlers.message_end ?? []) { - await handler( - { message: { role: "assistant", provider: "qwen-h200", stopReason: "stop" } }, - { sessionManager: { getSessionId: () => "run-test" } }, + return stream; + }; + const permitConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "permit.json") }); + permitConfig.endpointAddresses = ["93.184.216.34"]; + const provider = qwenProviderConfig(permitConfig, fakeStream); + const model = { ...provider.models[0], provider: runtime.QWEN_PROVIDER, api: runtime.QWEN_API }; + const firstPermitStream = provider.streamSimple(model, context); + let secondPermitStream; + for await (const event of firstPermitStream) { + if (event.type === "done") { + assert.doesNotThrow( + () => { secondPermitStream = provider.streamSimple(model, context); }, + "permit must be released before the terminal result event becomes observable", ); } - assert.equal(labels[0].name, "vinci-qwen-output-label"); - assert.equal(labels[0].value.authority, "non-authoritative"); - assert.equal(labels[0].value.independent_check_required, true); + } + assert.ok(secondPermitStream); + await secondPermitStream.result(); + + const truncatedProvider = qwenProviderConfig(permitConfig, () => { + const stream = createAssistantMessageEventStream(); + queueMicrotask(() => stream.end(terminalMessage)); + return stream; + }); + const truncated = await truncatedProvider.streamSimple(model, context).result(); + assert.equal(truncated.stopReason, "error"); + assert.match(truncated.errorMessage, /qwen_stream_truncated/); + + const vectors = join(root, "vinci/test/fixtures/contract-vectors"); + const emptyCriteriaOrder = { + ...JSON.parse(readFileSync(join(vectors, "work-order-1-minimal/input.json"), "utf8")), + acceptanceCriteria: [], + }; + assert.throws(() => digest.workOrderDigest(emptyCriteriaOrder), /criteria_required/); + + const fakeBin = join(temp, "bin"); + const fakeVinci = join(fakeBin, "vinci"); + const spawnRecord = join(temp, "spawn-record.json"); + mkdirSync(fakeBin); + writeFileSync(fakeVinci, `#!/usr/bin/env node +import { fstatSync, readFileSync, writeFileSync } from "node:fs"; +let stdin = ""; +process.stdin.setEncoding("utf8"); +for await (const chunk of process.stdin) stdin += chunk; +const secret = readFileSync(3); +writeFileSync(process.env.QWEN_TEST_SPAWN_RECORD, JSON.stringify({ argv: process.argv.slice(2), stdin, qwenEnvKeys: Object.keys(process.env).filter((key) => key.includes("QWEN_SECRET")), secretBytes: fstatSync(3).size, secretReadBytes: secret.length })); +`, { mode: 0o700 }); + chmodSync(fakeVinci, 0o700); + const stateDir = join(temp, "run-state"); + mkdirSync(join(stateDir, "tasks"), { recursive: true }); + writeFileSync(join(stateDir, "tasks", "task-spawn.json"), JSON.stringify({ attempt: 1 }), { mode: 0o600 }); + const originalPath = process.env.PATH; + process.env.PATH = `${fakeBin}:${originalPath}`; + try { + await runVinci({ + envelope: { + provider: runtime.QWEN_PROVIDER, + model: runtime.QWEN_MODEL, + work_order_id: "wo-spawn", + spec: "synthetic prompt must not be argv", + tools: ["read"], + max_runtime_s: 10, + budget_usd: 1, + }, + repoDir: temp, + stateDir, + taskId: "task-spawn", + sessionId: "run-spawn", + env: { + PATH: `${fakeBin}:${originalPath}`, + QWEN_TEST_SPAWN_RECORD: spawnRecord, + VINCI_QWEN_SECRET_REF: `file:${secretFile}`, + }, + envDelta: {}, + }); } finally { - globalThis.fetch = nativeFetch; - for (const [name, value] of prior) { - if (value === undefined) delete process.env[name]; - else process.env[name] = value; - } + process.env.PATH = originalPath; } + const spawned = JSON.parse(readFileSync(spawnRecord, "utf8")); + assert.equal(spawned.argv.includes("synthetic prompt must not be argv"), false); + assert.equal(spawned.argv.includes("synthetic-test-secret"), false); + assert.equal(spawned.argv.includes(secretFile), false); + assert.equal(spawned.stdin, "synthetic prompt must not be argv"); + assert.equal(spawned.stdin.includes("synthetic-test-secret"), false); + assert.deepEqual(spawned.qwenEnvKeys, ["VINCI_QWEN_SECRET_FD"]); + assert.equal(spawned.secretBytes, spawned.secretReadBytes); + + const lifecycle = new TaskLifecycle(join(temp, "attempt-state"), "task-attempt"); + const first = lifecycle.startAttempt({ id: "task-attempt", envelope: { provider: runtime.QWEN_PROVIDER, evidence: "none" } }, "test"); + const second = lifecycle.startAttempt({ id: "task-attempt", envelope: { provider: runtime.QWEN_PROVIDER, evidence: "none" } }, "test"); + assert.equal(first.sessionId, "task-attempt-qwen-attempt-1"); + assert.equal(second.sessionId, "task-attempt-qwen-attempt-2"); assert.equal(cleanroom.CLEAN_ROOM_ENV_ALLOWLIST.includes("VINCI_QWEN_SECRET_REF"), false); - assert.ok(cleanroom.PROVIDER_KEY_ENV["qwen-h200"].includes("VINCI_QWEN_SECRET_REF")); + assert.ok(cleanroom.PROVIDER_KEY_ENV[runtime.QWEN_PROVIDER].includes("VINCI_QWEN_SECRET_REF")); const scoped = cleanroom.providerScopedEnv({ - base: { - OPENROUTER_API_KEY: "must-drop", - VINCI_QWEN_SECRET_REF: "env:QWEN_DYNAMIC_TEST_SECRET", - QWEN_DYNAMIC_TEST_SECRET: "dynamic-test-secret", - }, - provider: "qwen-h200", + base: { OPENROUTER_API_KEY: "drop", VINCI_QWEN_SECRET_REF: `file:${secretFile}` }, + provider: runtime.QWEN_PROVIDER, agentDir: join(temp, "agent"), }); assert.equal(scoped.OPENROUTER_API_KEY, undefined); - assert.equal(scoped.VINCI_QWEN_SECRET_REF, "env:QWEN_DYNAMIC_TEST_SECRET"); - assert.equal(scoped.QWEN_DYNAMIC_TEST_SECRET, "dynamic-test-secret"); - const cleanScoped = cleanroom.cleanRoomEnv({ - base: scoped, - provider: "qwen-h200", - homeDir: join(temp, "clean-home"), - tmpDir: join(temp, "clean-tmp"), - }); - assert.equal(cleanScoped.QWEN_DYNAMIC_TEST_SECRET, "dynamic-test-secret"); - const otherProvider = cleanroom.providerScopedEnv({ - base: scoped, - provider: "openrouter", - agentDir: join(temp, "other-agent"), - }); - assert.equal(otherProvider.VINCI_QWEN_SECRET_REF, undefined); - assert.equal(otherProvider.QWEN_DYNAMIC_TEST_SECRET, undefined, "a dynamic Qwen secret must not cross provider boundaries"); - const envSecretConfig = { ...env, VINCI_QWEN_SECRET_REF: "env:QWEN_DYNAMIC_TEST_SECRET", QWEN_DYNAMIC_TEST_SECRET: "dynamic-test-secret" }; - assert.equal(runtime.loadQwenRuntimeConfig(envSecretConfig).secret, "dynamic-test-secret"); - assert.equal(envSecretConfig.QWEN_DYNAMIC_TEST_SECRET, undefined, "the resolved secret must be scrubbed before repository tools run"); + assert.equal(scoped.VINCI_QWEN_SECRET_REF, `file:${secretFile}`); + assert.equal(cleanroom.providerScopedEnv({ base: scoped, provider: "openrouter", agentDir: join(temp, "other") }).VINCI_QWEN_SECRET_REF, undefined); const summary = economics.buildEconomicsSummary({ workOrderId: "wo-test", @@ -308,23 +704,21 @@ try { sessionId: "run-test", started: "2026-09-04T10:00:00.000Z", finished: "2026-09-04T10:00:02.000Z", - usageEntries: [{ provider: "qwen-h200", model: runtime.QWEN_MODEL, model_calls: 1, input_tokens: 10, output_tokens: 2, cost_microusd: 4 }], + usageEntries: [{ provider: runtime.QWEN_PROVIDER, model: runtime.QWEN_MODEL, model_calls: 1, input_tokens: 10, output_tokens: 2, cost_microusd: 4 }], sessionState: { path: "/fake/session", source: "usage_entries", costUsd: 0.000004 }, receipt: { verificationStatus: "passed" }, run: { exit_code: 0, limit_tripped: null, harness_stops: [] }, taskState: "UNVERIFIED", }); assert.equal(summary.route.policy_id, "single-provider-no-automatic-fallback"); - assert.equal(summary.route.initial_provider, "qwen-h200"); - assert.equal(summary.route.initial_model, "Qwen/Qwen3.8-27B"); assert.equal(summary.work_order_id, "wo-test"); assert.equal(summary.session_id, "run-test"); assert.equal(summary.attempt_label, "task-test/1"); - assert.equal(summary.started_at, "2026-09-04T10:00:00.000Z"); - assert.equal(summary.finished_at, "2026-09-04T10:00:02.000Z"); } finally { - chmodSync(qualificationFile, 0o600); + try { + chmodSync(qualificationFile, 0o600); + } catch {} rmSync(temp, { recursive: true, force: true }); } -process.stdout.write(" Qwen H200 provider: qualification, readiness, auth, circuit, canary, attribution, and telemetry guards pass\n"); +process.stdout.write("PASS worker-qwen-provider signed qualification, bounded transport, containment, attribution, and concurrency guards\n"); diff --git a/vinci/worker/README.md b/vinci/worker/README.md index 4226a8dee..f11b479bc 100644 --- a/vinci/worker/README.md +++ b/vinci/worker/README.md @@ -313,42 +313,52 @@ Enable it only in the Worker process that should admit the lane by including `qw digest-WorkOrder-only; legacy prose handoffs are refused because they cannot carry the validated acceptance criteria and immutable contract binding required by this lane. -The lane is fail-closed. Before it is registered, the client requires: - -- `VINCI_QWEN_BASE_URL`: the operator-supplied HTTPS endpoint (loopback HTTP is allowed for local - tests). Credentials, query strings, and fragments are refused. -- `VINCI_QWEN_SECRET_REF`: `file:/absolute/private/path` or `env:NAME`; this setting is a reference, - never a secret value. Credential files must be private, regular, and non-symlinked. The bootstrap - reference is scrubbed before repository tools run. -- `VINCI_QWEN_QUALIFICATION_FILE` and `VINCI_QWEN_QUALIFICATION_SHA256`: an operator-owned, - non-writable qualification record and an exact process-level byte pin. -- a deterministic Governor lease, plus WorkOrder, Run, and Attempt ids derived by the worker. - Model output supplies none of these identities. - -Ayush's non-secret handoff is intentionally small: the externally reachable base URL; confirmation -that bearer authentication is required by `/health`, `/v1/models`, and `/v1/chat/completions`; the -exact `Qwen/Qwen3.8-27B` identifier and immutable served revision; runtime engine and version; the -SHA-256 of the runtime artifact and canonical launch arguments; and the served context/output -limits. `/v1/models` must return the exact revision and runtime tuple either on its one matching -model object or through the documented `X-Vinci-Model-Revision` and `X-Vinci-Runtime-*` headers. -Ayush supplies only the name of the operator-installed secret reference mechanism, never the -credential itself in a WorkOrder, issue, log, or qualification record. Runtime launch flags, model -download, GPU placement, and endpoint operation remain exclusively his lane. - -The closed qualification record binds the endpoint hash, exact model, immutable served revision, -runtime engine/version/artifact/arguments tuple, exact task-prompt hash, exact ordered tool-list -hash, streaming SSE and structured tool-call JSON, request timeout, concurrency, retries, -retry-delay cap, context/output bounds, and operator token-cost estimates. `/health` and -`/v1/models` are probed with authentication; an anonymous models request must be refused, and the -model response must repeat the exact revision/runtime tuple. Three consecutive readiness failures -open the persistent circuit for 60 seconds by default. `VINCI_QWEN_CIRCUIT_THRESHOLD` and -`VINCI_QWEN_CIRCUIT_OPEN_MS` are bounded overrides. - -Concurrency starts at one. The qualification schema allows an explicit increase up to eight after -burn-in; excess requests are rejected as backpressure, never queued without a bound. Transport -retries are capped at two. High available token volume does not admit work: each digest handoff -still needs bounded resources and acceptance criteria, and the worker refuses an invalid or -over-broad WorkOrder before inference. +The lane is fail-closed. Runtime registration requires all of the following: + +- `VINCI_QWEN_BASE_URL`: the operator HTTPS endpoint. Credentials, query strings, fragments, + redirects, IPv6, and private/local/reserved DNS answers are refused; loopback HTTP exists only for + tests. Every connection uses the public IPv4 addresses resolved and pinned during readiness. +- `VINCI_QWEN_SECRET_REF=file:/absolute/private/path`: the worker opens a private, regular, + non-symlinked credential file and passes only inherited descriptor 3 to the child. The reference + is removed before spawn, the child consumes and closes the descriptor during provider bootstrap, + and neither the secret nor its reference is put in general child environment, argv, logs, or a + generated file. The WorkOrder prompt is sent on stdin, never argv. +- `VINCI_QWEN_QUALIFICATION_FILE` plus its exact `VINCI_QWEN_QUALIFICATION_SHA256`, and an + independently controlled Ed25519 trust key plus `VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_SHA256` and + `VINCI_QWEN_QUALIFICATION_ISSUER`. Qualification bytes and key bytes must be non-writable regular + files. An unsigned or self-admitted record is invalid. +- a deterministic Governor lease and worker-derived WorkOrder, Run, and Attempt identities. Model + output supplies none of them. Each Worker attempt has its own session; every bounded transport + retry is recorded beneath that Attempt with a distinct idempotency key. + +The signed v2 envelope has a maximum seven-day lifetime, cites a canary no more than 24 hours old, +and binds the independent review message/body digest and burn-in report digest. Its closed payload +binds the endpoint and served-identity digests; exact model and immutable revision; runtime +engine/version/artifact/launch-arguments tuple; exact WorkOrder prompt; full assembled system +prompt; ordered tool names, complete tool schemas, and governed tool policy; client and extension +builds; outbound encoding; SSE/tool-call/usage capabilities; total deadline, retry and retry-delay +caps; request/success/error body byte bounds; context/output limits; concurrency ceiling; and USD +pricing basis/rates. Any missing, extra, stale, mismatched, or unverified field refuses registration. + +Authenticated `/health` and `/v1/models` must succeed while anonymous requests to both are refused. +The models response and every successful inference response must repeat the exact model, revision, +endpoint identity, and runtime tuple. The chat transport calls only the qualified URL, refuses +redirects, keeps one total deadline across headers/body/retries, bounds error and successful SSE +bodies, requires strict OpenAI chunk object/model/usage identities and `[DONE]`, and disables SDK +retries. Real HTTP 500s and transport/protocol failures count toward an atomically persisted circuit; +three failures open it for 60 seconds by default. `VINCI_QWEN_CIRCUIT_THRESHOLD` and +`VINCI_QWEN_CIRCUIT_OPEN_MS` are bounded operator overrides. + +Concurrency defaults to 1. The signed schema understands only the ladder +`1 → 2 → 4 → 8 → 16 → 24 → 32`, never an intermediate value, and never above Ayush's advertised +ceiling. Any stage above 1 must cite the immediately prior stage with at least 168 continuous hours +and 1,000 WorkOrders, 100% acceptance pass and usage coverage, at most 0.5% transport errors, and +zero identity failures, verification failures, circuit opens, resource alarms, or Governor stops. +Promotion is a new independent review and signature; it is never automatic. Today the runtime has +only its single-process permit, so every signed value above 1 still fails closed with +`fleet_permit_authority_missing`. A future fleet authority must issue bounded, expiring, fenced +permits keyed by WorkOrder/Run/Attempt, enforce the signed and advertised ceilings atomically across +workers, and define a bounded queue before any stage above 1 can run. Qwen output gets a `vinci-qwen-output-label` session record marking it non-authoritative and requiring independent checking. It is never permission, a Governor ruling, merge authorization, @@ -366,48 +376,58 @@ VINCI_QWEN_SECRET_REF=file:/run/secrets/vinci-qwen-token \ node --experimental-strip-types vinci/extensions/lib/qwen-runtime.ts --canary ``` -The canary requires the endpoint's models response (or headers) to expose the served revision and -runtime tuple. It requests one streaming `report_ready` tool call, checks that the assembled -arguments are exactly `{ "status": "ready" }`, and prints JSON to stdout. It does not write or -admit a qualification record. +The canary requests one streaming `report_ready` tool call, requires exact arguments +`{ "status": "ready" }` plus strict usage, and prints a non-authoritative JSON report. It cannot +write or admit qualification. -After independently reviewing that report, generate the exact per-WorkOrder qualification bytes -locally. The command writes JSON to stdout only; redirect it to an operator-owned file, make that -file non-writable, and pin its SHA-256 in the service environment. Required non-secret inputs are -the endpoint, prompt file, ordered tool list, served revision, runtime engine/version, runtime -artifact and arguments digests, context/output limits, and four estimated per-million-token rates. -Defaults are timeout 120 seconds, one retry with a 5-second cap, and concurrency 1. Admission is an -explicit operator act (`VINCI_QWEN_ADMIT=qualified`), never an inference result: +After reviewing the canary and numeric burn-in report, generate an **unsigned request** for the +never-builder reviewer. The command prints JSON only. The full system-prompt, tool-schema, canary, +burn-in, and WorkOrder-prompt files must be operator-owned and non-writable: ``` -VINCI_QWEN_ADMIT=qualified \ VINCI_QWEN_BASE_URL=https://operator-endpoint.example \ VINCI_QWEN_QUALIFICATION_PROMPT_FILE=/absolute/work-order-prompt.txt \ +VINCI_QWEN_QUALIFICATION_SYSTEM_PROMPT_FILE=/absolute/full-system-prompt.txt \ +VINCI_QWEN_QUALIFICATION_TOOL_SCHEMAS_FILE=/absolute/ordered-tool-schemas.json \ VINCI_QWEN_QUALIFICATION_TOOLS='["read","grep","find","ls","bash","edit","write"]' \ +VINCI_QWEN_CANARY_REPORT_FILE=/absolute/canary-v2.json \ +VINCI_QWEN_BURN_IN_REPORT_FILE=/absolute/burn-in-v1.json \ VINCI_QWEN_SERVED_REVISION=<40-or-64-hex> \ VINCI_QWEN_RUNTIME_ENGINE=vllm \ VINCI_QWEN_RUNTIME_VERSION= \ VINCI_QWEN_RUNTIME_ARTIFACT_SHA256=<64hex> \ VINCI_QWEN_RUNTIME_ARGUMENTS_SHA256=<64hex> \ +VINCI_QWEN_ENDPOINT_IDENTITY_SHA256=<64hex> \ +VINCI_QWEN_CLIENT_BUILD_SHA256=<64hex> \ +VINCI_QWEN_EXTENSION_BUILD_SHA256=<64hex> \ +VINCI_QWEN_ADVERTISED_MAX_CONCURRENCY=<1..32> \ VINCI_QWEN_CONTEXT_WINDOW= VINCI_QWEN_MAX_TOKENS= \ +VINCI_QWEN_PRICE_BASIS= \ VINCI_QWEN_INPUT_PER_MILLION_USD= \ VINCI_QWEN_OUTPUT_PER_MILLION_USD= \ VINCI_QWEN_CACHE_READ_PER_MILLION_USD= \ VINCI_QWEN_CACHE_WRITE_PER_MILLION_USD= \ -node --experimental-strip-types vinci/extensions/lib/qwen-runtime.ts --qualification-template +node --experimental-strip-types vinci/extensions/lib/qwen-runtime.ts --qualification-request ``` +The independent reviewer verifies the evidence, adds issuer/timestamps/review provenance, and signs +the canonical qualification with the separately controlled key. The Qwen builder/operator must not +possess that signing key. Requalification is mandatory after a failed/expired canary or any change +to capabilities/limits, client build, endpoint identity/address policy, model/revision, outbound +encoding, pricing basis, runtime artifact/arguments, system prompt, or tool schema/policy. + Per-attempt telemetry remains in the existing economics summary: `work_order_id`, `session_id` (Run), `attempt_label`, `started_at`/`finished_at` (wall latency), terminal `local_result`, and the per-provider/model roll-up of calls, input/cache/output/reasoning tokens and estimated micro-USD. The route is `single-provider-no-automatic-fallback` and names Qwen when inference occurred. -Burn-in is deliberately small and ordered: one canary; one WorkOrder at concurrency 1; three -sequential WorkOrders; then a 15-minute soak of batches of at most four WorkOrders with concurrency -still 1. Review token totals, wall latency (`started_at`/`finished_at`), terminal outcome, estimated -cost, readiness failures, and circuit state after each step. Raise qualified concurrency to 2 only -after every acceptance criterion passes; repeat the soak before any further explicit increase. -Hundreds of millions of available tokens are capacity, not an acceptance signal. +Stop admission and drain in-flight work on any identity/auth/signature mismatch, failed canary, +acceptance or verification failure, missing/malformed usage, circuit open, resource alarm, Governor +stop/denial, or a transport error rate above 0.5%. Do not skip a ladder stage and do not resume a +stopped stage: `safe_resume` is always `false`; recovery requires fresh canary/burn-in evidence and a +new independent qualification. Hundreds of millions of available tokens are capacity, not an +acceptance signal. There is no automatic fallback: OpenRouter is a new, explicitly authorized +attempt with its own envelope, lease, Run, Attempt, session, and accounting. ### Base checkout (`baseRef` / `baseCommit`) @@ -653,8 +673,9 @@ configured nothing changes (no downgrade), so soak boxes may run without it. - `VINCI_DECLARATION_REFRESH_S`: how often (seconds) a governed daemon re-posts its capability declaration; default `21600` (6h), and anything that is not a positive number falls back to the default. It must stay comfortably below the Governor's `VGC_DECLARATION_MAX_AGE_S` (default 86400), which is when a declaration expires and admission starts answering `eligible: false, reason: stale_declaration`. **The default is chosen against row retention, not liveness** (gpu-control §32): the Governor's `worker_declarations` table is append-only with a DELETE trigger and every refresh writes an audit row, so the volume cannot be pruned later. 6h keeps four refreshes inside the 24h window — three consecutive failed re-posts can be absorbed before one goes stale — at a quarter the rows of hourly, which buys no liveness at all - `GH_TOKEN`: (optional) GitHub machine user token for cloning/pushing private repos and creating PRs - `OPENROUTER_API_KEY`: (or provider-specific key) via vinci's standard configuration -- `VINCI_QWEN_BASE_URL` + `VINCI_QWEN_SECRET_REF`: direct Qwen endpoint and credential reference; - never place the credential value in worker configuration +- `VINCI_QWEN_BASE_URL` + `VINCI_QWEN_SECRET_REF=file:/absolute/private/path`: direct Qwen endpoint + and worker-only credential-file reference; the worker converts the reference to child descriptor + 3 and removes it before spawn. Never place the credential value in worker configuration Never hardcode. Use systemd SecureString parameters, AWS Secrets Manager, or similar. @@ -810,12 +831,13 @@ byte-for-byte what it was. **The child's environment (exact allowlist).** Copied verbatim from the daemon when set: `PATH`, `LANG`, `VINCI_ENV`, `VINCI_BASE_URL`, `VINCI_PLATFORM_URL`, `VINCI_NO_BOOTSTRAP_HEAL`, `VINCI_TOOL_BOOTSTRAP`, `VINCI_SHOW_OTHER_PROVIDERS`, `VINCI_SOURCE_CLI`, and the Qwen -endpoint/secret-reference/qualification/circuit settings above — the variables +endpoint/worker-only secret-reference/qualification/circuit settings above — the variables `vinci/bin/vinci` and the install shim read to find the backend and the run mode. Plus **only** the key the envelope's `provider:` authenticates with: `OPENROUTER_API_KEY` for `openrouter`, `VINCI_API_KEY` for `vinci`, `VINCI_INTERNAL_DEEPINFRA_API_KEY` for `deepinfra` (an unknown provider gets no key and the launcher refuses it, as today). `qwen-h200` carries no credential value through -the allowlist; its extension resolves the narrow reference during bootstrap. Set by the daemon, never copied: `HOME` and +the allowlist; immediately before spawn the worker replaces its narrow file reference with inherited +descriptor 3, and the extension consumes that descriptor during bootstrap. Set by the daemon, never copied: `HOME` and `TMPDIR` (per attempt), `VINCI_CODING_AGENT_DIR` and `PI_CODING_AGENT_DIR` (both spellings, both `.home/agent` — the daemon's own slot is **not** passed through: it holds `auth.json` for every provider, every prior session and `bin/`), `VINCI_HOME` (the launcher's install root — the diff --git a/vinci/worker/cleanroom.mjs b/vinci/worker/cleanroom.mjs index d1a756cd8..68a1b2124 100644 --- a/vinci/worker/cleanroom.mjs +++ b/vinci/worker/cleanroom.mjs @@ -110,6 +110,9 @@ export const PROVIDER_KEY_ENV = Object.freeze({ "VINCI_QWEN_SECRET_REF", "VINCI_QWEN_QUALIFICATION_FILE", "VINCI_QWEN_QUALIFICATION_SHA256", + "VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_FILE", + "VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_SHA256", + "VINCI_QWEN_QUALIFICATION_ISSUER", "VINCI_QWEN_CIRCUIT_THRESHOLD", "VINCI_QWEN_CIRCUIT_OPEN_MS", ], @@ -126,6 +129,9 @@ export const PROVIDER_CREDENTIAL_ENV = Object.freeze([ "VINCI_QWEN_SECRET_REF", "VINCI_QWEN_QUALIFICATION_FILE", "VINCI_QWEN_QUALIFICATION_SHA256", + "VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_FILE", + "VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_SHA256", + "VINCI_QWEN_QUALIFICATION_ISSUER", "VINCI_QWEN_CIRCUIT_THRESHOLD", "VINCI_QWEN_CIRCUIT_OPEN_MS", "AI_GATEWAY_API_KEY", @@ -176,15 +182,6 @@ export const PROVIDER_CREDENTIAL_ENV = Object.freeze([ "ZAI_CODING_CN_API_KEY", ]); -const QWEN_ENV_SECRET_REFERENCE = /^env:([A-Z][A-Z0-9_]{0,127})$/; - -function qwenSecretEnvName(base) { - const match = typeof base.VINCI_QWEN_SECRET_REF === "string" - ? base.VINCI_QWEN_SECRET_REF.match(QWEN_ENV_SECRET_REFERENCE) - : null; - return match?.[1]; -} - // The provider boundary for the NORMAL (non-clean-room) path. // // PROVIDER_KEY_ENV above promises a child gets ONLY the key its envelope's provider @@ -212,8 +209,6 @@ export function providerScopedEnv({ base = process.env, provider, agentDir }) { const keep = new Set(Object.hasOwn(PROVIDER_KEY_ENV, provider) ? PROVIDER_KEY_ENV[provider] : []); const env = { ...base }; for (const key of PROVIDER_CREDENTIAL_ENV) if (!keep.has(key)) delete env[key]; - const referencedQwenSecret = qwenSecretEnvName(base); - if (provider !== "qwen-h200" && referencedQwenSecret) delete env[referencedQwenSecret]; // Do not let normal mode's provider selection be bypassed by the daemon's shared auth.json. // This is a resolution boundary, not uid isolation: a same-uid child can still deliberately // read the daemon's files. Both launchers resolve stored credentials from this isolated slot. @@ -247,10 +242,6 @@ export function cleanRoomEnv({ base = process.env, provider, homeDir, tmpDir }) // Same prototype-chain hazard as providerScopedEnv below. const providerKeys = Object.hasOwn(PROVIDER_KEY_ENV, provider) ? PROVIDER_KEY_ENV[provider] : []; for (const key of providerKeys) if (base[key] !== undefined) env[key] = base[key]; - const referencedQwenSecret = qwenSecretEnvName(base); - if (provider === "qwen-h200" && referencedQwenSecret && base[referencedQwenSecret] !== undefined) { - env[referencedQwenSecret] = base[referencedQwenSecret]; - } const vinciHome = base.VINCI_HOME ?? (base.HOME ? join(base.HOME, ".vinci-code") : undefined); if (vinciHome !== undefined) env.VINCI_HOME = vinciHome; env.HOME = homeDir; diff --git a/vinci/worker/run.mjs b/vinci/worker/run.mjs index d1053559e..42bd2289f 100644 --- a/vinci/worker/run.mjs +++ b/vinci/worker/run.mjs @@ -2,7 +2,9 @@ import { spawn } from "node:child_process"; import { createHash, randomBytes } from "node:crypto"; import { closeSync, + constants, existsSync, + fstatSync, fsyncSync, lstatSync, mkdirSync, @@ -15,6 +17,7 @@ import { writeFileSync, } from "node:fs"; import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; import { resolveBin } from "./build.mjs"; import { command } from "./exec.mjs"; @@ -25,11 +28,44 @@ import { readSessionState } from "./session-read.mjs"; const REPO = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; const utf8Decoder = new TextDecoder("utf-8", { fatal: true }); +const WORKER_DIR = dirname(fileURLToPath(import.meta.url)); +const MAX_QWEN_PROMPT_BYTES = 1024 * 1024; function sha256(value) { return createHash("sha256").update(value).digest("hex"); } +function openQwenSecretReference(reference) { + if (typeof reference !== "string" || !reference.startsWith("file:")) { + throw blocked("qwen_credential_invalid", "qwen_credential_invalid: qwen-h200 requires a file:/absolute/private/path secret reference"); + } + const path = reference.slice(5); + if (!isAbsolute(path)) throw blocked("qwen_credential_invalid", "qwen_credential_invalid: Qwen secret path must be absolute"); + let before; + let descriptor; + try { + before = lstatSync(path); + descriptor = openSync(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + const after = fstatSync(descriptor); + if ( + !before.isFile() || before.isSymbolicLink() || before.nlink !== 1 || (before.mode & 0o077) !== 0 || + !after.isFile() || after.nlink !== 1 || (after.mode & 0o077) !== 0 || before.dev !== after.dev || before.ino !== after.ino + ) { + throw new Error("unsafe identity or permissions"); + } + return descriptor; + } catch (error) { + if (descriptor !== undefined) closeSync(descriptor); + throw blocked("qwen_credential_invalid", `qwen_credential_invalid: Qwen secret descriptor could not be opened safely (${error.message})`); + } +} + +function qwenExtensionBuildSha256() { + const provider = readFileSync(join(WORKER_DIR, "..", "extensions", "vinci-qwen-provider.ts")); + const runtime = readFileSync(join(WORKER_DIR, "..", "extensions", "lib", "qwen-runtime.ts")); + return sha256(Buffer.concat([Buffer.from("vinci-qwen-provider.ts\0"), provider, Buffer.from("\0qwen-runtime.ts\0"), runtime])); +} + function canonicalBytes(value) { return Buffer.from(`${canonicalize(value)}\n`, "utf8"); } @@ -1208,8 +1244,13 @@ export function runVinci({ envelope, repoDir, stateDir, taskId, sessionId, env, const killGraceMs = Number(process.env.VINCI_WORKER_KILL_GRACE_MS) || 30_000; const abortKillGraceMs = Number(process.env.VINCI_WORKER_LEASE_KILL_GRACE_MS) || 10_000; const tools = Array.isArray(envelope.tools) && envelope.tools.length > 0 ? envelope.tools.join(",") : "read,grep,find,ls,bash,edit,write"; + const transmittedPrompt = envelope.provider === "qwen-h200" ? envelope.spec.trim() : envelope.spec; + if (envelope.provider === "qwen-h200" && (!transmittedPrompt || Buffer.byteLength(transmittedPrompt) > MAX_QWEN_PROMPT_BYTES)) { + throw blocked("qwen_prompt_invalid", `qwen_prompt_invalid: Qwen WorkOrder prompt must be 1-${MAX_QWEN_PROMPT_BYTES} UTF-8 bytes`); + } const taskEnvironment = applyEnvDelta(env ?? process.env, envDelta); taskEnvironment.VINCI_UPDATE_DISABLED = "1"; + let qwenSecretReference; // The direct H200 lane is one exact, pre-qualified provider. These values are derived by the // worker, not accepted from the model or repository, and bind the provider extension to the // WorkOrder/Run/Attempt plus the exact prompt and tool surface for this invocation. @@ -1237,9 +1278,21 @@ export function runVinci({ envelope, repoDir, stateDir, taskId, sessionId, env, taskEnvironment.VINCI_QWEN_WORK_ORDER_ID = workOrderId; taskEnvironment.VINCI_QWEN_RUN_ID = sessionId; taskEnvironment.VINCI_QWEN_ATTEMPT_ID = `${taskId}/${attempt}`; - taskEnvironment.VINCI_QWEN_PROMPT_SHA256 = sha256(envelope.spec); - taskEnvironment.VINCI_QWEN_TOOLS_SHA256 = sha256(canonicalize(Array.isArray(envelope.tools) && envelope.tools.length > 0 ? envelope.tools : tools.split(","))); + const orderedTools = Array.isArray(envelope.tools) && envelope.tools.length > 0 ? envelope.tools : tools.split(","); + taskEnvironment.VINCI_QWEN_PROMPT_SHA256 = sha256(transmittedPrompt); + taskEnvironment.VINCI_QWEN_TOOLS_SHA256 = sha256(canonicalize(orderedTools)); + taskEnvironment.VINCI_QWEN_TOOL_POLICY_SHA256 = sha256(canonicalize({ + ordered_tools: orderedTools, + unattended_policy: "governed", + authority: "Governor", + safe_resume: false, + })); + taskEnvironment.VINCI_QWEN_CLIENT_BUILD_SHA256 = sha256(readFileSync(resolveBin("vinci"))); + taskEnvironment.VINCI_QWEN_EXTENSION_BUILD_SHA256 = qwenExtensionBuildSha256(); taskEnvironment.VINCI_QWEN_CIRCUIT_FILE = join(stateDir, "qwen-h200", "circuit.json"); + qwenSecretReference = taskEnvironment.VINCI_QWEN_SECRET_REF; + delete taskEnvironment.VINCI_QWEN_SECRET_REF; + taskEnvironment.VINCI_QWEN_SECRET_FD = "3"; } else { for (const name of [ "VINCI_QWEN_SELECTED", @@ -1248,7 +1301,11 @@ export function runVinci({ envelope, repoDir, stateDir, taskId, sessionId, env, "VINCI_QWEN_ATTEMPT_ID", "VINCI_QWEN_PROMPT_SHA256", "VINCI_QWEN_TOOLS_SHA256", + "VINCI_QWEN_TOOL_POLICY_SHA256", + "VINCI_QWEN_CLIENT_BUILD_SHA256", + "VINCI_QWEN_EXTENSION_BUILD_SHA256", "VINCI_QWEN_CIRCUIT_FILE", + "VINCI_QWEN_SECRET_FD", ]) delete taskEnvironment[name]; } for (const name of [ @@ -1263,9 +1320,13 @@ export function runVinci({ envelope, repoDir, stateDir, taskId, sessionId, env, mkdirSync(sessionDir, { recursive: true }); return new Promise((resolveRun) => { - const child = spawn( - resolveBin("vinci"), - [ + let child; + let qwenSecretDescriptor; + try { + if (envelope.provider === "qwen-h200") qwenSecretDescriptor = openQwenSecretReference(qwenSecretReference); + child = spawn( + resolveBin("vinci"), + [ "-p", "--session-id", sessionId, @@ -1277,8 +1338,8 @@ export function runVinci({ envelope, repoDir, stateDir, taskId, sessionId, env, envelope.model, "--tools", tools, - envelope.spec, - ], + ...(envelope.provider === "qwen-h200" ? [] : [envelope.spec]), + ], // Post-0.0.51 rule (#18): a task NEVER runs under a self-updating launcher. The daemon // probes `vinci --version` immediately before this spawn and records it as the task's // `vinci_binary`; with VINCI_UPDATE_DISABLED=1 the launcher cannot swap its payload between @@ -1290,9 +1351,18 @@ export function runVinci({ envelope, repoDir, stateDir, taskId, sessionId, env, // `env` (clean-room mode) may be an allowlisted subset; `envDelta` is the per-run policy // stamp. Debris-authority capabilities are daemon-only and are always removed above. env: taskEnvironment, - stdio: ["ignore", "inherit", "inherit"], + stdio: envelope.provider === "qwen-h200" + ? ["pipe", "inherit", "inherit", qwenSecretDescriptor] + : ["ignore", "inherit", "inherit"], }, - ); + ); + } finally { + if (qwenSecretDescriptor !== undefined) closeSync(qwenSecretDescriptor); + } + if (envelope.provider === "qwen-h200") { + child.stdin.on("error", () => {}); + child.stdin.end(transmittedPrompt); + } let limitTripped = null; let aborted = null; let killTimer; diff --git a/vinci/worker/task.mjs b/vinci/worker/task.mjs index 3b5c39ae0..fbaef9605 100644 --- a/vinci/worker/task.mjs +++ b/vinci/worker/task.mjs @@ -570,11 +570,17 @@ export class TaskLifecycle { startAttempt(task, vinciVersion, builds = {}) { if (this.isTerminal()) throw new Error(`cannot start an attempt on terminal state ${this.state.state}`); const firstAttempt = !(Number.isInteger(this.state.attempt) && this.state.attempt > 0); - const sessionId = typeof this.state.session_id === "string" && this.state.session_id ? this.state.session_id : task.id; + const nextAttempt = (Number.isInteger(this.state.attempt) ? this.state.attempt : 0) + 1; + // Qwen accounting is attempt-scoped. Reusing the task's durable session id blended usage and + // latency from a prior failed Attempt into its retry; a fresh deterministic session prevents + // that while the task id remains the WorkOrder lineage. + const sessionId = task.envelope.provider === "qwen-h200" + ? `${task.id}-qwen-attempt-${nextAttempt}` + : typeof this.state.session_id === "string" && this.state.session_id ? this.state.session_id : task.id; this.state = { ...this.state, task: task.id, - attempt: (Number.isInteger(this.state.attempt) ? this.state.attempt : 0) + 1, + attempt: nextAttempt, session_id: sessionId, state: "PENDING", started_at: new Date().toISOString(), From bc5cd3b3006682de14199a0193d3b9ab360b2ffe Mon Sep 17 00:00:00 2001 From: George Pu <19347973+thegeorgepu@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:14:29 -0400 Subject: [PATCH 3/9] fix(worker): require external qwen admission --- vinci/extensions/lib/qwen-runtime.ts | 335 +++++++++++++++++++++--- vinci/extensions/vinci-qwen-provider.ts | 69 ++++- vinci/test/worker-qwen-provider.mjs | 273 ++++++++++++++----- vinci/worker/README.md | 45 ++-- vinci/worker/cleanroom.mjs | 6 - vinci/worker/run.mjs | 32 ++- 6 files changed, 622 insertions(+), 138 deletions(-) diff --git a/vinci/extensions/lib/qwen-runtime.ts b/vinci/extensions/lib/qwen-runtime.ts index 84ae39459..f927f1285 100644 --- a/vinci/extensions/lib/qwen-runtime.ts +++ b/vinci/extensions/lib/qwen-runtime.ts @@ -14,7 +14,7 @@ import { writeFileSync, } from "node:fs"; import { isIP } from "node:net"; -import { dirname, isAbsolute } from "node:path"; +import { dirname, isAbsolute, join, relative, sep } from "node:path"; import { pathToFileURL } from "node:url"; import type { Context, Model, ProviderHeaders } from "@earendil-works/pi-ai"; import { Agent, fetch as undiciFetch } from "undici"; @@ -43,6 +43,7 @@ const MAX_CANARY_AGE_AT_ISSUE_MS = 24 * 60 * 60 * 1000; const LOCK_WAIT_MS = 1_000; const LOCK_POLL_MS = 10; const SLEEP_CELL = new Int32Array(new SharedArrayBuffer(4)); +const QWEN_AUTHORITY_ROOT = "/run/vinci/qwen-authority"; export const QWEN_REQUEST_ENCODING = Object.freeze({ schema: "vinci.qwen-openai-chat-request.v1", @@ -188,6 +189,33 @@ export type QwenAttemptRecord = { cost_usd: number; input_tokens: number; output_tokens: number; + request_sha256: string; +}; + +export type QwenAuthorityBoundary = { + qualificationTrust: { + issuer: string; + publicKeyFile: string; + publicKeySha256: string; + }; + fleetPermit: { + schema: "vinci.qwen-fleet-permit.v1"; + authority: "vgc-fleet-permit-authority"; + permitId: string; + workOrderId: string; + runId: string; + attemptId: string; + maxConcurrency: number; + lockDirectory: string; + issuedAt: string; + expiresAt: string; + }; +}; + +export type QwenSemanticSettlement = { + accepted?: QwenAttemptRecord; + transportFailed: boolean; + settled: boolean; }; export type QwenRuntimeConfig = { @@ -204,6 +232,11 @@ export type QwenRuntimeConfig = { circuitFile: string; circuitThreshold: number; circuitOpenMs: number; + fleetPermit: { + permitId: string; + lockDirectory: string; + expiresAt: string; + }; attribution: { workOrderId: string; runId: string; @@ -534,6 +567,108 @@ function secureRegularFile(path: string, label: string, maximumBytes: number): B return bytes; } +function secureAuthorityFile(path: string, label: string, maximumBytes: number): Buffer { + const bytes = secureRegularFile(path, label, maximumBytes); + if (lstatSync(path).uid !== 0) fail("authority_boundary_unsafe", `${label} must be owned by root`); + return bytes; +} + +function pathWithin(root: string, candidate: string): boolean { + const remainder = relative(root, candidate); + return remainder !== "" && remainder !== ".." && !remainder.startsWith(`..${sep}`) && !isAbsolute(remainder); +} + +function validateProductionAuthorityRoot(): void { + let stat; + try { + stat = lstatSync(QWEN_AUTHORITY_ROOT); + } catch { + fail("config_unavailable", "Qwen independent authority root is unavailable"); + } + if (!stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== 0 || (stat.mode & 0o022) !== 0) { + fail("authority_boundary_unsafe", "Qwen independent authority root must be a root-owned non-writable directory"); + } +} + +function loadIndependentAuthority( + workOrderId: string, + runId: string, + attemptId: string, + nowMs: number, + injected?: QwenAuthorityBoundary, +): QwenAuthorityBoundary { + let boundary: QwenAuthorityBoundary; + if (injected) boundary = injected; + else { + validateProductionAuthorityRoot(); + const identity = qwenSha256(`${workOrderId}\0${runId}\0${attemptId}`); + const bytes = secureAuthorityFile( + join(QWEN_AUTHORITY_ROOT, `${identity}.json`), + "Qwen independent authority record", + MAX_QUALIFICATION_BYTES, + ); + try { + boundary = JSON.parse(bytes.toString("utf8")) as QwenAuthorityBoundary; + } catch { + fail("authority_boundary_invalid", "Qwen independent authority record is not JSON"); + } + } + exactKeys(boundary, ["qualificationTrust", "fleetPermit"], "independent authority boundary"); + exactKeys(boundary.qualificationTrust, ["issuer", "publicKeyFile", "publicKeySha256"], "qualification trust boundary"); + const trust = boundary.qualificationTrust; + if (!IDENTIFIER.test(trust.issuer) || !isAbsolute(trust.publicKeyFile) || !HEX64.test(trust.publicKeySha256)) { + fail("authority_boundary_invalid", "qualification trust boundary is malformed"); + } + exactKeys( + boundary.fleetPermit, + ["schema", "authority", "permitId", "workOrderId", "runId", "attemptId", "maxConcurrency", "lockDirectory", "issuedAt", "expiresAt"], + "fleet permit", + ); + const permit = boundary.fleetPermit; + if (permit.schema !== "vinci.qwen-fleet-permit.v1" || permit.authority !== "vgc-fleet-permit-authority") { + fail("fleet_permit_invalid", "Qwen requires the external VGC fleet permit authority"); + } + nonEmptyString(permit.permitId, "fleet permit id"); + if (permit.workOrderId !== workOrderId || permit.runId !== runId || permit.attemptId !== attemptId) { + fail("fleet_permit_invalid", "fleet permit attribution does not match this exact WorkOrder/Run/Attempt"); + } + if (permit.maxConcurrency !== 1) fail("fleet_permit_invalid", "current Qwen fleet permit must enforce concurrency 1"); + if (!isAbsolute(permit.lockDirectory)) fail("fleet_permit_invalid", "fleet permit lock directory must be absolute"); + const issued = timestamp(permit.issuedAt, "fleet permit issuedAt"); + const expires = timestamp(permit.expiresAt, "fleet permit expiresAt"); + if (issued.time > nowMs + 30_000 || expires.time <= nowMs || expires.time - issued.time > 5 * 60_000) { + fail("fleet_permit_invalid", "fleet permit must be current and live for no more than five minutes"); + } + if (!injected) { + const keyRoot = join(QWEN_AUTHORITY_ROOT, "keys"); + let keyRootStat; + try { + keyRootStat = lstatSync(keyRoot); + } catch { + fail("config_unavailable", "qualification trust key root is unavailable"); + } + if (!keyRootStat.isDirectory() || keyRootStat.isSymbolicLink() || keyRootStat.uid !== 0 || (keyRootStat.mode & 0o022) !== 0) { + fail("authority_boundary_unsafe", "qualification trust key root must be a root-owned non-writable directory"); + } + if (!pathWithin(keyRoot, trust.publicKeyFile) || dirname(trust.publicKeyFile) !== keyRoot) { + fail("authority_boundary_unsafe", "qualification trust key must be inside the independent authority key root"); + } + if (permit.lockDirectory !== join(QWEN_AUTHORITY_ROOT, "locks")) { + fail("authority_boundary_unsafe", "fleet permit lock directory must be the independent authority lock root"); + } + let lockStat; + try { + lockStat = lstatSync(permit.lockDirectory); + } catch { + fail("config_unavailable", "fleet permit lock directory is unavailable"); + } + if (!lockStat.isDirectory() || lockStat.isSymbolicLink() || lockStat.uid !== 0 || (lockStat.mode & 0o1000) === 0) { + fail("authority_boundary_unsafe", "fleet permit lock directory must be a root-owned sticky directory"); + } + } + return boundary; +} + function readSecretDescriptor(env: NodeJS.ProcessEnv): string { const raw = env.VINCI_QWEN_SECRET_FD; delete env.VINCI_QWEN_SECRET_FD; @@ -642,19 +777,23 @@ export async function pinQwenEndpoint(config: QwenRuntimeConfig, lookupImpl: Qwe config.endpointAddresses = addresses; } -function readQualification(env: NodeJS.ProcessEnv, nowMs: number): { qualification: Qualification; digest: string } { +function readQualification( + env: NodeJS.ProcessEnv, + nowMs: number, + trust: QwenAuthorityBoundary["qualificationTrust"], + injectedAuthority: boolean, +): { qualification: Qualification; digest: string } { const path = env.VINCI_QWEN_QUALIFICATION_FILE; const expectedDigest = env.VINCI_QWEN_QUALIFICATION_SHA256; - const publicKeyPath = env.VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_FILE; - const publicKeyDigest = env.VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_SHA256; - const expectedIssuer = env.VINCI_QWEN_QUALIFICATION_ISSUER; if (!path || !expectedDigest || !HEX64.test(expectedDigest)) fail("config_missing", "qualification file and byte digest pin are required"); - if (!publicKeyPath || !publicKeyDigest || !HEX64.test(publicKeyDigest)) fail("config_missing", "qualification public key file and digest pin are required"); - if (!expectedIssuer || !IDENTIFIER.test(expectedIssuer)) fail("config_missing", "a bounded qualification issuer pin is required"); const bytes = secureRegularFile(path, "Qwen qualification artifact", MAX_QUALIFICATION_BYTES); if (qwenSha256(bytes) !== expectedDigest) fail("qualification_digest_mismatch", "qualification bytes do not match the process pin"); - const publicKeyBytes = secureRegularFile(publicKeyPath, "Qwen qualification public key", 64 * 1024); - if (qwenSha256(publicKeyBytes) !== publicKeyDigest) fail("qualification_key_mismatch", "qualification public key bytes do not match the process pin"); + const publicKeyBytes = injectedAuthority + ? secureRegularFile(trust.publicKeyFile, "Qwen qualification public key", 64 * 1024) + : secureAuthorityFile(trust.publicKeyFile, "Qwen qualification public key", 64 * 1024); + if (qwenSha256(publicKeyBytes) !== trust.publicKeySha256) { + fail("qualification_key_mismatch", "qualification public key bytes do not match the independent authority pin"); + } let parsed: unknown; try { parsed = JSON.parse(bytes.toString("utf8")); @@ -680,13 +819,22 @@ function readQualification(env: NodeJS.ProcessEnv, nowMs: number): { qualificati if (!verify(null, signedBytes, publicKey, Buffer.from(parsed.signature.signature_base64, "base64"))) { fail("qualification_signature_invalid", "qualification signature does not verify under the pinned trust key"); } - return { qualification: validateQualification(parsed.qualification, expectedIssuer, nowMs), digest: expectedDigest }; + return { qualification: validateQualification(parsed.qualification, trust.issuer, nowMs), digest: expectedDigest }; } -export function loadQwenRuntimeConfig(env: NodeJS.ProcessEnv = process.env, nowMs = Date.now()): QwenRuntimeConfig { +export function loadQwenRuntimeConfig( + env: NodeJS.ProcessEnv = process.env, + nowMs = Date.now(), + injectedAuthority?: QwenAuthorityBoundary, +): QwenRuntimeConfig { const urls = normalizeQwenBaseUrl(env.VINCI_QWEN_BASE_URL); const secret = readSecretDescriptor(env); - const admitted = readQualification(env, nowMs); + const workOrderId = env.VINCI_QWEN_WORK_ORDER_ID; + const runId = env.VINCI_QWEN_RUN_ID; + const attemptId = env.VINCI_QWEN_ATTEMPT_ID; + if (!workOrderId || !runId || !attemptId) fail("attribution_missing", "WorkOrder, Run, and Attempt attribution are required"); + const authority = loadIndependentAuthority(workOrderId, runId, attemptId, nowMs, injectedAuthority); + const admitted = readQualification(env, nowMs, authority.qualificationTrust, injectedAuthority !== undefined); const qualification = admitted.qualification; const bindings = qualification.bindings; if (bindings.endpoint_sha256 !== qwenSha256(urls.baseUrl)) fail("endpoint_mismatch", "qualification is bound to a different base URL"); @@ -701,13 +849,9 @@ export function loadQwenRuntimeConfig(env: NodeJS.ProcessEnv = process.env, nowM if (env.VINCI_UNATTENDED_POLICY !== "governed" || !env.VINCI_UNATTENDED_LEASE) { fail("authority_forbidden", "Qwen Worker runs require a deterministic Governor lease"); } - if (qualification.limits.max_concurrency > 1) { - fail("fleet_permit_authority_missing", "concurrency above one requires the future fleet-wide permit authority interface"); + if (qualification.limits.max_concurrency !== authority.fleetPermit.maxConcurrency) { + fail("fleet_permit_invalid", "qualification concurrency differs from the external fleet permit"); } - const workOrderId = env.VINCI_QWEN_WORK_ORDER_ID; - const runId = env.VINCI_QWEN_RUN_ID; - const attemptId = env.VINCI_QWEN_ATTEMPT_ID; - if (!workOrderId || !runId || !attemptId) fail("attribution_missing", "WorkOrder, Run, and Attempt attribution are required"); const circuitFile = env.VINCI_QWEN_CIRCUIT_FILE; if (!circuitFile || !isAbsolute(circuitFile)) fail("config_missing", "VINCI_QWEN_CIRCUIT_FILE must be an absolute path"); return { @@ -719,6 +863,11 @@ export function loadQwenRuntimeConfig(env: NodeJS.ProcessEnv = process.env, nowM circuitFile, circuitThreshold: boundedInteger(Number(env.VINCI_QWEN_CIRCUIT_THRESHOLD ?? "3"), 1, 10, "VINCI_QWEN_CIRCUIT_THRESHOLD"), circuitOpenMs: boundedInteger(Number(env.VINCI_QWEN_CIRCUIT_OPEN_MS ?? "60000"), 1_000, 3_600_000, "VINCI_QWEN_CIRCUIT_OPEN_MS"), + fleetPermit: { + permitId: authority.fleetPermit.permitId, + lockDirectory: authority.fleetPermit.lockDirectory, + expiresAt: authority.fleetPermit.expiresAt, + }, attribution: { workOrderId, runId, attemptId }, }; } @@ -982,9 +1131,9 @@ export async function probeQwenReadiness( export async function ensureQwenReady( env: NodeJS.ProcessEnv = process.env, - options: { fetchImpl?: QwenFetch; signal?: AbortSignal; nowMs?: number; lookupImpl?: QwenLookup } = {}, + options: { fetchImpl?: QwenFetch; signal?: AbortSignal; nowMs?: number; lookupImpl?: QwenLookup; authorityBoundary?: QwenAuthorityBoundary } = {}, ): Promise { - const config = loadQwenRuntimeConfig(env, options.nowMs); + const config = loadQwenRuntimeConfig(env, options.nowMs, options.authorityBoundary); await pinQwenEndpoint(config, options.lookupImpl); await probeQwenReadiness(config, options); return config; @@ -1072,12 +1221,12 @@ async function cancellableDelay(delayMs: number, signal: AbortSignal): Promise void, injectedFetch?: QwenFetch, + semanticSettlement: QwenSemanticSettlement = { transportFailed: false, settled: false }, ): QwenFetch { return async (input, init = {}) => { assertQwenCircuitClosed(config); + const sourceRequest = input instanceof Request ? input : undefined; const target = new URL(typeof input === "string" || input instanceof URL ? input : input.url); - if (target.href !== config.chatUrl || (init.method ?? "GET").toUpperCase() !== "POST") { + const method = (init.method ?? sourceRequest?.method ?? "GET").toUpperCase(); + if (target.href !== config.chatUrl || method !== "POST") { fail("ssrf_forbidden", "inference transport may call only the exact qualified chat-completions URL"); } - if (bodyByteLength(init.body) > config.qualification.limits.max_request_bytes) { + const requestBytes = init.body !== undefined + ? bodyBytes(init.body) + : sourceRequest + ? Buffer.from(await sourceRequest.clone().arrayBuffer()) + : Buffer.alloc(0); + if (requestBytes.length > config.qualification.limits.max_request_bytes) { fail("request_oversized", "inference request exceeded the signed request-byte bound"); } + let finalPayload: unknown; + try { + finalPayload = JSON.parse(requestBytes.toString("utf8")); + } catch { + fail("request_invalid", "final serialized Qwen request body is not JSON"); + } + validateQwenOutboundPayload(config, finalPayload); + const requestSha256 = qwenSha256(requestBytes); const controller = new AbortController(); const timeout = setTimeout(() => controller.abort("total_timeout"), config.qualification.limits.total_timeout_ms); - const externalSignal = init.signal; + const externalSignal = init.signal ?? sourceRequest?.signal; const abort = () => controller.abort(externalSignal?.reason ?? "cancelled"); if (externalSignal?.aborted) abort(); else externalSignal?.addEventListener("abort", abort, { once: true }); @@ -1185,17 +1350,21 @@ export function createQwenInferenceFetch( const started = Date.now(); const startedAt = new Date(started).toISOString(); const real = injectedFetch ? null : realPinnedFetch(config, transportAttempt); - const headers = new Headers(init.headers); + const headers = new Headers(sourceRequest?.headers); + if (init.headers) new Headers(init.headers).forEach((value, key) => headers.set(key, value)); + headers.set("authorization", `Bearer ${config.secret}`); + for (const [name, value] of Object.entries(qwenProviderHeaders(config, requestId))) { + if (value !== null) headers.set(name, value); + } headers.set("x-vinci-idempotency-key", `${requestId}/${transportAttempt}`); + headers.set("x-vinci-qwen-request-sha256", requestSha256); let attemptReported = false; const finishAttempt = (outcome: string, status: number | null, inputTokens = 0, outputTokens = 0) => { if (attemptReported) return; attemptReported = true; real?.close(); const finished = Date.now(); - if (outcome === "success") recordQwenCircuitOutcome(config, true, "success", finished); - else if (outcome !== "cancelled") recordQwenCircuitOutcome(config, false, outcome, finished); - onAttempt({ + const record: QwenAttemptRecord = { request_id: requestId, transport_attempt: transportAttempt, started_at: startedAt, @@ -1203,17 +1372,28 @@ export function createQwenInferenceFetch( latency_ms: finished - started, outcome, status, - cost_usd: outcome === "success" + cost_usd: outcome === "transport_accepted" ? (inputTokens * config.qualification.pricing.input_per_million_usd + outputTokens * config.qualification.pricing.output_per_million_usd) / 1_000_000 : 0, input_tokens: inputTokens, output_tokens: outputTokens, - }); + request_sha256: requestSha256, + }; + if (outcome === "transport_accepted") { + semanticSettlement.accepted = record; + } else { + semanticSettlement.transportFailed = true; + semanticSettlement.settled = true; + if (outcome !== "cancelled") recordQwenCircuitOutcome(config, false, outcome, finished); + onAttempt(record); + } }; let response: Response; try { response = await (injectedFetch ?? real!.fetchImpl)(target, { ...init, + method, + body: requestBytes, headers, redirect: "error", signal: controller.signal, @@ -1275,6 +1455,30 @@ export function createQwenInferenceFetch( }; } +export function settleQwenSemanticOutcome( + config: QwenRuntimeConfig, + settlement: QwenSemanticSettlement, + accepted: boolean, + reason: string, + onAttempt: (record: QwenAttemptRecord) => void, +): void { + if (settlement.settled) return; + settlement.settled = true; + if (!settlement.accepted) { + if (!settlement.transportFailed) recordQwenCircuitOutcome(config, false, reason); + return; + } + const finished = Date.now(); + const outcome = accepted ? "success" : reason; + recordQwenCircuitOutcome(config, accepted, outcome, finished); + onAttempt({ + ...settlement.accepted, + finished_at: new Date(finished).toISOString(), + latency_ms: Math.max(0, finished - Date.parse(settlement.accepted.started_at)), + outcome, + }); +} + export function assertQwenContextBindings(config: QwenRuntimeConfig, context: Context): void { const bindings = config.qualification.bindings; const workOrderMessage = context.messages.find((message) => message.role === "user"); @@ -1315,6 +1519,34 @@ export function validateQwenOutboundPayload(config: QwenRuntimeConfig, payload: fail("request_invalid", "outbound request must stream the exact qualified model and messages"); } if (!Array.isArray(value.tools)) fail("request_invalid", "outbound request must carry the qualified tool schemas"); + const messages = value.messages as Array; + const records = messages.filter((message): message is Record => Boolean(message) && typeof message === "object" && !Array.isArray(message)); + if (records.length !== messages.length) fail("request_invalid", "outbound messages must be objects"); + const system = records.find((message) => message.role === "system" || message.role === "developer"); + const workOrder = records.find((message) => message.role === "user"); + if (!system || typeof system.content !== "string" || qwenSha256(system.content) !== config.qualification.bindings.system_prompt_sha256) { + fail("system_prompt_mismatch", "final serialized request does not contain the exact qualified system prompt"); + } + if (!workOrder || typeof workOrder.content !== "string" || qwenSha256(workOrder.content) !== config.qualification.bindings.work_order_prompt_sha256) { + fail("prompt_mismatch", "final serialized request does not contain the exact qualified WorkOrder prompt"); + } + const wireTools = (value.tools as Array).map((tool) => { + if (!tool || typeof tool !== "object" || Array.isArray(tool)) fail("request_invalid", "outbound tool schema must be an object"); + const wire = tool as Record; + exactKeys(wire, ["type", "function"], "outbound tool schema"); + if (wire.type !== "function" || !wire.function || typeof wire.function !== "object" || Array.isArray(wire.function)) { + fail("request_invalid", "outbound tool schema must be an OpenAI function"); + } + const fn = wire.function as Record; + exactKeys(fn, ["name", "description", "parameters"], "outbound tool function"); + return { name: fn.name, description: fn.description, parameters: fn.parameters }; + }); + if (qwenSha256(qwenCanonical(wireTools.map((tool) => tool.name))) !== config.qualification.bindings.tool_names_sha256) { + fail("tools_mismatch", "final serialized request tool order differs from qualification"); + } + if (qwenSha256(qwenCanonical(wireTools)) !== config.qualification.bindings.tool_schemas_sha256) { + fail("tool_schema_mismatch", "final serialized request tool schemas differ from qualification"); + } if (typeof value.max_tokens !== "number" || !Number.isSafeInteger(value.max_tokens) || value.max_tokens < 1 || value.max_tokens > config.qualification.limits.max_tokens) { fail("request_invalid", "outbound max_tokens exceeds the signed bound"); } @@ -1324,6 +1556,32 @@ export function validateQwenOutboundPayload(config: QwenRuntimeConfig, payload: } } +export function acquireQwenFleetPermit(config: QwenRuntimeConfig): () => void { + const expiresAt = Date.parse(config.fleetPermit.expiresAt); + if (!Number.isFinite(expiresAt) || expiresAt - Date.now() < config.qualification.limits.total_timeout_ms) { + fail("fleet_permit_expired", "the external Qwen fleet permit cannot cover the full bounded request deadline"); + } + const lockPath = join(config.fleetPermit.lockDirectory, "qwen-h200-concurrency-1"); + try { + mkdirSync(lockPath, { mode: 0o700 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") { + fail("concurrency_exceeded", "the external Qwen fleet permit is already held by another provider process"); + } + fail("fleet_permit_unavailable", "the external Qwen fleet permit could not be acquired"); + } + let released = false; + return () => { + if (released) return; + released = true; + try { + rmdirSync(lockPath); + } catch { + fail("fleet_permit_release_failed", "the external Qwen fleet permit could not be released cleanly"); + } + }; +} + function canaryConfig(env: NodeJS.ProcessEnv): QwenRuntimeConfig { const urls = normalizeQwenBaseUrl(env.VINCI_QWEN_BASE_URL); const secret = readCanarySecret(env.VINCI_QWEN_SECRET_REF); @@ -1404,6 +1662,7 @@ function canaryConfig(env: NodeJS.ProcessEnv): QwenRuntimeConfig { circuitFile: "/canary/unused", circuitThreshold: 1, circuitOpenMs: 1_000, + fleetPermit: { permitId: "canary-only", lockDirectory: "/canary", expiresAt: new Date(Date.now() + 1_000).toISOString() }, attribution: { workOrderId: "canary-read-only", runId: "canary-read-only", attemptId: "canary-read-only/1" }, }; } @@ -1648,9 +1907,6 @@ export function scrubQwenBootstrapEnvironment(env: NodeJS.ProcessEnv = process.e for (const name of [ "VINCI_QWEN_QUALIFICATION_FILE", "VINCI_QWEN_QUALIFICATION_SHA256", - "VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_FILE", - "VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_SHA256", - "VINCI_QWEN_QUALIFICATION_ISSUER", ]) delete env[name]; } @@ -1660,6 +1916,7 @@ export function qwenProviderHeaders(config: QwenRuntimeConfig, requestId: string "x-vinci-run-id": config.attribution.runId, "x-vinci-attempt-id": config.attribution.attemptId, "x-vinci-qwen-request-id": requestId, + "x-vinci-qwen-fleet-permit-id": config.fleetPermit.permitId, "x-vinci-qwen-output-authority": "non-authoritative", "x-vinci-qwen-qualification-sha256": config.qualificationSha256, }; diff --git a/vinci/extensions/vinci-qwen-provider.ts b/vinci/extensions/vinci-qwen-provider.ts index 51a7b6012..5a250d4a5 100644 --- a/vinci/extensions/vinci-qwen-provider.ts +++ b/vinci/extensions/vinci-qwen-provider.ts @@ -8,6 +8,7 @@ import { } from "@earendil-works/pi-ai"; import { streamSimpleOpenAICompletions } from "@earendil-works/pi-ai/compat"; import { + acquireQwenFleetPermit, assertQwenCircuitClosed, assertQwenContextBindings, createQwenInferenceFetch, @@ -20,7 +21,10 @@ import { qwenSha256, scrubQwenBootstrapEnvironment, type QwenAttemptRecord, + type QwenFetch, type QwenRuntimeConfig, + type QwenSemanticSettlement, + settleQwenSemanticOutcome, validateQwenOutboundPayload, } from "./lib/qwen-runtime.ts"; @@ -48,10 +52,37 @@ function validUsage(message: { usage?: unknown }): boolean { return Math.abs((cost.total as number) - costParts) <= Number.EPSILON * Math.max(1, costParts) * 8; } +function validSemanticMessage(message: { content?: unknown; stopReason?: unknown }, context: Context): boolean { + if (!Array.isArray(message.content) || !["stop", "length", "toolUse"].includes(String(message.stopReason))) return false; + const qualifiedTools = new Set((context.tools ?? []).map((tool) => tool.name)); + let toolCalls = 0; + for (const block of message.content) { + if (!block || typeof block !== "object") return false; + const value = block as Record; + if (value.type === "text") { + if (typeof value.text !== "string") return false; + continue; + } + if (value.type === "thinking") { + if (typeof value.thinking !== "string") return false; + continue; + } + if ( + value.type !== "toolCall" || + typeof value.id !== "string" || !value.id || + typeof value.name !== "string" || !qualifiedTools.has(value.name) || + !value.arguments || typeof value.arguments !== "object" || Array.isArray(value.arguments) + ) return false; + toolCalls += 1; + } + return (message.stopReason === "toolUse") === (toolCalls > 0); +} + export function qwenProviderConfig( runtime: QwenRuntimeConfig, streamOpenAI = streamSimpleOpenAICompletions, onAttempt: (record: QwenAttemptRecord) => void = () => {}, + injectedFetch?: QwenFetch, ) { let inFlight = 0; let requestOrdinal = 0; @@ -68,15 +99,21 @@ export function qwenProviderConfig( if (inFlight >= runtime.qualification.limits.max_concurrency) { throw new Error("qwen_concurrency_exceeded: the qualified single-request bound is already in use"); } + const releaseFleetPermit = acquireQwenFleetPermit(runtime); inFlight += 1; let released = false; const release = () => { if (released) return; released = true; - inFlight -= 1; + try { + releaseFleetPermit(); + } finally { + inFlight -= 1; + } }; requestOrdinal += 1; const requestId = qwenSha256(`${runtime.attribution.workOrderId}\0${runtime.attribution.runId}\0${runtime.attribution.attemptId}\0${requestOrdinal}`); + const semanticSettlement: QwenSemanticSettlement = { transportFailed: false, settled: false }; let source; try { source = streamOpenAI( @@ -88,12 +125,10 @@ export function qwenProviderConfig( headers: { ...options?.headers, ...qwenProviderHeaders(runtime, requestId) }, timeoutMs: runtime.qualification.limits.total_timeout_ms, maxRetries: 0, - fetch: createQwenInferenceFetch(runtime, requestId, onAttempt), - onPayload: async (payload, requestModel) => { - const candidate = await options?.onPayload?.(payload, requestModel); - const finalPayload = candidate ?? payload; - validateQwenOutboundPayload(runtime, finalPayload); - return finalPayload; + fetch: createQwenInferenceFetch(runtime, requestId, onAttempt, injectedFetch, semanticSettlement), + onPayload: (payload) => { + validateQwenOutboundPayload(runtime, payload); + return payload; }, } as SimpleStreamOptions & { fetch: typeof globalThis.fetch }, ); @@ -108,9 +143,22 @@ export function qwenProviderConfig( for await (const event of source) { if (event.type === "done" || event.type === "error") { terminalSeen = true; - const message = event.type === "done" ? event.message : event.error; - if (message.provider !== QWEN_PROVIDER || message.model !== QWEN_MODEL || !validUsage(message)) { - throw new Error("qwen_usage_invalid: terminal response lacks exact model identity or strict usage/cost telemetry"); + if (event.type === "error") { + settleQwenSemanticOutcome(runtime, semanticSettlement, false, "parser_error", onAttempt); + } else { + const message = event.message; + if ( + message.provider !== QWEN_PROVIDER || + message.model !== QWEN_MODEL || + !validUsage(message) || + !validSemanticMessage(message, context) + ) { + throw new Error("qwen_semantic_invalid: terminal response failed exact identity, usage, finish, or tool semantics"); + } + // A permit-release failure is itself a failed request. Never persist semantic + // success until the external concurrency authority has been released cleanly. + release(); + settleQwenSemanticOutcome(runtime, semanticSettlement, true, "success", onAttempt); } // Release before the terminal event can resolve result() or become observable to a // caller that immediately starts the next qualified request. @@ -120,6 +168,7 @@ export function qwenProviderConfig( } if (!terminalSeen) throw new Error("qwen_stream_truncated: provider stream ended without a terminal event"); } catch (error) { + settleQwenSemanticOutcome(runtime, semanticSettlement, false, "parser_semantic_invalid", onAttempt); release(); const message = { role: "assistant" as const, diff --git a/vinci/test/worker-qwen-provider.mjs b/vinci/test/worker-qwen-provider.mjs index aea3e92cf..22575b8e3 100644 --- a/vinci/test/worker-qwen-provider.mjs +++ b/vinci/test/worker-qwen-provider.mjs @@ -5,7 +5,7 @@ import { chmodSync, existsSync, mkdirSync, mkdtempSync, openSync, readFileSync, import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; -import { createAssistantMessageEventStream } from "@earendil-works/pi-ai"; +import { streamSimple as sourceStreamSimpleOpenAICompletions } from "../../packages/ai/src/api/openai-completions.ts"; import * as runtime from "../extensions/lib/qwen-runtime.ts"; import { qwenProviderConfig } from "../extensions/vinci-qwen-provider.ts"; import * as cleanroom from "../worker/cleanroom.mjs"; @@ -24,6 +24,7 @@ const canaryFile = join(temp, "canary.json"); const burnInFile = join(temp, "burn-in.json"); const qualificationFile = join(temp, "qualification.json"); const publicKeyFile = join(temp, "qualification-key.pem"); +const permitLockDirectory = join(temp, "permit-locks"); const endpointIdentity = "12".repeat(32); const revision = "ab".repeat(20); const runtimeTuple = { @@ -78,7 +79,42 @@ recordQwenCircuitOutcome({ circuitFile, circuitThreshold: 100, circuitOpenMs: 60 }))); } +async function exerciseCrossProcessFleetPermit(config) { + const release = runtime.acquireQwenFleetPermit(config); + const workerFile = join(temp, "permit-worker.mjs"); + const runtimeUrl = pathToFileURL(join(root, "vinci/extensions/lib/qwen-runtime.ts")).href; + writeFileSync(workerFile, `import { acquireQwenFleetPermit } from ${JSON.stringify(runtimeUrl)}; +const [lockDirectory, permitId, expiresAt] = process.argv.slice(2); +try { + acquireQwenFleetPermit({ fleetPermit: { lockDirectory, permitId, expiresAt }, qualification: { limits: { total_timeout_ms: 1_000 } } }); + process.exitCode = 2; +} catch (error) { + if (error?.code !== "concurrency_exceeded") throw error; +} +`, { mode: 0o600 }); + try { + const child = spawn(process.execPath, [...process.execArgv, workerFile, config.fleetPermit.lockDirectory, config.fleetPermit.permitId, config.fleetPermit.expiresAt], { + stdio: ["ignore", "pipe", "pipe"], + }); + await new Promise((resolveExit, rejectExit) => { + let stderr = ""; + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk) => { stderr += chunk; }); + child.on("error", rejectExit); + child.on("exit", (code) => code === 0 ? resolveExit() : rejectExit(new Error(`permit worker exited ${code}: ${stderr}`))); + }); + } finally { + release(); + } +} + +function freshenPermit(config) { + config.fleetPermit.expiresAt = new Date(Date.now() + 60_000).toISOString(); + return config; +} + writeFileSync(secretFile, "synthetic-test-secret\n", { mode: 0o600 }); +mkdirSync(permitLockDirectory, { mode: 0o700 }); writeFileSync(promptFile, workOrderPrompt, { mode: 0o400 }); writeFileSync(systemPromptFile, systemPrompt, { mode: 0o400 }); writeFileSync(toolSchemasFile, `${JSON.stringify(tools)}\n`, { mode: 0o400 }); @@ -194,8 +230,8 @@ function qualificationFromRequest(overrides = {}, qualificationRequestEnv = requ }; } -function signedEnvelope(qualification) { - const signature = sign(null, Buffer.from(runtime.qwenCanonical(qualification)), privateKey).toString("base64"); +function signedEnvelope(qualification, signingKey = privateKey) { + const signature = sign(null, Buffer.from(runtime.qwenCanonical(qualification)), signingKey).toString("base64"); return { schema: "vinci.qwen-worker-qualification-envelope.v2", qualification, @@ -220,9 +256,6 @@ const baseRuntimeEnv = { VINCI_QWEN_BASE_URL: requestEnv.VINCI_QWEN_BASE_URL, VINCI_QWEN_QUALIFICATION_FILE: qualificationFile, VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest, - VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_FILE: publicKeyFile, - VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_SHA256: runtime.qwenSha256(publicKeyBytes), - VINCI_QWEN_QUALIFICATION_ISSUER: "reviewer:test", VINCI_QWEN_PROMPT_SHA256: qualification.bindings.work_order_prompt_sha256, VINCI_QWEN_TOOLS_SHA256: qualification.bindings.tool_names_sha256, VINCI_QWEN_TOOL_POLICY_SHA256: qualification.bindings.tool_policy_sha256, @@ -238,12 +271,32 @@ const baseRuntimeEnv = { VINCI_QWEN_CIRCUIT_OPEN_MS: "60000", }; +const authorityBoundary = { + qualificationTrust: { + issuer: "reviewer:test", + publicKeyFile, + publicKeySha256: runtime.qwenSha256(publicKeyBytes), + }, + fleetPermit: { + schema: "vinci.qwen-fleet-permit.v1", + authority: "vgc-fleet-permit-authority", + permitId: "permit-test-1", + workOrderId: "wo-test", + runId: "run-test", + attemptId: "task-test/1", + maxConcurrency: 1, + lockDirectory: permitLockDirectory, + issuedAt: "2026-09-04T17:59:00.000Z", + expiresAt: "2026-09-04T18:04:00.000Z", + }, +}; + function runtimeEnv(overrides = {}) { return { ...baseRuntimeEnv, VINCI_QWEN_SECRET_FD: String(openSync(secretFile, "r")), ...overrides }; } function loadConfig(overrides = {}) { - return runtime.loadQwenRuntimeConfig(runtimeEnv(overrides), nowMs); + return runtime.loadQwenRuntimeConfig(runtimeEnv(overrides), nowMs, authorityBoundary); } function identityHeaders(overrides = {}) { @@ -272,6 +325,14 @@ const validSse = [ "data: [DONE]", "", ].join("\n"); +const requestBody = JSON.stringify({ + model: runtime.QWEN_MODEL, + messages: [{ role: "system", content: systemPrompt }, { role: "user", content: workOrderPrompt }], + tools: tools.map(({ name, description, parameters }) => ({ type: "function", function: { name, description, parameters } })), + stream: true, + stream_options: { include_usage: true }, + max_tokens: 64, +}); try { assert.equal(qualification.safe_resume, false); @@ -282,15 +343,38 @@ try { assert.equal(config.secret, "synthetic-test-secret"); assert.equal(config.qualification.provenance.authority, "independent-never-builder-review"); assert.throws( - () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_CLIENT_BUILD_SHA256: "99".repeat(32) }), nowMs), + () => runtime.loadQwenRuntimeConfig(runtimeEnv(), nowMs), + /authority record.*unavailable|config_unavailable/, + "runtime must not derive qualification trust or fleet permission from Worker environment", + ); + assert.throws( + () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_CLIENT_BUILD_SHA256: "99".repeat(32) }), nowMs, authorityBoundary), /client_build_mismatch/, ); + const rogueKeys = generateKeyPairSync("ed25519"); + const roguePublicFile = join(temp, "rogue-key.pem"); + const roguePublicBytes = rogueKeys.publicKey.export({ type: "spki", format: "pem" }); + writeFileSync(roguePublicFile, roguePublicBytes, { mode: 0o400 }); + qualificationDigest = writeQualification(signedEnvelope(qualification, rogueKeys.privateKey)); + assert.throws( + () => runtime.loadQwenRuntimeConfig(runtimeEnv({ + VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest, + VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_FILE: roguePublicFile, + VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_SHA256: runtime.qwenSha256(roguePublicBytes), + VINCI_QWEN_QUALIFICATION_ISSUER: "reviewer:test", + }), nowMs, authorityBoundary), + /qualification_signature_invalid/, + "a Worker-chosen trust key must not self-admit a qualification", + ); + qualificationDigest = writeQualification(signedEnvelope(qualification)); + baseRuntimeEnv.VINCI_QWEN_QUALIFICATION_SHA256 = qualificationDigest; + const invalidSignature = structuredClone(signedEnvelope(qualification)); invalidSignature.qualification.bindings.model = "Qwen/Qwen3.8-27B-tampered"; qualificationDigest = writeQualification(invalidSignature); assert.throws( - () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest }), nowMs), + () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest }), nowMs, authorityBoundary), /qualification_signature_invalid/, ); qualificationDigest = writeQualification(signedEnvelope(qualification)); @@ -298,7 +382,7 @@ try { qualificationDigest = writeQualification(qualification); assert.throws( - () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest }), nowMs), + () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest }), nowMs, authorityBoundary), /qualification envelope.*missing fields|qualification envelope.*unexpected/, ); qualificationDigest = writeQualification(signedEnvelope(qualification)); @@ -309,7 +393,7 @@ try { }); qualificationDigest = writeQualification(signedEnvelope(expired)); assert.throws( - () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest }), nowMs), + () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest }), nowMs, authorityBoundary), /qualification_expired/, ); qualificationDigest = writeQualification(signedEnvelope(qualification)); @@ -328,8 +412,8 @@ try { }); qualificationDigest = writeQualification(signedEnvelope(concurrency32)); assert.throws( - () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest }), nowMs), - /fleet_permit_authority_missing/, + () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest }), nowMs, authorityBoundary), + /fleet_permit_invalid/, ); qualificationDigest = writeQualification(signedEnvelope(qualification)); baseRuntimeEnv.VINCI_QWEN_QUALIFICATION_SHA256 = qualificationDigest; @@ -340,7 +424,7 @@ try { }); qualificationDigest = writeQualification(signedEnvelope(skippedStage)); assert.throws( - () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest }), nowMs), + () => runtime.loadQwenRuntimeConfig(runtimeEnv({ VINCI_QWEN_QUALIFICATION_SHA256: qualificationDigest }), nowMs, authorityBoundary), /burn_in_gate_failed/, ); qualificationDigest = writeQualification(signedEnvelope(qualification)); @@ -378,7 +462,7 @@ try { `data: ${JSON.stringify({ id: "canary-usage", object: "chat.completion.chunk", created: 2, model: runtime.QWEN_MODEL, choices: [], usage })}`, "data: [DONE]", "", - ].join("\n"); + ].join("\n\n"); const canaryFetch = async (url, init = {}) => { const target = String(url); const authenticated = new Headers(init.headers).has("authorization"); @@ -414,23 +498,30 @@ try { breakerConfig.endpointAddresses = ["93.184.216.34"]; let failedCalls = 0; const retryKeys = []; + const requestDigests = []; + const fleetPermitIds = []; const failingTransport = runtime.createQwenInferenceFetch( breakerConfig, "request-500", (record) => records.push(record), async (_url, init = {}) => { failedCalls += 1; - retryKeys.push(new Headers(init.headers).get("x-vinci-idempotency-key")); + const headers = new Headers(init.headers); + retryKeys.push(headers.get("x-vinci-idempotency-key")); + requestDigests.push(headers.get("x-vinci-qwen-request-sha256")); + fleetPermitIds.push(headers.get("x-vinci-qwen-fleet-permit-id")); return new Response("failure", { status: 500 }); }, ); await assert.rejects( - failingTransport(breakerConfig.chatUrl, { method: "POST", body: "{}" }), + failingTransport(breakerConfig.chatUrl, { method: "POST", body: requestBody }), /qwen_http_status/, ); assert.equal(failedCalls, 2, "threshold two must count both real HTTP 500 responses"); assert.equal(records.length, 2); assert.deepEqual(retryKeys, ["request-500/0", "request-500/1"]); + assert.deepEqual(requestDigests, [runtime.qwenSha256(requestBody), runtime.qwenSha256(requestBody)]); + assert.deepEqual(fleetPermitIds, [authorityBoundary.fleetPermit.permitId, authorityBoundary.fleetPermit.permitId]); assert.deepEqual(records.map((record) => record.transport_attempt), [0, 1]); assert.ok(records.every((record) => record.cost_usd === 0 && record.input_tokens === 0 && record.output_tokens === 0)); assert.throws(() => runtime.assertQwenCircuitClosed(breakerConfig), /qwen_circuit_open/); @@ -450,7 +541,7 @@ try { () => {}, async () => new Response(validSse, { headers: identityHeaders({ "x-vinci-model-id": "Qwen/wrong" }) }), ); - await assert.rejects(mismatchTransport(mismatchConfig.chatUrl, { method: "POST", body: "{}" }), /response_identity_mismatch/); + await assert.rejects(mismatchTransport(mismatchConfig.chatUrl, { method: "POST", body: requestBody }), /response_identity_mismatch/); const runtimeMismatchRecords = []; const runtimeMismatchConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "runtime-mismatch.json") }); @@ -461,7 +552,7 @@ try { "request-runtime-mismatch", (record) => runtimeMismatchRecords.push(record), async () => new Response(validSse, { headers: identityHeaders({ "x-vinci-runtime-version": "wrong" }) }), - )(runtimeMismatchConfig.chatUrl, { method: "POST", body: "{}" }), + )(runtimeMismatchConfig.chatUrl, { method: "POST", body: requestBody }), /response_identity_mismatch/, ); assert.equal(runtimeMismatchRecords[0].outcome, "response_identity_mismatch"); @@ -471,14 +562,14 @@ try { await assert.rejects( runtime.createQwenInferenceFetch(redirectConfig, "request-redirect", () => {}, async () => new Response("", { status: 302 }))( redirectConfig.chatUrl, - { method: "POST", body: "{}" }, + { method: "POST", body: requestBody }, ), /redirect_forbidden/, ); await assert.rejects( runtime.createQwenInferenceFetch(redirectConfig, "request-ssrf", () => {}, async () => new Response(validSse, { headers: identityHeaders() }))( "http://169.254.169.254/latest/meta-data", - { method: "POST", body: "{}" }, + { method: "POST", body: requestBody }, ), /ssrf_forbidden/, ); @@ -489,6 +580,15 @@ try { ), /request_oversized/, ); + const mutatedBody = JSON.stringify({ ...JSON.parse(requestBody), messages: [{ role: "system", content: systemPrompt }, { role: "user", content: "post-hook mutation" }] }); + await assert.rejects( + runtime.createQwenInferenceFetch(redirectConfig, "request-mutated", () => {}, async () => new Response(validSse, { headers: identityHeaders() }))( + redirectConfig.chatUrl, + { method: "POST", body: mutatedBody }, + ), + /prompt_mismatch/, + "the transport must reject bytes changed after payload hooks", + ); const cancelledConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "cancelled-retry.json") }); cancelledConfig.endpointAddresses = ["93.184.216.34"]; @@ -500,7 +600,7 @@ try { cancelledCalls += 1; queueMicrotask(() => retryAbort.abort("operator_stop")); return new Response("retry", { status: 429, headers: { "retry-after": "1" } }); - })(cancelledConfig.chatUrl, { method: "POST", body: "{}", signal: retryAbort.signal }), + })(cancelledConfig.chatUrl, { method: "POST", body: requestBody, signal: retryAbort.signal }), /cancelled/, ); assert.equal(cancelledCalls, 1, "cancellation during Retry-After must prevent the next transport attempt"); @@ -510,7 +610,7 @@ try { await assert.rejects( runtime.createQwenInferenceFetch(oversizedConfig, "request-oversized", () => {}, async () => new Response("x".repeat(300), { status: 500 }))( oversizedConfig.chatUrl, - { method: "POST", body: "{}" }, + { method: "POST", body: requestBody }, ), /response_oversized/, ); @@ -518,13 +618,18 @@ try { const successRecords = []; const successConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "success.json") }); successConfig.endpointAddresses = ["93.184.216.34"]; + const successSettlement = { transportFailed: false, settled: false }; + const recordSuccess = (record) => successRecords.push(record); const successResponse = await runtime.createQwenInferenceFetch( successConfig, "request-success", - (record) => successRecords.push(record), + recordSuccess, async () => new Response(validSse, { headers: identityHeaders() }), - )(successConfig.chatUrl, { method: "POST", body: "{}" }); + successSettlement, + )(successConfig.chatUrl, { method: "POST", body: requestBody }); assert.equal(await successResponse.text(), validSse); + assert.equal(successRecords.length, 0, "raw SSE completion must not be recorded as success before parser acceptance"); + runtime.settleQwenSemanticOutcome(successConfig, successSettlement, true, "success", recordSuccess); assert.equal(successRecords[0].outcome, "success"); assert.equal(successRecords[0].input_tokens, 10); assert.equal(successRecords[0].output_tokens, 2); @@ -535,7 +640,7 @@ try { `data: ${JSON.stringify({ id: "chunk-bad-usage", object: "chat.completion.chunk", created: 1, model: runtime.QWEN_MODEL, choices: [], usage: invalidUsage })}`, "data: [DONE]", "", - ].join("\n"); + ].join("\n\n"); const invalidUsageConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "invalid-usage.json") }); invalidUsageConfig.endpointAddresses = ["93.184.216.34"]; const invalidUsageResponse = await runtime.createQwenInferenceFetch( @@ -543,7 +648,7 @@ try { "request-invalid-usage", () => {}, async () => new Response(invalidUsageSse, { headers: identityHeaders() }), - )(invalidUsageConfig.chatUrl, { method: "POST", body: "{}" }); + )(invalidUsageConfig.chatUrl, { method: "POST", body: requestBody }); await assert.rejects(invalidUsageResponse.text(), /usage_invalid/); const oversizedSuccessConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "oversized-success.json") }); @@ -553,7 +658,7 @@ try { "request-oversized-success", () => {}, async () => new Response(`data: ${"x".repeat(5_000)}\n`, { headers: identityHeaders() }), - )(oversizedSuccessConfig.chatUrl, { method: "POST", body: "{}" }); + )(oversizedSuccessConfig.chatUrl, { method: "POST", body: requestBody }); await assert.rejects(oversizedSuccessResponse.text(), /response_oversized/); const timeoutConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "timeout.json") }); @@ -567,41 +672,41 @@ try { init.signal.addEventListener("abort", () => controller.error(new DOMException("aborted", "AbortError")), { once: true }); }, }), { headers: identityHeaders() }), - )(timeoutConfig.chatUrl, { method: "POST", body: "{}" }); + )(timeoutConfig.chatUrl, { method: "POST", body: requestBody }); await assert.rejects(timeoutResponse.text(), /AbortError|aborted/); const context = { systemPrompt, messages: [{ role: "user", content: workOrderPrompt, timestamp: Date.now() }], tools }; - const terminalMessage = { - role: "assistant", - content: [], - api: "openai-completions", - provider: runtime.QWEN_PROVIDER, - model: runtime.QWEN_MODEL, - usage: { - input: 10, - output: 2, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 12, - cost: { input: 0.0000025, output: 0.0000015, cacheRead: 0, cacheWrite: 0, total: 0.000004 }, - }, - stopReason: "stop", - timestamp: Date.now(), - }; - const fakeStream = (model) => { - const stream = createAssistantMessageEventStream(); - queueMicrotask(() => { - stream.push({ type: "done", reason: "stop", message: { ...terminalMessage, api: model.api } }); - stream.end({ ...terminalMessage, api: model.api }); - }); - return stream; + const parserValidSse = [ + `data: ${JSON.stringify({ id: "parsed-1", object: "chat.completion.chunk", created: 1, model: runtime.QWEN_MODEL, choices: [{ index: 0, delta: { content: "ok" }, finish_reason: null }] })}`, + `data: ${JSON.stringify({ id: "parsed-2", object: "chat.completion.chunk", created: 2, model: runtime.QWEN_MODEL, choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })}`, + `data: ${JSON.stringify({ id: "parsed-usage", object: "chat.completion.chunk", created: 3, model: runtime.QWEN_MODEL, choices: [], usage })}`, + "data: [DONE]", + "", + ].join("\n\n"); + let parserTransportCalls = 0; + const parserTransport = async () => { + parserTransportCalls += 1; + return new Response(parserValidSse, { headers: identityHeaders() }); }; - const permitConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "permit.json") }); + const permitConfig = freshenPermit(loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "permit.json") })); permitConfig.endpointAddresses = ["93.184.216.34"]; - const provider = qwenProviderConfig(permitConfig, fakeStream); - const model = { ...provider.models[0], provider: runtime.QWEN_PROVIDER, api: runtime.QWEN_API }; + assert.throws( + () => runtime.acquireQwenFleetPermit({ ...permitConfig, fleetPermit: { ...permitConfig.fleetPermit, expiresAt: new Date(Date.now() - 1).toISOString() } }), + /fleet_permit_expired/, + ); + await exerciseCrossProcessFleetPermit(permitConfig); + const semanticRecords = []; + const provider = qwenProviderConfig(permitConfig, sourceStreamSimpleOpenAICompletions, (record) => semanticRecords.push(record), parserTransport); + const model = { ...provider.models[0], provider: runtime.QWEN_PROVIDER, api: runtime.QWEN_API, baseUrl: provider.baseUrl }; const firstPermitStream = provider.streamSimple(model, context); + const competingProvider = qwenProviderConfig(permitConfig, sourceStreamSimpleOpenAICompletions, () => {}, parserTransport); + assert.throws( + () => competingProvider.streamSimple(model, context), + /concurrency_exceeded/, + "two provider instances sharing an external permit must not run concurrently", + ); let secondPermitStream; + let firstPermitError; for await (const event of firstPermitStream) { if (event.type === "done") { assert.doesNotThrow( @@ -609,18 +714,54 @@ try { "permit must be released before the terminal result event becomes observable", ); } + if (event.type === "error") firstPermitError = event.error.errorMessage; } - assert.ok(secondPermitStream); + assert.ok(secondPermitStream, `${firstPermitError}; transport calls=${parserTransportCalls}`); await secondPermitStream.result(); + assert.deepEqual(semanticRecords.map((record) => record.outcome), ["success", "success"]); - const truncatedProvider = qwenProviderConfig(permitConfig, () => { - const stream = createAssistantMessageEventStream(); - queueMicrotask(() => stream.end(terminalMessage)); - return stream; - }); - const truncated = await truncatedProvider.streamSimple(model, context).result(); + const missingFinishSse = [ + `data: ${JSON.stringify({ id: "missing-finish", object: "chat.completion.chunk", created: 1, model: runtime.QWEN_MODEL, choices: [{ index: 0, delta: { content: "partial" }, finish_reason: null }] })}`, + `data: ${JSON.stringify({ id: "missing-finish-usage", object: "chat.completion.chunk", created: 2, model: runtime.QWEN_MODEL, choices: [], usage })}`, + "data: [DONE]", + "", + ].join("\n\n"); + const truncatedRecords = []; + const truncatedConfig = freshenPermit(loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "truncated.json"), VINCI_QWEN_CIRCUIT_THRESHOLD: "1" })); + truncatedConfig.endpointAddresses = ["93.184.216.34"]; + const truncatedProvider = qwenProviderConfig( + truncatedConfig, + sourceStreamSimpleOpenAICompletions, + (record) => truncatedRecords.push(record), + async () => new Response(missingFinishSse, { headers: identityHeaders() }), + ); + const truncated = await truncatedProvider.streamSimple({ ...truncatedProvider.models[0], provider: runtime.QWEN_PROVIDER, api: runtime.QWEN_API, baseUrl: truncatedProvider.baseUrl }, context).result(); assert.equal(truncated.stopReason, "error"); - assert.match(truncated.errorMessage, /qwen_stream_truncated/); + assert.match(truncated.errorMessage, /finish_reason/); + assert.equal(truncatedRecords[0].outcome, "parser_error"); + assert.throws(() => runtime.assertQwenCircuitClosed(truncatedConfig), /qwen_circuit_open/); + + const unknownToolSse = [ + `data: ${JSON.stringify({ id: "bad-tool-1", object: "chat.completion.chunk", created: 1, model: runtime.QWEN_MODEL, choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: "call-1", type: "function", function: { name: "unqualified_tool", arguments: "{}" } }] }, finish_reason: null }] })}`, + `data: ${JSON.stringify({ id: "bad-tool-2", object: "chat.completion.chunk", created: 2, model: runtime.QWEN_MODEL, choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }] })}`, + `data: ${JSON.stringify({ id: "bad-tool-usage", object: "chat.completion.chunk", created: 3, model: runtime.QWEN_MODEL, choices: [], usage })}`, + "data: [DONE]", + "", + ].join("\n\n"); + const toolRecords = []; + const toolConfig = freshenPermit(loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "tool-semantic.json"), VINCI_QWEN_CIRCUIT_THRESHOLD: "1" })); + toolConfig.endpointAddresses = ["93.184.216.34"]; + const toolProvider = qwenProviderConfig( + toolConfig, + sourceStreamSimpleOpenAICompletions, + (record) => toolRecords.push(record), + async () => new Response(unknownToolSse, { headers: identityHeaders() }), + ); + const badTool = await toolProvider.streamSimple({ ...toolProvider.models[0], provider: runtime.QWEN_PROVIDER, api: runtime.QWEN_API, baseUrl: toolProvider.baseUrl }, context).result(); + assert.equal(badTool.stopReason, "error"); + assert.match(badTool.errorMessage, /qwen_semantic_invalid/); + assert.equal(toolRecords[0].outcome, "parser_semantic_invalid"); + assert.throws(() => runtime.assertQwenCircuitClosed(toolConfig), /qwen_circuit_open/); const vectors = join(root, "vinci/test/fixtures/contract-vectors"); const emptyCriteriaOrder = { @@ -639,7 +780,7 @@ let stdin = ""; process.stdin.setEncoding("utf8"); for await (const chunk of process.stdin) stdin += chunk; const secret = readFileSync(3); -writeFileSync(process.env.QWEN_TEST_SPAWN_RECORD, JSON.stringify({ argv: process.argv.slice(2), stdin, qwenEnvKeys: Object.keys(process.env).filter((key) => key.includes("QWEN_SECRET")), secretBytes: fstatSync(3).size, secretReadBytes: secret.length })); +writeFileSync(process.env.QWEN_TEST_SPAWN_RECORD, JSON.stringify({ argv: process.argv.slice(2), stdin, qwenEnvKeys: Object.keys(process.env).filter((key) => key.includes("QWEN_SECRET")), clientBuild: process.env.VINCI_QWEN_CLIENT_BUILD_SHA256, secretBytes: fstatSync(3).size, secretReadBytes: secret.length })); `, { mode: 0o700 }); chmodSync(fakeVinci, 0o700); const stateDir = join(temp, "run-state"); @@ -679,6 +820,8 @@ writeFileSync(process.env.QWEN_TEST_SPAWN_RECORD, JSON.stringify({ argv: process assert.equal(spawned.stdin, "synthetic prompt must not be argv"); assert.equal(spawned.stdin.includes("synthetic-test-secret"), false); assert.deepEqual(spawned.qwenEnvKeys, ["VINCI_QWEN_SECRET_FD"]); + assert.match(spawned.clientBuild, /^[0-9a-f]{64}$/); + assert.notEqual(spawned.clientBuild, runtime.qwenSha256(readFileSync(fakeVinci)), "client build must include executed parser and coding-agent dependencies, not only the launcher"); assert.equal(spawned.secretBytes, spawned.secretReadBytes); const lifecycle = new TaskLifecycle(join(temp, "attempt-state"), "task-attempt"); diff --git a/vinci/worker/README.md b/vinci/worker/README.md index f11b479bc..4245dc542 100644 --- a/vinci/worker/README.md +++ b/vinci/worker/README.md @@ -323,10 +323,12 @@ The lane is fail-closed. Runtime registration requires all of the following: is removed before spawn, the child consumes and closes the descriptor during provider bootstrap, and neither the secret nor its reference is put in general child environment, argv, logs, or a generated file. The WorkOrder prompt is sent on stdin, never argv. -- `VINCI_QWEN_QUALIFICATION_FILE` plus its exact `VINCI_QWEN_QUALIFICATION_SHA256`, and an - independently controlled Ed25519 trust key plus `VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_SHA256` and - `VINCI_QWEN_QUALIFICATION_ISSUER`. Qualification bytes and key bytes must be non-writable regular - files. An unsigned or self-admitted record is invalid. +- `VINCI_QWEN_QUALIFICATION_FILE` plus its exact `VINCI_QWEN_QUALIFICATION_SHA256`. Trust never + comes from Worker environment: a root-owned, non-writable record under + `/run/vinci/qwen-authority` pins the independent issuer, Ed25519 key path and digest, and a live + VGC fleet permit for the exact WorkOrder/Run/Attempt. The pinned key is also root-owned beneath + that authority root. An unsigned record, a Worker-selected key, or a self-admitted record is + invalid. - a deterministic Governor lease and worker-derived WorkOrder, Run, and Attempt identities. Model output supplies none of them. Each Worker attempt has its own session; every bounded transport retry is recorded beneath that Attempt with a distinct idempotency key. @@ -344,21 +346,27 @@ Authenticated `/health` and `/v1/models` must succeed while anonymous requests t The models response and every successful inference response must repeat the exact model, revision, endpoint identity, and runtime tuple. The chat transport calls only the qualified URL, refuses redirects, keeps one total deadline across headers/body/retries, bounds error and successful SSE -bodies, requires strict OpenAI chunk object/model/usage identities and `[DONE]`, and disables SDK -retries. Real HTTP 500s and transport/protocol failures count toward an atomically persisted circuit; -three failures open it for 60 seconds by default. `VINCI_QWEN_CIRCUIT_THRESHOLD` and +bodies, validates and hashes the exact serialized request bytes after SDK hooks, requires strict +OpenAI chunk object/model/usage identities and `[DONE]`, and disables SDK retries. A raw complete +SSE body is only transport acceptance: success is recorded only after the OpenAI parser produces a +valid terminal finish reason, usage, and qualified tool semantics. Parser and semantic failures +count toward the atomically persisted circuit just like HTTP and transport/protocol failures; three +failures open it for 60 seconds by default. `VINCI_QWEN_CIRCUIT_THRESHOLD` and `VINCI_QWEN_CIRCUIT_OPEN_MS` are bounded operator overrides. -Concurrency defaults to 1. The signed schema understands only the ladder +Concurrency is fixed at 1. The external VGC record carries a five-minute-or-shorter permit for the +exact WorkOrder/Run/Attempt and names the shared authority lock directory. Every provider instance +and process must atomically hold the same fleet lock before it can start inference; the local +closure counter is defense in depth. A crash leaves the lock closed until the external authority +reclaims it, rather than permitting overlapping work. The signed schema understands only the ladder `1 → 2 → 4 → 8 → 16 → 24 → 32`, never an intermediate value, and never above Ayush's advertised ceiling. Any stage above 1 must cite the immediately prior stage with at least 168 continuous hours and 1,000 WorkOrders, 100% acceptance pass and usage coverage, at most 0.5% transport errors, and zero identity failures, verification failures, circuit opens, resource alarms, or Governor stops. -Promotion is a new independent review and signature; it is never automatic. Today the runtime has -only its single-process permit, so every signed value above 1 still fails closed with -`fleet_permit_authority_missing`. A future fleet authority must issue bounded, expiring, fenced -permits keyed by WorkOrder/Run/Attempt, enforce the signed and advertised ceilings atomically across -workers, and define a bounded queue before any stage above 1 can run. +Promotion is a new independent review and signature; it is never automatic. Today VGC issues only +concurrency-1 permits, so every signed value above 1 fails closed. A future permit schema must retain +bounded, expiring, fenced admission keyed by WorkOrder/Run/Attempt, enforce signed and advertised +ceilings atomically across workers, and define a bounded queue before any stage above 1 can run. Qwen output gets a `vinci-qwen-output-label` session record marking it non-authoritative and requiring independent checking. It is never permission, a Governor ruling, merge authorization, @@ -411,10 +419,13 @@ node --experimental-strip-types vinci/extensions/lib/qwen-runtime.ts --qualifica ``` The independent reviewer verifies the evidence, adds issuer/timestamps/review provenance, and signs -the canonical qualification with the separately controlled key. The Qwen builder/operator must not -possess that signing key. Requalification is mandatory after a failed/expired canary or any change -to capabilities/limits, client build, endpoint identity/address policy, model/revision, outbound -encoding, pricing basis, runtime artifact/arguments, system prompt, or tool schema/policy. +the canonical qualification with the separately controlled key. VGC installs the trust pin and live +permit in the root-owned authority boundary; the Worker cannot nominate them through environment. +The Qwen builder/operator must not possess the signing key. The client-build digest covers the +executed launcher, Node version, AI parser, OpenAI SDK, and coding-agent distribution. +Requalification is mandatory after a failed/expired canary or any change to capabilities/limits, +client build, endpoint identity/address policy, model/revision, outbound encoding, pricing basis, +runtime artifact/arguments, system prompt, or tool schema/policy. Per-attempt telemetry remains in the existing economics summary: `work_order_id`, `session_id` (Run), `attempt_label`, `started_at`/`finished_at` (wall latency), terminal `local_result`, and the diff --git a/vinci/worker/cleanroom.mjs b/vinci/worker/cleanroom.mjs index 68a1b2124..5476a5c3e 100644 --- a/vinci/worker/cleanroom.mjs +++ b/vinci/worker/cleanroom.mjs @@ -110,9 +110,6 @@ export const PROVIDER_KEY_ENV = Object.freeze({ "VINCI_QWEN_SECRET_REF", "VINCI_QWEN_QUALIFICATION_FILE", "VINCI_QWEN_QUALIFICATION_SHA256", - "VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_FILE", - "VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_SHA256", - "VINCI_QWEN_QUALIFICATION_ISSUER", "VINCI_QWEN_CIRCUIT_THRESHOLD", "VINCI_QWEN_CIRCUIT_OPEN_MS", ], @@ -129,9 +126,6 @@ export const PROVIDER_CREDENTIAL_ENV = Object.freeze([ "VINCI_QWEN_SECRET_REF", "VINCI_QWEN_QUALIFICATION_FILE", "VINCI_QWEN_QUALIFICATION_SHA256", - "VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_FILE", - "VINCI_QWEN_QUALIFICATION_PUBLIC_KEY_SHA256", - "VINCI_QWEN_QUALIFICATION_ISSUER", "VINCI_QWEN_CIRCUIT_THRESHOLD", "VINCI_QWEN_CIRCUIT_OPEN_MS", "AI_GATEWAY_API_KEY", diff --git a/vinci/worker/run.mjs b/vinci/worker/run.mjs index 42bd2289f..2469b4443 100644 --- a/vinci/worker/run.mjs +++ b/vinci/worker/run.mjs @@ -11,6 +11,8 @@ import { openSync, readFileSync, readdirSync, + readlinkSync, + realpathSync, renameSync, rmSync, unlinkSync, @@ -29,6 +31,7 @@ import { readSessionState } from "./session-read.mjs"; const REPO = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; const utf8Decoder = new TextDecoder("utf-8", { fatal: true }); const WORKER_DIR = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(WORKER_DIR, "../.."); const MAX_QWEN_PROMPT_BYTES = 1024 * 1024; function sha256(value) { @@ -66,6 +69,33 @@ function qwenExtensionBuildSha256() { return sha256(Buffer.concat([Buffer.from("vinci-qwen-provider.ts\0"), provider, Buffer.from("\0qwen-runtime.ts\0"), runtime])); } +function qwenClientBuildSha256() { + const hash = createHash("sha256"); + const add = (path, label) => { + const stat = lstatSync(path); + if (stat.isSymbolicLink()) { + hash.update(`link\0${label}\0${readlinkSync(path)}\0`); + add(realpathSync(path), `${label}@resolved`); + return; + } + if (stat.isDirectory()) { + hash.update(`dir\0${label}\0`); + for (const name of readdirSync(path).sort()) add(join(path, name), `${label}/${name}`); + return; + } + if (!stat.isFile()) throw new Error(`unsupported client build entry: ${label}`); + hash.update(`file\0${label}\0${stat.size}\0`); + hash.update(readFileSync(path)); + hash.update("\0"); + }; + hash.update(`node\0${process.versions.node}\0`); + add(resolveBin("vinci"), "vinci-launcher"); + add(join(REPO_ROOT, "packages", "ai", "dist"), "packages/ai/dist"); + add(join(REPO_ROOT, "packages", "coding-agent", "dist"), "packages/coding-agent/dist"); + add(join(REPO_ROOT, "node_modules", "openai"), "node_modules/openai"); + return hash.digest("hex"); +} + function canonicalBytes(value) { return Buffer.from(`${canonicalize(value)}\n`, "utf8"); } @@ -1287,7 +1317,7 @@ export function runVinci({ envelope, repoDir, stateDir, taskId, sessionId, env, authority: "Governor", safe_resume: false, })); - taskEnvironment.VINCI_QWEN_CLIENT_BUILD_SHA256 = sha256(readFileSync(resolveBin("vinci"))); + taskEnvironment.VINCI_QWEN_CLIENT_BUILD_SHA256 = qwenClientBuildSha256(); taskEnvironment.VINCI_QWEN_EXTENSION_BUILD_SHA256 = qwenExtensionBuildSha256(); taskEnvironment.VINCI_QWEN_CIRCUIT_FILE = join(stateDir, "qwen-h200", "circuit.json"); qwenSecretReference = taskEnvironment.VINCI_QWEN_SECRET_REF; From 0d056f8c94f9a094daabe710aeb01fd0a5893e77 Mon Sep 17 00:00:00 2001 From: George Pu <19347973+thegeorgepu@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:30:27 -0400 Subject: [PATCH 4/9] fix(worker): validate qwen response semantics --- vinci/extensions/lib/qwen-runtime.ts | 27 +++++---- vinci/extensions/vinci-qwen-provider.ts | 17 +++++- vinci/test/worker-qwen-provider.mjs | 76 ++++++++++++++++++++++--- vinci/worker/README.md | 8 ++- vinci/worker/run.mjs | 20 ++++++- 5 files changed, 123 insertions(+), 25 deletions(-) diff --git a/vinci/extensions/lib/qwen-runtime.ts b/vinci/extensions/lib/qwen-runtime.ts index f927f1285..1d8b72383 100644 --- a/vinci/extensions/lib/qwen-runtime.ts +++ b/vinci/extensions/lib/qwen-runtime.ts @@ -190,6 +190,7 @@ export type QwenAttemptRecord = { input_tokens: number; output_tokens: number; request_sha256: string; + response_id: string | null; }; export type QwenAuthorityBoundary = { @@ -1173,7 +1174,7 @@ function validateUsage(value: unknown): { input: number; output: number } { return { input: prompt, output: completion }; } -function validateSseData(data: string): { done: boolean; usage?: { input: number; output: number } } { +function validateSseData(data: string): { done: boolean; responseId?: string; usage?: { input: number; output: number } } { if (data === "[DONE]") return { done: true }; let chunk: unknown; try { @@ -1185,13 +1186,13 @@ function validateSseData(data: string): { done: boolean; usage?: { input: number const record = chunk as Record; const allowed = new Set(["id", "object", "created", "model", "choices", "usage", "system_fingerprint", "service_tier"]); if (Object.keys(record).some((key) => !allowed.has(key))) fail("stream_invalid", "inference stream chunk has an unexpected field"); - if (record.object !== "chat.completion.chunk" || record.model !== QWEN_MODEL || !Array.isArray(record.choices)) { + if (typeof record.id !== "string" || !record.id || record.object !== "chat.completion.chunk" || record.model !== QWEN_MODEL || !Array.isArray(record.choices)) { fail("response_identity_mismatch", "inference stream chunk does not identify the exact qualified model/object"); } if (record.usage !== undefined && record.usage !== null) { - return { done: false, usage: validateUsage(record.usage) }; + return { done: false, responseId: record.id, usage: validateUsage(record.usage) }; } - return { done: false }; + return { done: false, responseId: record.id }; } function retryDelayMs(response: Response, maximum: number): number { @@ -1234,7 +1235,7 @@ function inferenceBody( response: Response, config: QwenRuntimeConfig, abort: AbortController, - finish: (outcome: string, status: number | null, inputTokens?: number, outputTokens?: number) => void, + finish: (outcome: string, status: number | null, inputTokens?: number, outputTokens?: number, responseId?: string | null) => void, ): ReadableStream { if (!response.body) fail("stream_invalid", "successful inference response has no body"); const reader = response.body.getReader(); @@ -1245,11 +1246,12 @@ function inferenceBody( let usageSeen = false; let inputTokens = 0; let outputTokens = 0; + let responseId: string | null = null; let settled = false; const settle = (outcome: string, status: number | null) => { if (settled) return; settled = true; - finish(outcome, status); + finish(outcome, status, 0, 0, responseId); }; const inspect = (text: string, final: boolean) => { pending += text; @@ -1261,6 +1263,10 @@ function inferenceBody( if (!line.startsWith("data:")) fail("stream_invalid", "inference stream contains a non-SSE field"); if (doneSeen) fail("stream_invalid", "inference stream contains data after [DONE]"); const result = validateSseData(line.slice(5).trim()); + if (result.responseId && responseId && result.responseId !== responseId) { + fail("response_identity_mismatch", "inference stream changed response id between chunks"); + } + responseId ??= result.responseId ?? null; if (result.usage && usageSeen) fail("usage_invalid", "inference stream contains more than one usage object"); if (result.done && !usageSeen) fail("stream_invalid", "inference stream ended before its strict usage object"); doneSeen = result.done; @@ -1281,7 +1287,7 @@ function inferenceBody( if (!doneSeen || !usageSeen) fail("stream_invalid", "inference stream omitted [DONE] or its strict usage object"); if (!settled) { settled = true; - finish("transport_accepted", response.status, inputTokens, outputTokens); + finish("transport_accepted", response.status, inputTokens, outputTokens, responseId); } controller.close(); return; @@ -1359,7 +1365,7 @@ export function createQwenInferenceFetch( headers.set("x-vinci-idempotency-key", `${requestId}/${transportAttempt}`); headers.set("x-vinci-qwen-request-sha256", requestSha256); let attemptReported = false; - const finishAttempt = (outcome: string, status: number | null, inputTokens = 0, outputTokens = 0) => { + const finishAttempt = (outcome: string, status: number | null, inputTokens = 0, outputTokens = 0, responseId: string | null = null) => { if (attemptReported) return; attemptReported = true; real?.close(); @@ -1378,6 +1384,7 @@ export function createQwenInferenceFetch( input_tokens: inputTokens, output_tokens: outputTokens, request_sha256: requestSha256, + response_id: responseId, }; if (outcome === "transport_accepted") { semanticSettlement.accepted = record; @@ -1436,10 +1443,10 @@ export function createQwenInferenceFetch( finishAttempt("content_type_invalid", response.status); fail("stream_invalid", "inference response is not text/event-stream"); } - const finish = (outcome: string, status: number | null, inputTokens = 0, outputTokens = 0) => { + const finish = (outcome: string, status: number | null, inputTokens = 0, outputTokens = 0, responseId: string | null = null) => { clearTimeout(timeout); externalSignal?.removeEventListener("abort", abort); - finishAttempt(outcome, status, inputTokens, outputTokens); + finishAttempt(outcome, status, inputTokens, outputTokens, responseId); }; const body = inferenceBody(response, config, controller, finish); timerOwnedByBody = true; diff --git a/vinci/extensions/vinci-qwen-provider.ts b/vinci/extensions/vinci-qwen-provider.ts index 5a250d4a5..c553cdcb1 100644 --- a/vinci/extensions/vinci-qwen-provider.ts +++ b/vinci/extensions/vinci-qwen-provider.ts @@ -7,6 +7,7 @@ import { type SimpleStreamOptions, } from "@earendil-works/pi-ai"; import { streamSimpleOpenAICompletions } from "@earendil-works/pi-ai/compat"; +import { Value } from "typebox/value"; import { acquireQwenFleetPermit, assertQwenCircuitClosed, @@ -52,9 +53,14 @@ function validUsage(message: { usage?: unknown }): boolean { return Math.abs((cost.total as number) - costParts) <= Number.EPSILON * Math.max(1, costParts) * 8; } -function validSemanticMessage(message: { content?: unknown; stopReason?: unknown }, context: Context): boolean { +function validSemanticMessage( + message: { content?: unknown; responseId?: unknown; stopReason?: unknown }, + context: Context, + settlement: QwenSemanticSettlement, +): boolean { if (!Array.isArray(message.content) || !["stop", "length", "toolUse"].includes(String(message.stopReason))) return false; - const qualifiedTools = new Set((context.tools ?? []).map((tool) => tool.name)); + if (!settlement.accepted?.response_id || message.responseId !== settlement.accepted.response_id) return false; + const qualifiedTools = new Map((context.tools ?? []).map((tool) => [tool.name, tool])); let toolCalls = 0; for (const block of message.content) { if (!block || typeof block !== "object") return false; @@ -73,6 +79,11 @@ function validSemanticMessage(message: { content?: unknown; stopReason?: unknown typeof value.name !== "string" || !qualifiedTools.has(value.name) || !value.arguments || typeof value.arguments !== "object" || Array.isArray(value.arguments) ) return false; + try { + if (!Value.Check(qualifiedTools.get(value.name)!.parameters, value.arguments)) return false; + } catch { + return false; + } toolCalls += 1; } return (message.stopReason === "toolUse") === (toolCalls > 0); @@ -151,7 +162,7 @@ export function qwenProviderConfig( message.provider !== QWEN_PROVIDER || message.model !== QWEN_MODEL || !validUsage(message) || - !validSemanticMessage(message, context) + !validSemanticMessage(message, context, semanticSettlement) ) { throw new Error("qwen_semantic_invalid: terminal response failed exact identity, usage, finish, or tool semantics"); } diff --git a/vinci/test/worker-qwen-provider.mjs b/vinci/test/worker-qwen-provider.mjs index 22575b8e3..878d7cc01 100644 --- a/vinci/test/worker-qwen-provider.mjs +++ b/vinci/test/worker-qwen-provider.mjs @@ -11,7 +11,7 @@ import { qwenProviderConfig } from "../extensions/vinci-qwen-provider.ts"; import * as cleanroom from "../worker/cleanroom.mjs"; import * as digest from "../worker/contracts/digest.mjs"; import * as economics from "../worker/economics.mjs"; -import { runVinci } from "../worker/run.mjs"; +import { qwenClientBuildSha256, runVinci } from "../worker/run.mjs"; import { TaskLifecycle } from "../worker/task.mjs"; const root = resolve(import.meta.dirname, "../.."); @@ -459,7 +459,7 @@ try { model: runtime.QWEN_MODEL, choices: [{ index: 0, delta: { tool_calls: [{ index: 0, function: { name: "report_ready", arguments: '{"status":"ready"}' } }] } }], })}`, - `data: ${JSON.stringify({ id: "canary-usage", object: "chat.completion.chunk", created: 2, model: runtime.QWEN_MODEL, choices: [], usage })}`, + `data: ${JSON.stringify({ id: "canary-tool", object: "chat.completion.chunk", created: 2, model: runtime.QWEN_MODEL, choices: [], usage })}`, "data: [DONE]", "", ].join("\n\n"); @@ -634,6 +634,26 @@ try { assert.equal(successRecords[0].input_tokens, 10); assert.equal(successRecords[0].output_tokens, 2); assert.equal(successRecords[0].cost_usd, 0.000004); + assert.equal(successRecords[0].response_id, "chunk-1"); + + const conflictingIdSse = [ + `data: ${JSON.stringify({ id: "response-a", object: "chat.completion.chunk", created: 1, model: runtime.QWEN_MODEL, choices: [{ index: 0, delta: { content: "partial" }, finish_reason: null }] })}`, + `data: ${JSON.stringify({ id: "response-b", object: "chat.completion.chunk", created: 2, model: runtime.QWEN_MODEL, choices: [], usage })}`, + "data: [DONE]", + "", + ].join("\n\n"); + const conflictingIdRecords = []; + const conflictingIdConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "conflicting-id.json"), VINCI_QWEN_CIRCUIT_THRESHOLD: "1" }); + conflictingIdConfig.endpointAddresses = ["93.184.216.34"]; + const conflictingIdResponse = await runtime.createQwenInferenceFetch( + conflictingIdConfig, + "request-conflicting-id", + (record) => conflictingIdRecords.push(record), + async () => new Response(conflictingIdSse, { headers: identityHeaders() }), + )(conflictingIdConfig.chatUrl, { method: "POST", body: requestBody }); + await assert.rejects(conflictingIdResponse.text(), /response id/); + assert.equal(conflictingIdRecords[0].outcome, "response_identity_mismatch"); + assert.throws(() => runtime.assertQwenCircuitClosed(conflictingIdConfig), /qwen_circuit_open/); const invalidUsage = { ...usage, total_tokens: 99 }; const invalidUsageSse = [ @@ -678,8 +698,8 @@ try { const context = { systemPrompt, messages: [{ role: "user", content: workOrderPrompt, timestamp: Date.now() }], tools }; const parserValidSse = [ `data: ${JSON.stringify({ id: "parsed-1", object: "chat.completion.chunk", created: 1, model: runtime.QWEN_MODEL, choices: [{ index: 0, delta: { content: "ok" }, finish_reason: null }] })}`, - `data: ${JSON.stringify({ id: "parsed-2", object: "chat.completion.chunk", created: 2, model: runtime.QWEN_MODEL, choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })}`, - `data: ${JSON.stringify({ id: "parsed-usage", object: "chat.completion.chunk", created: 3, model: runtime.QWEN_MODEL, choices: [], usage })}`, + `data: ${JSON.stringify({ id: "parsed-1", object: "chat.completion.chunk", created: 2, model: runtime.QWEN_MODEL, choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })}`, + `data: ${JSON.stringify({ id: "parsed-1", object: "chat.completion.chunk", created: 3, model: runtime.QWEN_MODEL, choices: [], usage })}`, "data: [DONE]", "", ].join("\n\n"); @@ -722,7 +742,7 @@ try { const missingFinishSse = [ `data: ${JSON.stringify({ id: "missing-finish", object: "chat.completion.chunk", created: 1, model: runtime.QWEN_MODEL, choices: [{ index: 0, delta: { content: "partial" }, finish_reason: null }] })}`, - `data: ${JSON.stringify({ id: "missing-finish-usage", object: "chat.completion.chunk", created: 2, model: runtime.QWEN_MODEL, choices: [], usage })}`, + `data: ${JSON.stringify({ id: "missing-finish", object: "chat.completion.chunk", created: 2, model: runtime.QWEN_MODEL, choices: [], usage })}`, "data: [DONE]", "", ].join("\n\n"); @@ -743,8 +763,8 @@ try { const unknownToolSse = [ `data: ${JSON.stringify({ id: "bad-tool-1", object: "chat.completion.chunk", created: 1, model: runtime.QWEN_MODEL, choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: "call-1", type: "function", function: { name: "unqualified_tool", arguments: "{}" } }] }, finish_reason: null }] })}`, - `data: ${JSON.stringify({ id: "bad-tool-2", object: "chat.completion.chunk", created: 2, model: runtime.QWEN_MODEL, choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }] })}`, - `data: ${JSON.stringify({ id: "bad-tool-usage", object: "chat.completion.chunk", created: 3, model: runtime.QWEN_MODEL, choices: [], usage })}`, + `data: ${JSON.stringify({ id: "bad-tool-1", object: "chat.completion.chunk", created: 2, model: runtime.QWEN_MODEL, choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }] })}`, + `data: ${JSON.stringify({ id: "bad-tool-1", object: "chat.completion.chunk", created: 3, model: runtime.QWEN_MODEL, choices: [], usage })}`, "data: [DONE]", "", ].join("\n\n"); @@ -763,6 +783,31 @@ try { assert.equal(toolRecords[0].outcome, "parser_semantic_invalid"); assert.throws(() => runtime.assertQwenCircuitClosed(toolConfig), /qwen_circuit_open/); + const invalidArgumentsSse = [ + `data: ${JSON.stringify({ id: "bad-arguments", object: "chat.completion.chunk", created: 1, model: runtime.QWEN_MODEL, choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: "call-2", type: "function", function: { name: "read", arguments: '{"path":42}' } }] }, finish_reason: null }] })}`, + `data: ${JSON.stringify({ id: "bad-arguments", object: "chat.completion.chunk", created: 2, model: runtime.QWEN_MODEL, choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }] })}`, + `data: ${JSON.stringify({ id: "bad-arguments", object: "chat.completion.chunk", created: 3, model: runtime.QWEN_MODEL, choices: [], usage })}`, + "data: [DONE]", + "", + ].join("\n\n"); + const argumentRecords = []; + const argumentConfig = freshenPermit(loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "tool-arguments.json"), VINCI_QWEN_CIRCUIT_THRESHOLD: "1" })); + argumentConfig.endpointAddresses = ["93.184.216.34"]; + const argumentProvider = qwenProviderConfig( + argumentConfig, + sourceStreamSimpleOpenAICompletions, + (record) => argumentRecords.push(record), + async () => new Response(invalidArgumentsSse, { headers: identityHeaders() }), + ); + const badArguments = await argumentProvider.streamSimple( + { ...argumentProvider.models[0], provider: runtime.QWEN_PROVIDER, api: runtime.QWEN_API, baseUrl: argumentProvider.baseUrl }, + context, + ).result(); + assert.equal(badArguments.stopReason, "error"); + assert.match(badArguments.errorMessage, /qwen_semantic_invalid/); + assert.equal(argumentRecords[0].outcome, "parser_semantic_invalid"); + assert.throws(() => runtime.assertQwenCircuitClosed(argumentConfig), /qwen_circuit_open/); + const vectors = join(root, "vinci/test/fixtures/contract-vectors"); const emptyCriteriaOrder = { ...JSON.parse(readFileSync(join(vectors, "work-order-1-minimal/input.json"), "utf8")), @@ -772,8 +817,12 @@ try { const fakeBin = join(temp, "bin"); const fakeVinci = join(fakeBin, "vinci"); + const mutatedUndici = join(temp, "mutated-undici"); const spawnRecord = join(temp, "spawn-record.json"); mkdirSync(fakeBin); + mkdirSync(mutatedUndici); + writeFileSync(join(mutatedUndici, "package.json"), JSON.stringify({ name: "undici", version: "0.0.0-mutated" }), { mode: 0o600 }); + writeFileSync(join(mutatedUndici, "index.js"), "export const mutation = true;\n", { mode: 0o600 }); writeFileSync(fakeVinci, `#!/usr/bin/env node import { fstatSync, readFileSync, writeFileSync } from "node:fs"; let stdin = ""; @@ -788,7 +837,19 @@ writeFileSync(process.env.QWEN_TEST_SPAWN_RECORD, JSON.stringify({ argv: process writeFileSync(join(stateDir, "tasks", "task-spawn.json"), JSON.stringify({ attempt: 1 }), { mode: 0o600 }); const originalPath = process.env.PATH; process.env.PATH = `${fakeBin}:${originalPath}`; + let expectedClientBuild; try { + expectedClientBuild = qwenClientBuildSha256(); + assert.notEqual( + qwenClientBuildSha256({ undiciPath: mutatedUndici }), + expectedClientBuild, + "changing the directly used undici version/content must change the client-build identity", + ); + assert.throws( + () => qwenClientBuildSha256({ undiciPath: "" }), + /undici dependency path is required/, + "omitting undici must refuse client-build identity generation", + ); await runVinci({ envelope: { provider: runtime.QWEN_PROVIDER, @@ -821,6 +882,7 @@ writeFileSync(process.env.QWEN_TEST_SPAWN_RECORD, JSON.stringify({ argv: process assert.equal(spawned.stdin.includes("synthetic-test-secret"), false); assert.deepEqual(spawned.qwenEnvKeys, ["VINCI_QWEN_SECRET_FD"]); assert.match(spawned.clientBuild, /^[0-9a-f]{64}$/); + assert.equal(spawned.clientBuild, expectedClientBuild); assert.notEqual(spawned.clientBuild, runtime.qwenSha256(readFileSync(fakeVinci)), "client build must include executed parser and coding-agent dependencies, not only the launcher"); assert.equal(spawned.secretBytes, spawned.secretReadBytes); diff --git a/vinci/worker/README.md b/vinci/worker/README.md index 4245dc542..c289fcdc2 100644 --- a/vinci/worker/README.md +++ b/vinci/worker/README.md @@ -350,8 +350,9 @@ bodies, validates and hashes the exact serialized request bytes after SDK hooks, OpenAI chunk object/model/usage identities and `[DONE]`, and disables SDK retries. A raw complete SSE body is only transport acceptance: success is recorded only after the OpenAI parser produces a valid terminal finish reason, usage, and qualified tool semantics. Parser and semantic failures -count toward the atomically persisted circuit just like HTTP and transport/protocol failures; three -failures open it for 60 seconds by default. `VINCI_QWEN_CIRCUIT_THRESHOLD` and +include conflicting streamed response IDs and tool arguments that fail the exact signed JSON +schema; they count toward the atomically persisted circuit just like HTTP and transport/protocol +failures. Three failures open it for 60 seconds by default. `VINCI_QWEN_CIRCUIT_THRESHOLD` and `VINCI_QWEN_CIRCUIT_OPEN_MS` are bounded operator overrides. Concurrency is fixed at 1. The external VGC record carries a five-minute-or-shorter permit for the @@ -422,7 +423,8 @@ The independent reviewer verifies the evidence, adds issuer/timestamps/review pr the canonical qualification with the separately controlled key. VGC installs the trust pin and live permit in the root-owned authority boundary; the Worker cannot nominate them through environment. The Qwen builder/operator must not possess the signing key. The client-build digest covers the -executed launcher, Node version, AI parser, OpenAI SDK, and coding-agent distribution. +executed launcher, Node version, AI parser, OpenAI SDK, Undici transport, TypeBox schema validator, +and coding-agent distribution, including each directly used package's version and content. Requalification is mandatory after a failed/expired canary or any change to capabilities/limits, client build, endpoint identity/address policy, model/revision, outbound encoding, pricing basis, runtime artifact/arguments, system prompt, or tool schema/policy. diff --git a/vinci/worker/run.mjs b/vinci/worker/run.mjs index 2469b4443..5ca2c1764 100644 --- a/vinci/worker/run.mjs +++ b/vinci/worker/run.mjs @@ -69,7 +69,7 @@ function qwenExtensionBuildSha256() { return sha256(Buffer.concat([Buffer.from("vinci-qwen-provider.ts\0"), provider, Buffer.from("\0qwen-runtime.ts\0"), runtime])); } -function qwenClientBuildSha256() { +export function qwenClientBuildSha256({ undiciPath = join(REPO_ROOT, "node_modules", "undici") } = {}) { const hash = createHash("sha256"); const add = (path, label) => { const stat = lstatSync(path); @@ -88,11 +88,27 @@ function qwenClientBuildSha256() { hash.update(readFileSync(path)); hash.update("\0"); }; + const addDependency = (path, label, expectedName) => { + if (!path || !isAbsolute(path)) throw new Error(`${label} dependency path is required and must be absolute`); + let manifest; + try { + manifest = JSON.parse(readFileSync(join(path, "package.json"), "utf8")); + } catch { + throw new Error(`${label} dependency manifest is unavailable`); + } + if (manifest?.name !== expectedName || typeof manifest.version !== "string" || !manifest.version) { + throw new Error(`${label} dependency identity is invalid`); + } + hash.update(`dependency\0${manifest.name}\0${manifest.version}\0`); + add(path, label); + }; hash.update(`node\0${process.versions.node}\0`); add(resolveBin("vinci"), "vinci-launcher"); add(join(REPO_ROOT, "packages", "ai", "dist"), "packages/ai/dist"); add(join(REPO_ROOT, "packages", "coding-agent", "dist"), "packages/coding-agent/dist"); - add(join(REPO_ROOT, "node_modules", "openai"), "node_modules/openai"); + addDependency(join(REPO_ROOT, "node_modules", "openai"), "node_modules/openai", "openai"); + addDependency(undiciPath, "node_modules/undici", "undici"); + addDependency(join(REPO_ROOT, "node_modules", "typebox"), "node_modules/typebox", "typebox"); return hash.digest("hex"); } From 4c306eabbe07ea302dc97281b310dd42a39547de Mon Sep 17 00:00:00 2001 From: George Pu <19347973+thegeorgepu@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:58:50 -0400 Subject: [PATCH 5/9] fix(coding-agent): make checkpoint SIGKILL recovery deterministic --- vinci/test/checkpoint-process-integration.mjs | 54 +++++++++++++++++-- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/vinci/test/checkpoint-process-integration.mjs b/vinci/test/checkpoint-process-integration.mjs index bafb2aaa9..b4eb7250b 100644 --- a/vinci/test/checkpoint-process-integration.mjs +++ b/vinci/test/checkpoint-process-integration.mjs @@ -9,11 +9,46 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); const launcher = join(root, "vinci/bin/vinci"); const providerExtension = join(root, "vinci/test/fixtures/checkpoint-faux-provider.ts"); const pauseExtension = join(root, "vinci/test/fixtures/checkpoint-pause.ts"); +// Cold loading every product extension can be filesystem-bound on a busy host. These are watchdogs, +// not substitutes for the marker that proves the write landed inside the intended crash window. +const STARTUP_TIMEOUT_MS = 60_000; +const PROCESS_TIMEOUT_MS = 75_000; +const processGroups = new Set(); + +function killProcessGroup(child) { + if (child.pid === undefined) return; + try { + process.kill(-child.pid, "SIGKILL"); + } catch (error) { + if (error?.code !== "ESRCH") throw error; + } +} + +function processGroupExists(child) { + if (child.pid === undefined) return false; + try { + process.kill(-child.pid, 0); + return true; + } catch (error) { + if (error?.code === "ESRCH") return false; + if (error?.code === "EPERM") return true; + throw error; + } +} + +async function waitForProcessGroupExit(child, timeoutMs) { + const started = Date.now(); + while (Date.now() - started < timeoutMs) { + if (!processGroupExists(child)) return; + await new Promise((resolveWait) => setTimeout(resolveWait, 10)); + } + throw new Error(`Vinci process group ${child.pid} survived SIGKILL`); +} function waitForExit(child, timeoutMs) { return new Promise((resolveExit, reject) => { const timer = setTimeout(() => { - child.kill("SIGKILL"); + killProcessGroup(child); reject(new Error(`child process did not exit within ${timeoutMs}ms`)); }, timeoutMs); child.once("error", (error) => { @@ -42,11 +77,15 @@ async function waitForFile(path, child, timeoutMs, outputPath) { function startVinci(cwd, outputPath, args, env) { const outputFd = openSync(outputPath, "a"); + // The launcher waits on a background Node child. Give both a private process group so SIGKILL + // cannot leave the runtime alive to race the resumed process against the same session file. const child = spawn("bash", [launcher, ...args], { cwd, + detached: true, env: { ...process.env, ...env }, stdio: ["ignore", outputFd, outputFd], }); + processGroups.add(child); child.once("exit", () => closeSync(outputFd)); return child; } @@ -88,14 +127,16 @@ try { ], commonEnv, ); - const firstExit = waitForExit(first, 15_000); - await waitForFile(marker, first, 10_000, firstLog); + const firstExit = waitForExit(first, PROCESS_TIMEOUT_MS); + await waitForFile(marker, first, STARTUP_TIMEOUT_MS, firstLog); const target = join(temp, "interrupted.txt"); assert.equal(readFileSync(target, "utf8"), "written once before process death\n"); const beforeResume = statSync(target, { bigint: true }).mtimeNs; - first.kill("SIGKILL"); + killProcessGroup(first); const killed = await firstExit; assert.equal(killed.signal, "SIGKILL"); + await waitForProcessGroupExit(first, 2_000); + processGroups.delete(first); const sessionFiles = readdirSync(sessions).filter((file) => file.endsWith(".jsonl")); assert.equal(sessionFiles.length, 1); @@ -137,8 +178,10 @@ try { ], { ...commonEnv, VINCI_CHECKPOINT_KILL_MARKER: "" }, ); - const resumed = await waitForExit(resume, 15_000); + const resumed = await waitForExit(resume, PROCESS_TIMEOUT_MS); assert.equal(resumed.code, 0, readFileSync(resumeLog, "utf8").slice(-4000)); + await waitForProcessGroupExit(resume, 2_000); + processGroups.delete(resume); assert.equal(readFileSync(target, "utf8"), "written once before process death\n"); assert.equal(statSync(target, { bigint: true }).mtimeNs, beforeResume, "resume must not rewrite the completed file"); @@ -162,6 +205,7 @@ try { ); assert.match(readFileSync(resumeLog, "utf8"), /Resume completed without replaying the write/); } finally { + for (const child of processGroups) killProcessGroup(child); rmSync(temp, { recursive: true, force: true }); } From 1c28e5a8aef0db961e813d8eb2798f597331041b Mon Sep 17 00:00:00 2001 From: George Pu <19347973+thegeorgepu@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:54:28 -0400 Subject: [PATCH 6/9] harden qwen invocation authority and fail closed --- vinci/extensions/lib/qwen-runtime.ts | 633 ++++++++++++++++--- vinci/extensions/vinci-qwen-provider.ts | 23 +- vinci/test/worker-qwen-provider.mjs | 401 +++++++++--- vinci/worker/README.md | 67 +- vinci/worker/economics.mjs | 29 +- vinci/worker/session-read.mjs | 21 +- vinci/worker/test/economics-session.test.mjs | 49 ++ 7 files changed, 1020 insertions(+), 203 deletions(-) diff --git a/vinci/extensions/lib/qwen-runtime.ts b/vinci/extensions/lib/qwen-runtime.ts index 1d8b72383..504d71e6f 100644 --- a/vinci/extensions/lib/qwen-runtime.ts +++ b/vinci/extensions/lib/qwen-runtime.ts @@ -5,12 +5,15 @@ import { constants, existsSync, fstatSync, + fsyncSync, lstatSync, + linkSync, mkdirSync, openSync, readFileSync, renameSync, rmdirSync, + unlinkSync, writeFileSync, } from "node:fs"; import { isIP } from "node:net"; @@ -179,6 +182,18 @@ export type QwenLookup = (hostname: string) => Promise; export type QwenFetch = (input: string | URL | Request, init?: RequestInit) => Promise; export type QwenAttemptRecord = { + invocation_id: string; + reservation_id: string; + work_order_id: string; + run_id: string; + attempt_id: string; + lease_id: string; + fencing_generation: number; + session_id: string; + worker_principal: string; + worker_build_sha256: string; + contract_digest: string; + execution_spec_digest: string; request_id: string; transport_attempt: number; started_at: string; @@ -193,6 +208,30 @@ export type QwenAttemptRecord = { response_id: string | null; }; +export type QwenPointOfUseReservationRequest = { + schema: "vinci.qwen-point-of-use-reservation-request.v1"; + invocation_id: string; + request_sha256: string; + permit_id: string; + work_order_id: string; + run_id: string; + attempt_id: string; + lease_id: string; + fencing_generation: number; + session_id: string; + worker_principal: string; + worker_build_sha256: string; + contract_digest: string; + execution_spec_digest: string; + endpoint_sha256: string; + endpoint_identity_sha256: string; + deployment_revision: string; +}; + +export type QwenReservationAuthority = ( + request: Readonly, +) => Promise> | Record; + export type QwenAuthorityBoundary = { qualificationTrust: { issuer: string; @@ -208,6 +247,14 @@ export type QwenAuthorityBoundary = { attemptId: string; maxConcurrency: number; lockDirectory: string; + reconciliationDirectory: string; + leaseId: string; + fencingGeneration: number; + sessionId: string; + workerPrincipal: string; + workerBuildSha256: string; + contractDigest: string; + executionSpecDigest: string; issuedAt: string; expiresAt: string; }; @@ -236,7 +283,15 @@ export type QwenRuntimeConfig = { fleetPermit: { permitId: string; lockDirectory: string; + reconciliationDirectory: string; expiresAt: string; + leaseId: string; + fencingGeneration: number; + sessionId: string; + workerPrincipal: string; + workerBuildSha256: string; + contractDigest: string; + executionSpecDigest: string; }; attribution: { workOrderId: string; @@ -622,7 +677,7 @@ function loadIndependentAuthority( } exactKeys( boundary.fleetPermit, - ["schema", "authority", "permitId", "workOrderId", "runId", "attemptId", "maxConcurrency", "lockDirectory", "issuedAt", "expiresAt"], + ["schema", "authority", "permitId", "workOrderId", "runId", "attemptId", "maxConcurrency", "lockDirectory", "reconciliationDirectory", "leaseId", "fencingGeneration", "sessionId", "workerPrincipal", "workerBuildSha256", "contractDigest", "executionSpecDigest", "issuedAt", "expiresAt"], "fleet permit", ); const permit = boundary.fleetPermit; @@ -634,7 +689,17 @@ function loadIndependentAuthority( fail("fleet_permit_invalid", "fleet permit attribution does not match this exact WorkOrder/Run/Attempt"); } if (permit.maxConcurrency !== 1) fail("fleet_permit_invalid", "current Qwen fleet permit must enforce concurrency 1"); - if (!isAbsolute(permit.lockDirectory)) fail("fleet_permit_invalid", "fleet permit lock directory must be absolute"); + for (const [label, value] of [["lease id", permit.leaseId], ["session id", permit.sessionId], ["worker principal", permit.workerPrincipal]] as const) { + if (!IDENTIFIER.test(value)) fail("fleet_permit_invalid", `fleet permit ${label} is invalid`); + } + if (!Number.isSafeInteger(permit.fencingGeneration) || permit.fencingGeneration < 1) fail("fleet_permit_invalid", "fleet permit fencing generation is invalid"); + for (const [label, value] of [["worker build", permit.workerBuildSha256], ["contract", permit.contractDigest], ["execution spec", permit.executionSpecDigest]] as const) { + if (!HEX64.test(value)) fail("fleet_permit_invalid", `fleet permit ${label} digest is invalid`); + } + if (permit.sessionId !== runId) fail("fleet_permit_invalid", "fleet permit session does not match the authoritative Run"); + if (!isAbsolute(permit.lockDirectory) || !isAbsolute(permit.reconciliationDirectory)) { + fail("fleet_permit_invalid", "fleet permit lock and reconciliation directories must be absolute"); + } const issued = timestamp(permit.issuedAt, "fleet permit issuedAt"); const expires = timestamp(permit.expiresAt, "fleet permit expiresAt"); if (issued.time > nowMs + 30_000 || expires.time <= nowMs || expires.time - issued.time > 5 * 60_000) { @@ -657,6 +722,18 @@ function loadIndependentAuthority( if (permit.lockDirectory !== join(QWEN_AUTHORITY_ROOT, "locks")) { fail("authority_boundary_unsafe", "fleet permit lock directory must be the independent authority lock root"); } + if (permit.reconciliationDirectory !== join(QWEN_AUTHORITY_ROOT, "reconciliations")) { + fail("authority_boundary_unsafe", "fleet reconciliation directory must be the independent authority reconciliation root"); + } + let reconciliationStat; + try { + reconciliationStat = lstatSync(permit.reconciliationDirectory); + } catch { + fail("config_unavailable", "fleet reconciliation directory is unavailable"); + } + if (!reconciliationStat.isDirectory() || reconciliationStat.isSymbolicLink() || reconciliationStat.uid !== 0 || (reconciliationStat.mode & 0o022) !== 0) { + fail("authority_boundary_unsafe", "fleet reconciliation directory must be root-owned and non-writable by the Worker"); + } let lockStat; try { lockStat = lstatSync(permit.lockDirectory); @@ -847,8 +924,8 @@ export function loadQwenRuntimeConfig( if (bindings.request_encoding_sha256 !== qwenSha256(qwenCanonical(QWEN_REQUEST_ENCODING))) { fail("request_encoding_mismatch", "outbound OpenAI request encoding is not qualified"); } - if (env.VINCI_UNATTENDED_POLICY !== "governed" || !env.VINCI_UNATTENDED_LEASE) { - fail("authority_forbidden", "Qwen Worker runs require a deterministic Governor lease"); + if (env.VINCI_UNATTENDED_POLICY !== "governed") { + fail("authority_forbidden", "Qwen Worker runs require governed policy; lease authority comes only from the external permit"); } if (qualification.limits.max_concurrency !== authority.fleetPermit.maxConcurrency) { fail("fleet_permit_invalid", "qualification concurrency differs from the external fleet permit"); @@ -867,7 +944,15 @@ export function loadQwenRuntimeConfig( fleetPermit: { permitId: authority.fleetPermit.permitId, lockDirectory: authority.fleetPermit.lockDirectory, + reconciliationDirectory: authority.fleetPermit.reconciliationDirectory, expiresAt: authority.fleetPermit.expiresAt, + leaseId: authority.fleetPermit.leaseId, + fencingGeneration: authority.fleetPermit.fencingGeneration, + sessionId: authority.fleetPermit.sessionId, + workerPrincipal: authority.fleetPermit.workerPrincipal, + workerBuildSha256: authority.fleetPermit.workerBuildSha256, + contractDigest: authority.fleetPermit.contractDigest, + executionSpecDigest: authority.fleetPermit.executionSpecDigest, }, attribution: { workOrderId, runId, attemptId }, }; @@ -984,7 +1069,20 @@ async function readBoundedText(response: Response, maximumBytes: number, abort?: } } -function authHeaders(config: Pick): Record { +function permitRequestHeaders(config: QwenRuntimeConfig): Record { + return { + "x-vinci-qwen-fleet-permit-id": config.fleetPermit.permitId, + "x-vinci-lease-id": config.fleetPermit.leaseId, + "x-vinci-fencing-generation": String(config.fleetPermit.fencingGeneration), + "x-vinci-session-id": config.fleetPermit.sessionId, + "x-vinci-worker-principal": config.fleetPermit.workerPrincipal, + "x-vinci-worker-build-sha256": config.fleetPermit.workerBuildSha256, + "x-vinci-contract-digest": config.fleetPermit.contractDigest, + "x-vinci-execution-spec-digest": config.fleetPermit.executionSpecDigest, + }; +} + +function authHeaders(config: QwenRuntimeConfig): Record { return { authorization: `Bearer ${config.secret}`, accept: "application/json", @@ -992,6 +1090,7 @@ function authHeaders(config: Pick): "x-vinci-run-id": config.attribution.runId, "x-vinci-attempt-id": config.attribution.attemptId, "x-vinci-qwen-output-authority": "non-authoritative", + ...permitRequestHeaders(config), }; } @@ -1024,8 +1123,17 @@ async function boundedRequest( url: string, init: RequestInit, maximumBytes: number, - options: { fetchImpl?: QwenFetch; signal?: AbortSignal } = {}, + options: { fetchImpl?: QwenFetch; signal?: AbortSignal; reservationAuthority?: QwenReservationAuthority } = {}, ): Promise<{ response: Response; text: string }> { + const method = (init.method ?? "GET").toUpperCase(); + const body = bodyBytes(init.body); + const requestSha256 = qwenSha256(Buffer.concat([Buffer.from(`${method}\0${url}\0`), body])); + const requestId = qwenSha256(`readiness\0${config.attribution.workOrderId}\0${url}\0${randomBytes(16).toString("hex")}`); + const invocation = prepareQwenInvocation(config, requestId, requestSha256, method, url, "readiness", 0); + const headers = new Headers(init.headers); + headers.set("x-vinci-qwen-invocation-id", invocation.record.invocation_id); + headers.set("x-vinci-qwen-request-sha256", requestSha256); + for (const [name, value] of Object.entries(permitRequestHeaders(config))) headers.set(name, value); const controller = new AbortController(); const timeout = setTimeout(() => controller.abort("total_timeout"), config.qualification.limits.total_timeout_ms); const abort = () => controller.abort(options.signal?.reason ?? "cancelled"); @@ -1033,9 +1141,20 @@ async function boundedRequest( else options.signal?.addEventListener("abort", abort, { once: true }); const real = options.fetchImpl ? null : realPinnedFetch(config, 0); try { - const response = await (options.fetchImpl ?? real!.fetchImpl)(url, { ...init, redirect: "error", signal: controller.signal }); - if (response.status >= 300 && response.status < 400) fail("redirect_forbidden", "endpoint redirects are refused"); + let reservationId: string; + try { + reservationId = await invocation.reserve(options.reservationAuthority); + } catch (error) { + invocation.abandonPrepared(); + throw error; + } + headers.set("x-vinci-qwen-reservation-id", reservationId); + invocation.dispatch(); + const response = await (options.fetchImpl ?? real!.fetchImpl)(url, { ...init, method, body: body.length > 0 ? body : undefined, headers, redirect: "error", signal: controller.signal }); + invocation.observe(response); const text = await readBoundedText(response, maximumBytes, controller); + invocation.reconcile(); + if (response.status >= 300 && response.status < 400) fail("redirect_forbidden", "endpoint redirects are refused"); return { response, text }; } catch (error) { if (error instanceof QwenReadinessError) throw error; @@ -1088,7 +1207,7 @@ async function validateHealthResponse(response: Response, text: string): Promise export async function probeQwenReadiness( config: QwenRuntimeConfig, - options: { fetchImpl?: QwenFetch; signal?: AbortSignal; nowMs?: number; lookupImpl?: QwenLookup } = {}, + options: { fetchImpl?: QwenFetch; signal?: AbortSignal; nowMs?: number; lookupImpl?: QwenLookup; reservationAuthority?: QwenReservationAuthority } = {}, ): Promise<{ revision: string; runtime: RuntimeTuple; endpointIdentity: string }> { const nowMs = options.nowMs ?? Date.now(); assertQwenCircuitClosed(config, nowMs); @@ -1132,7 +1251,7 @@ export async function probeQwenReadiness( export async function ensureQwenReady( env: NodeJS.ProcessEnv = process.env, - options: { fetchImpl?: QwenFetch; signal?: AbortSignal; nowMs?: number; lookupImpl?: QwenLookup; authorityBoundary?: QwenAuthorityBoundary } = {}, + options: { fetchImpl?: QwenFetch; signal?: AbortSignal; nowMs?: number; lookupImpl?: QwenLookup; authorityBoundary?: QwenAuthorityBoundary; reservationAuthority?: QwenReservationAuthority } = {}, ): Promise { const config = loadQwenRuntimeConfig(env, options.nowMs, options.authorityBoundary); await pinQwenEndpoint(config, options.lookupImpl); @@ -1235,6 +1354,7 @@ function inferenceBody( response: Response, config: QwenRuntimeConfig, abort: AbortController, + invocation: QwenInvocation, finish: (outcome: string, status: number | null, inputTokens?: number, outputTokens?: number, responseId?: string | null) => void, ): ReadableStream { if (!response.body) fail("stream_invalid", "successful inference response has no body"); @@ -1287,6 +1407,7 @@ function inferenceBody( if (!doneSeen || !usageSeen) fail("stream_invalid", "inference stream omitted [DONE] or its strict usage object"); if (!settled) { settled = true; + invocation.reconcile(responseId, { input: inputTokens, output: outputTokens }); finish("transport_accepted", response.status, inputTokens, outputTokens, responseId); } controller.close(); @@ -1318,6 +1439,7 @@ export function createQwenInferenceFetch( onAttempt: (record: QwenAttemptRecord) => void, injectedFetch?: QwenFetch, semanticSettlement: QwenSemanticSettlement = { transportFailed: false, settled: false }, + reservationAuthority?: QwenReservationAuthority, ): QwenFetch { return async (input, init = {}) => { assertQwenCircuitClosed(config); @@ -1356,13 +1478,25 @@ export function createQwenInferenceFetch( const started = Date.now(); const startedAt = new Date(started).toISOString(); const real = injectedFetch ? null : realPinnedFetch(config, transportAttempt); + const invocation = prepareQwenInvocation(config, requestId, requestSha256, method, target.href, "inference", transportAttempt); const headers = new Headers(sourceRequest?.headers); if (init.headers) new Headers(init.headers).forEach((value, key) => headers.set(key, value)); headers.set("authorization", `Bearer ${config.secret}`); for (const [name, value] of Object.entries(qwenProviderHeaders(config, requestId))) { if (value !== null) headers.set(name, value); } - headers.set("x-vinci-idempotency-key", `${requestId}/${transportAttempt}`); + for (const [name, value] of Object.entries(permitRequestHeaders(config))) headers.set(name, value); + let reservationId: string; + try { + reservationId = await invocation.reserve(reservationAuthority); + } catch (error) { + invocation.abandonPrepared(); + real?.close(); + throw error; + } + headers.set("x-vinci-qwen-reservation-id", reservationId); + headers.set("x-vinci-idempotency-key", invocation.record.invocation_id); + headers.set("x-vinci-qwen-invocation-id", invocation.record.invocation_id); headers.set("x-vinci-qwen-request-sha256", requestSha256); let attemptReported = false; const finishAttempt = (outcome: string, status: number | null, inputTokens = 0, outputTokens = 0, responseId: string | null = null) => { @@ -1371,6 +1505,18 @@ export function createQwenInferenceFetch( real?.close(); const finished = Date.now(); const record: QwenAttemptRecord = { + invocation_id: invocation.record.invocation_id, + reservation_id: reservationId, + work_order_id: config.attribution.workOrderId, + run_id: config.attribution.runId, + attempt_id: config.attribution.attemptId, + lease_id: config.fleetPermit.leaseId, + fencing_generation: config.fleetPermit.fencingGeneration, + session_id: config.fleetPermit.sessionId, + worker_principal: config.fleetPermit.workerPrincipal, + worker_build_sha256: config.fleetPermit.workerBuildSha256, + contract_digest: config.fleetPermit.contractDigest, + execution_spec_digest: config.fleetPermit.executionSpecDigest, request_id: requestId, transport_attempt: transportAttempt, started_at: startedAt, @@ -1397,6 +1543,7 @@ export function createQwenInferenceFetch( }; let response: Response; try { + invocation.dispatch(); response = await (injectedFetch ?? real!.fetchImpl)(target, { ...init, method, @@ -1414,7 +1561,15 @@ export function createQwenInferenceFetch( await cancellableDelay(Math.min(250, config.qualification.limits.max_retry_delay_ms), controller.signal); continue; } + try { + invocation.observe(response); + } catch (error) { + finishAttempt(error instanceof QwenReadinessError ? error.code : "invocation_identity_mismatch", response.status); + throw error; + } if (response.status >= 300 && response.status < 400) { + await readBoundedText(response, config.qualification.limits.max_error_bytes, controller); + invocation.reconcile(); finishAttempt("redirect_forbidden", response.status); fail("redirect_forbidden", "inference redirects are refused"); } @@ -1425,6 +1580,7 @@ export function createQwenInferenceFetch( finishAttempt(error instanceof QwenReadinessError ? error.code : "error_body_invalid", response.status); throw error; } + invocation.reconcile(); finishAttempt(`http_${response.status}`, response.status); const retryable = response.status === 408 || response.status === 409 || response.status === 429 || response.status >= 500; if (!retryable || transportAttempt === config.qualification.limits.max_retries) { @@ -1440,6 +1596,8 @@ export function createQwenInferenceFetch( throw error; } if (!response.headers.get("content-type")?.toLowerCase().startsWith("text/event-stream")) { + await readBoundedText(response, config.qualification.limits.max_response_bytes, controller); + invocation.reconcile(); finishAttempt("content_type_invalid", response.status); fail("stream_invalid", "inference response is not text/event-stream"); } @@ -1448,7 +1606,7 @@ export function createQwenInferenceFetch( externalSignal?.removeEventListener("abort", abort); finishAttempt(outcome, status, inputTokens, outputTokens, responseId); }; - const body = inferenceBody(response, config, controller, finish); + const body = inferenceBody(response, config, controller, invocation, finish); timerOwnedByBody = true; return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers }); } @@ -1563,35 +1721,355 @@ export function validateQwenOutboundPayload(config: QwenRuntimeConfig, payload: } } -export function acquireQwenFleetPermit(config: QwenRuntimeConfig): () => void { +type QwenInvocationState = "PREPARED" | "DISPATCHED" | "RESPONSE_OBSERVED" | "RECONCILED"; +type QwenInvocationRecord = { + schema: "vinci.qwen-invocation.v1"; + invocation_id: string; + request_id: string; + request_sha256: string; + method: string; + url: string; + kind: string; + transport_attempt: number; + permit_id: string; + work_order_id: string; + run_id: string; + attempt_id: string; + lease_id: string; + fencing_generation: number; + session_id: string; + worker_principal: string; + worker_build_sha256: string; + contract_digest: string; + execution_spec_digest: string; + endpoint_sha256: string; + endpoint_identity_sha256: string; + deployment_revision: string; + reservation_id: string | null; + reserved_at: string | null; + state: QwenInvocationState; + history: Array<{ state: QwenInvocationState; at: string }>; + prepared_at: string; + dispatched_at: string | null; + response_observed_at: string | null; + reconciled_at: string | null; + status: number | null; + response_id: string | null; + usage: { input: number; output: number } | null; +}; + +type QwenInvocation = { + record: QwenInvocationRecord; + reserve: (authority?: QwenReservationAuthority) => Promise; + abandonPrepared: () => void; + dispatch: () => void; + observe: (response: Response) => void; + reconcile: (responseId?: string | null, usage?: { input: number; output: number } | null) => void; +}; + +function durableJson(path: string, value: unknown): void { + const directory = dirname(path); + const temporary = `${path}.tmp-${process.pid}-${randomBytes(8).toString("hex")}`; + const descriptor = openSync(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600); + try { + writeFileSync(descriptor, `${qwenCanonical(value)}\n`); + fsyncSync(descriptor); + } finally { + closeSync(descriptor); + } + renameSync(temporary, path); + syncInvocationDirectory(directory); +} + +function syncInvocationDirectory(directory: string): void { + const directoryDescriptor = openSync(directory, constants.O_RDONLY); + try { + fsyncSync(directoryDescriptor); + } finally { + closeSync(directoryDescriptor); + } +} + +function invocationLockPath(config: QwenRuntimeConfig): string { + return join(config.fleetPermit.lockDirectory, "qwen-h200-concurrency-1"); +} + +function releaseInvocationLock(lockPath: string): void { + try { + unlinkSync(lockPath); + syncInvocationDirectory(dirname(lockPath)); + } catch { + fail("fleet_permit_release_failed", "the endpoint-global Qwen invocation lock could not be released cleanly"); + } +} + +function readInvocation(path: string): QwenInvocationRecord { + let value: unknown; + try { + value = JSON.parse(readFileSync(path, "utf8")); + } catch { + fail("invocation_ledger_invalid", "the durable Qwen invocation ledger is unreadable"); + } + if (!value || typeof value !== "object" || (value as Record).schema !== "vinci.qwen-invocation.v1") { + fail("invocation_ledger_invalid", "the durable Qwen invocation ledger has the wrong schema"); + } + return value as QwenInvocationRecord; +} + +function prepareQwenInvocation( + config: QwenRuntimeConfig, + requestId: string, + requestSha256: string, + method: string, + url: string, + kind: string, + transportAttempt: number, +): QwenInvocation { const expiresAt = Date.parse(config.fleetPermit.expiresAt); if (!Number.isFinite(expiresAt) || expiresAt - Date.now() < config.qualification.limits.total_timeout_ms) { fail("fleet_permit_expired", "the external Qwen fleet permit cannot cover the full bounded request deadline"); } - const lockPath = join(config.fleetPermit.lockDirectory, "qwen-h200-concurrency-1"); + const lockPath = invocationLockPath(config); + const invocationId = qwenSha256(`${config.fleetPermit.permitId}\0${requestId}\0${transportAttempt}\0${requestSha256}\0${randomBytes(16).toString("hex")}`); + const ledgerPath = join(config.fleetPermit.lockDirectory, `qwen-invocation-${invocationId}.json`); + const now = new Date().toISOString(); + const record: QwenInvocationRecord = { + schema: "vinci.qwen-invocation.v1", + invocation_id: invocationId, + request_id: requestId, + request_sha256: requestSha256, + method, + url, + kind, + transport_attempt: transportAttempt, + permit_id: config.fleetPermit.permitId, + work_order_id: config.attribution.workOrderId, + run_id: config.attribution.runId, + attempt_id: config.attribution.attemptId, + lease_id: config.fleetPermit.leaseId, + fencing_generation: config.fleetPermit.fencingGeneration, + session_id: config.fleetPermit.sessionId, + worker_principal: config.fleetPermit.workerPrincipal, + worker_build_sha256: config.fleetPermit.workerBuildSha256, + contract_digest: config.fleetPermit.contractDigest, + execution_spec_digest: config.fleetPermit.executionSpecDigest, + endpoint_sha256: config.qualification.bindings.endpoint_sha256, + endpoint_identity_sha256: config.qualification.bindings.endpoint_identity_sha256, + deployment_revision: config.qualification.bindings.revision, + reservation_id: null, + reserved_at: null, + state: "PREPARED", + history: [{ state: "PREPARED", at: now }], + prepared_at: now, + dispatched_at: null, + response_observed_at: null, + reconciled_at: null, + status: null, + response_id: null, + usage: null, + }; + const preparedLock = join(config.fleetPermit.lockDirectory, `.qwen-lock-${invocationId}`); try { - mkdirSync(lockPath, { mode: 0o700 }); + durableJson(ledgerPath, record); + durableJson(preparedLock, { invocation_id: invocationId, ledger_path: ledgerPath }); + linkSync(preparedLock, lockPath); + unlinkSync(preparedLock); + syncInvocationDirectory(config.fleetPermit.lockDirectory); } catch (error) { + try { unlinkSync(preparedLock); } catch {} if ((error as NodeJS.ErrnoException).code === "EEXIST") { - fail("concurrency_exceeded", "the external Qwen fleet permit is already held by another provider process"); + fail("invocation_unresolved", "an earlier Qwen invocation remains remotely unresolved"); } - fail("fleet_permit_unavailable", "the external Qwen fleet permit could not be acquired"); + throw error; } - let released = false; - return () => { - if (released) return; - released = true; - try { - rmdirSync(lockPath); - } catch { - fail("fleet_permit_release_failed", "the external Qwen fleet permit could not be released cleanly"); - } + const transition = (state: QwenInvocationState) => { + record.state = state; + record.history.push({ state, at: new Date().toISOString() }); + durableJson(ledgerPath, record); + }; + return { + record, + async reserve(authority) { + if (record.state !== "PREPARED" || record.reservation_id !== null) { + fail("invocation_state_invalid", "Qwen invocation may be reserved exactly once before dispatch"); + } + if (!authority) { + fail("dispatcher_unavailable", "the separate-UID VGC dispatcher is required to mint a point-of-use reservation"); + } + const request: QwenPointOfUseReservationRequest = { + schema: "vinci.qwen-point-of-use-reservation-request.v1", + invocation_id: record.invocation_id, + request_sha256: record.request_sha256, + permit_id: record.permit_id, + work_order_id: record.work_order_id, + run_id: record.run_id, + attempt_id: record.attempt_id, + lease_id: record.lease_id, + fencing_generation: record.fencing_generation, + session_id: record.session_id, + worker_principal: record.worker_principal, + worker_build_sha256: record.worker_build_sha256, + contract_digest: record.contract_digest, + execution_spec_digest: record.execution_spec_digest, + endpoint_sha256: record.endpoint_sha256, + endpoint_identity_sha256: record.endpoint_identity_sha256, + deployment_revision: record.deployment_revision, + }; + const trusted = await authority(Object.freeze(request)); + const keys = [ + "schema", "authority", "scope", "reservation_id", "invocation_id", "request_sha256", "permit_id", + "work_order_id", "run_id", "attempt_id", "lease_id", "fencing_generation", "session_id", + "worker_principal", "worker_build_sha256", "contract_digest", "execution_spec_digest", "endpoint_sha256", + "endpoint_identity_sha256", "deployment_revision", "issued_at", "expires_at", + ]; + exactKeys(trusted, keys, "Qwen point-of-use reservation"); + const exactBindings = [ + "invocation_id", "request_sha256", "permit_id", "work_order_id", "run_id", "attempt_id", "lease_id", + "fencing_generation", "session_id", "worker_principal", "worker_build_sha256", "contract_digest", + "execution_spec_digest", "endpoint_sha256", "endpoint_identity_sha256", "deployment_revision", + ] as const; + if ( + trusted.schema !== "vinci.qwen-point-of-use-reservation.v1" || + trusted.authority !== "vgc-fleet-permit-authority" || trusted.scope !== "endpoint-global" || + exactBindings.some((key) => trusted[key] !== request[key]) || + typeof trusted.reservation_id !== "string" || !IDENTIFIER.test(trusted.reservation_id) + ) { + fail("reservation_invalid", "server-minted point-of-use reservation does not bind the exact invocation authority tuple"); + } + const issued = timestamp(trusted.issued_at, "point-of-use reservation issued_at"); + const expires = timestamp(trusted.expires_at, "point-of-use reservation expires_at"); + const nowMs = Date.now(); + if (issued.time > nowMs + 30_000 || expires.time <= nowMs || expires.time - issued.time > 5 * 60_000 || expires.time - nowMs < config.qualification.limits.total_timeout_ms) { + fail("reservation_invalid", "point-of-use reservation is stale or cannot cover the bounded request deadline"); + } + record.reservation_id = trusted.reservation_id; + record.reserved_at = new Date().toISOString(); + durableJson(ledgerPath, record); + return record.reservation_id; + }, + abandonPrepared() { + if (record.state !== "PREPARED") fail("invocation_state_invalid", "only an undispatched Qwen invocation may abandon local occupancy"); + releaseInvocationLock(lockPath); + }, + dispatch() { + if (record.state !== "PREPARED") fail("invocation_state_invalid", "Qwen invocation was not prepared exactly once"); + if (!record.reservation_id || !record.reserved_at) fail("reservation_missing", "Qwen invocation lacks a fresh server-minted point-of-use reservation"); + record.dispatched_at = new Date().toISOString(); + transition("DISPATCHED"); + }, + observe(response) { + if (record.state !== "DISPATCHED") fail("invocation_state_invalid", "Qwen response arrived outside a dispatched invocation"); + if ( + response.headers.get("x-vinci-qwen-invocation-id") !== invocationId || + response.headers.get("x-vinci-qwen-request-sha256") !== requestSha256 || + response.headers.get("x-vinci-qwen-reservation-id") !== record.reservation_id || + response.headers.get("x-vinci-model-revision") !== record.deployment_revision || + response.headers.get("x-vinci-qwen-fleet-permit-id") !== record.permit_id || + response.headers.get("x-vinci-lease-id") !== record.lease_id || + response.headers.get("x-vinci-fencing-generation") !== String(record.fencing_generation) || + response.headers.get("x-vinci-session-id") !== record.session_id || + response.headers.get("x-vinci-worker-principal") !== record.worker_principal || + response.headers.get("x-vinci-worker-build-sha256") !== record.worker_build_sha256 || + response.headers.get("x-vinci-contract-digest") !== record.contract_digest || + response.headers.get("x-vinci-execution-spec-digest") !== record.execution_spec_digest || + response.headers.get("x-vinci-qwen-permit-authority") !== "vgc-fleet-permit-authority" || + response.headers.get("x-vinci-qwen-permit-scope") !== "endpoint-global" || + response.headers.get("x-vinci-qwen-permit-state") !== "held" + ) { + fail("invocation_identity_mismatch", "response did not bind the reservation, invocation, request, deployment, and full endpoint-global authority tuple"); + } + record.response_observed_at = new Date().toISOString(); + record.status = response.status; + transition("RESPONSE_OBSERVED"); + }, + reconcile(responseId = null, usage = null) { + if (record.state !== "RESPONSE_OBSERVED") fail("invocation_state_invalid", "Qwen invocation cannot reconcile before a trusted response"); + record.response_id = responseId; + record.usage = usage; + record.reconciled_at = new Date().toISOString(); + transition("RECONCILED"); + releaseInvocationLock(lockPath); + }, }; } -function canaryConfig(env: NodeJS.ProcessEnv): QwenRuntimeConfig { +export function reconcileQwenInvocationFromAuthority( + config: QwenRuntimeConfig, + receipt?: Record, +): void { + if (receipt && process.env.VINCI_QWEN_TEST_RECONCILIATION !== "1") { + fail("authority_forbidden", "in-memory Qwen reconciliation is available only to the local adversarial test harness"); + } + const lockPath = invocationLockPath(config); + let active: { invocation_id: string; ledger_path: string }; + try { + active = JSON.parse(readFileSync(lockPath, "utf8")) as typeof active; + } catch { + fail("invocation_unresolved", "no readable active invocation is available for reconciliation"); + } + const ledger = readInvocation(active.ledger_path); + const trusted = receipt ?? (() => { + const path = join(config.fleetPermit.reconciliationDirectory, `${active.invocation_id}.json`); + return JSON.parse(secureAuthorityFile(path, "Qwen authority reconciliation", MAX_QUALIFICATION_BYTES).toString("utf8")) as Record; + })(); + exactKeys(trusted, [ + "schema", "authority", "invocation_id", "reservation_id", "request_sha256", "permit_id", "work_order_id", "run_id", + "attempt_id", "lease_id", "fencing_generation", "session_id", "worker_principal", "worker_build_sha256", + "contract_digest", "execution_spec_digest", "endpoint_sha256", "endpoint_identity_sha256", "deployment_revision", + "status", "response_id", "usage", + ], "Qwen authority reconciliation"); + const reconciliationBindings = [ + "invocation_id", "reservation_id", "request_sha256", "permit_id", "work_order_id", "run_id", "attempt_id", "lease_id", + "fencing_generation", "session_id", "worker_principal", "worker_build_sha256", "contract_digest", "execution_spec_digest", + "endpoint_sha256", "endpoint_identity_sha256", "deployment_revision", + ] as const; + if ( + trusted.schema !== "vinci.qwen-invocation-reconciliation.v1" || trusted.authority !== "vgc-fleet-permit-authority" || + reconciliationBindings.some((key) => trusted[key] !== ledger[key]) || typeof trusted.status !== "number" + ) fail("invocation_reconciliation_invalid", "authority reconciliation does not match the unresolved invocation"); + if (!Number.isSafeInteger(trusted.status) || trusted.status < 100 || trusted.status > 599) { + fail("invocation_reconciliation_invalid", "authority reconciliation status is invalid"); + } + if (trusted.response_id !== null && (typeof trusted.response_id !== "string" || !trusted.response_id || trusted.response_id.length > 512)) { + fail("invocation_reconciliation_invalid", "authority reconciliation response id is invalid"); + } + if (trusted.usage !== null) { + exactKeys(trusted.usage, ["input", "output"], "authority reconciliation usage"); + const usage = trusted.usage as Record; + if (![usage.input, usage.output].every((value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 0)) { + fail("invocation_reconciliation_invalid", "authority reconciliation usage is invalid"); + } + } + if (ledger.kind === "inference" && trusted.status >= 200 && trusted.status < 300 && (trusted.response_id === null || trusted.usage === null)) { + fail("invocation_reconciliation_invalid", "successful inference reconciliation requires bound response id and usage"); + } + ledger.status = trusted.status; + ledger.response_id = typeof trusted.response_id === "string" ? trusted.response_id : null; + ledger.usage = trusted.usage && typeof trusted.usage === "object" ? trusted.usage as { input: number; output: number } : null; + ledger.reconciled_at = new Date().toISOString(); + ledger.state = "RECONCILED"; + ledger.history.push({ state: "RECONCILED", at: ledger.reconciled_at }); + durableJson(active.ledger_path, ledger); + releaseInvocationLock(lockPath); +} + +function canaryConfig(env: NodeJS.ProcessEnv, nowMs: number, injectedAuthority?: QwenAuthorityBoundary): QwenRuntimeConfig { const urls = normalizeQwenBaseUrl(env.VINCI_QWEN_BASE_URL); const secret = readCanarySecret(env.VINCI_QWEN_SECRET_REF); + const workOrderId = requiredEnv(env, "VINCI_QWEN_CANARY_WORK_ORDER_ID"); + const runId = requiredEnv(env, "VINCI_QWEN_CANARY_RUN_ID"); + const attemptId = requiredEnv(env, "VINCI_QWEN_CANARY_ATTEMPT_ID"); + const authority = loadIndependentAuthority(workOrderId, runId, attemptId, nowMs, injectedAuthority); + const revision = requiredEnv(env, "VINCI_QWEN_SERVED_REVISION"); + if (!IMMUTABLE_REVISION.test(revision)) fail("config_invalid", "canary served revision must be immutable"); + const runtime = validateRuntime({ + engine: requiredEnv(env, "VINCI_QWEN_RUNTIME_ENGINE"), + version: requiredEnv(env, "VINCI_QWEN_RUNTIME_VERSION"), + artifact_sha256: requiredEnv(env, "VINCI_QWEN_RUNTIME_ARTIFACT_SHA256"), + arguments_sha256: requiredEnv(env, "VINCI_QWEN_RUNTIME_ARGUMENTS_SHA256"), + }); + const endpointIdentity = requiredEnv(env, "VINCI_QWEN_ENDPOINT_IDENTITY_SHA256"); + if (!HEX64.test(endpointIdentity)) fail("config_invalid", "canary endpoint identity must be SHA-256"); return { ...urls, endpointAddresses: [], @@ -1629,10 +2107,10 @@ function canaryConfig(env: NodeJS.ProcessEnv): QwenRuntimeConfig { }, bindings: { model: QWEN_MODEL, - revision: "0".repeat(40), - runtime: { engine: "canary", version: "canary", artifact_sha256: "0".repeat(64), arguments_sha256: "0".repeat(64) }, + revision, + runtime, endpoint_sha256: qwenSha256(urls.baseUrl), - endpoint_identity_sha256: "0".repeat(64), + endpoint_identity_sha256: endpointIdentity, work_order_prompt_sha256: "0".repeat(64), system_prompt_sha256: "0".repeat(64), tool_names_sha256: "0".repeat(64), @@ -1669,8 +2147,20 @@ function canaryConfig(env: NodeJS.ProcessEnv): QwenRuntimeConfig { circuitFile: "/canary/unused", circuitThreshold: 1, circuitOpenMs: 1_000, - fleetPermit: { permitId: "canary-only", lockDirectory: "/canary", expiresAt: new Date(Date.now() + 1_000).toISOString() }, - attribution: { workOrderId: "canary-read-only", runId: "canary-read-only", attemptId: "canary-read-only/1" }, + fleetPermit: { + permitId: authority.fleetPermit.permitId, + lockDirectory: authority.fleetPermit.lockDirectory, + reconciliationDirectory: authority.fleetPermit.reconciliationDirectory, + expiresAt: authority.fleetPermit.expiresAt, + leaseId: authority.fleetPermit.leaseId, + fencingGeneration: authority.fleetPermit.fencingGeneration, + sessionId: authority.fleetPermit.sessionId, + workerPrincipal: authority.fleetPermit.workerPrincipal, + workerBuildSha256: authority.fleetPermit.workerBuildSha256, + contractDigest: authority.fleetPermit.contractDigest, + executionSpecDigest: authority.fleetPermit.executionSpecDigest, + }, + attribution: { workOrderId, runId, attemptId }, }; } @@ -1678,21 +2168,25 @@ export async function runQwenCanary( env: NodeJS.ProcessEnv = process.env, fetchImpl?: QwenFetch, lookupImpl?: QwenLookup, + authorityBoundary?: QwenAuthorityBoundary, + reservationAuthority?: QwenReservationAuthority, ): Promise> { - const config = canaryConfig(env); + const config = canaryConfig(env, Date.now(), authorityBoundary); await pinQwenEndpoint(config, lookupImpl); const maximum = config.qualification.limits.max_error_bytes; const started = Date.now(); - const health = await boundedRequest(config, config.healthUrl, { headers: authHeaders(config) }, maximum, { fetchImpl }); + const health = await boundedRequest(config, config.healthUrl, { headers: authHeaders(config) }, maximum, { fetchImpl, reservationAuthority }); await validateHealthResponse(health.response, health.text); - const models = await boundedRequest(config, config.modelsUrl, { headers: authHeaders(config) }, maximum, { fetchImpl }); + const models = await boundedRequest(config, config.modelsUrl, { headers: authHeaders(config) }, maximum, { fetchImpl, reservationAuthority }); if (!models.response.ok) fail("models_failed", `authenticated /v1/models returned ${models.response.status}`); const identity = servedIdentity(models.response, JSON.parse(models.text)); - config.qualification.bindings.revision = identity.revision; - config.qualification.bindings.runtime = identity.runtime; - config.qualification.bindings.endpoint_identity_sha256 = identity.endpointIdentity; + if ( + identity.revision !== config.qualification.bindings.revision || + qwenCanonical(identity.runtime) !== qwenCanonical(config.qualification.bindings.runtime) || + identity.endpointIdentity !== config.qualification.bindings.endpoint_identity_sha256 + ) fail("runtime_mismatch", "canary endpoint identity differs from the externally bound deployment"); for (const [url, path] of [[config.healthUrl, "/health"], [config.modelsUrl, "/v1/models"]] as const) { - const anonymous = await boundedRequest(config, url, { headers: { accept: "application/json" } }, maximum, { fetchImpl }); + const anonymous = await boundedRequest(config, url, { headers: { accept: "application/json" } }, maximum, { fetchImpl, reservationAuthority }); if (anonymous.response.status !== 401 && anonymous.response.status !== 403) fail("auth_not_enforced", `unauthenticated ${path} was not refused`); } const response = await boundedRequest( @@ -1708,36 +2202,21 @@ export async function runQwenCanary( temperature: 0, max_tokens: 64, messages: [ - { role: "system", content: "Call report_ready exactly once. Do not return prose." }, - { role: "user", content: "Report readiness." }, + { role: "system", content: "Return exactly READY and nothing else." }, + { role: "user", content: "READY" }, ], - tools: [{ - type: "function", - function: { - name: "report_ready", - description: "Reports deterministic worker compatibility.", - strict: false, - parameters: { - type: "object", - properties: { status: { type: "string", enum: ["ready"] } }, - required: ["status"], - additionalProperties: false, - }, - }, - }], - tool_choice: { type: "function", function: { name: "report_ready" } }, }), }, MAX_CANARY_BYTES, - { fetchImpl }, + { fetchImpl, reservationAuthority }, ); - if (!response.response.ok) fail("canary_failed", `streaming tool-call inference returned ${response.response.status}`); + if (!response.response.ok) fail("canary_failed", `streaming tool-free inference returned ${response.response.status}`); responseIdentityHeaders(response.response, config); if (!response.response.headers.get("content-type")?.toLowerCase().startsWith("text/event-stream")) { - fail("canary_invalid", "streaming tool-call inference did not return text/event-stream"); + fail("canary_invalid", "streaming tool-free inference did not return text/event-stream"); } - let toolName = ""; - let argumentsText = ""; + let content = ""; + let terminalStop = false; let usageSeen = false; let doneSeen = false; for (const line of response.text.split(/\r?\n/)) { @@ -1758,27 +2237,18 @@ export async function runQwenCanary( const choices = Array.isArray(chunk.choices) ? chunk.choices : []; for (const choice of choices) { const delta = choice && typeof choice === "object" ? (choice as Record).delta : null; - const toolCalls = delta && typeof delta === "object" && Array.isArray((delta as Record).tool_calls) - ? ((delta as Record).tool_calls as unknown[]) - : []; - for (const call of toolCalls) { - const fn = call && typeof call === "object" ? (call as Record).function : null; - if (!fn || typeof fn !== "object") continue; - const name = (fn as Record).name; - const args = (fn as Record).arguments; - if (typeof name === "string") toolName += name; - if (typeof args === "string") argumentsText += args; + if (delta && typeof delta === "object") { + const deltaRecord = delta as Record; + if (Array.isArray(deltaRecord.tool_calls) && deltaRecord.tool_calls.length > 0) { + fail("canary_invalid", "tool-free canary emitted a tool call"); + } + if (typeof deltaRecord.content === "string") content += deltaRecord.content; } + if (choice && typeof choice === "object" && (choice as Record).finish_reason === "stop") terminalStop = true; } } - let argumentsValue: unknown; - try { - argumentsValue = JSON.parse(argumentsText); - } catch { - fail("canary_invalid", "tool-call arguments were not complete JSON"); - } - if (toolName !== "report_ready" || qwenCanonical(argumentsValue) !== qwenCanonical({ status: "ready" }) || !usageSeen || !doneSeen) { - fail("canary_invalid", "stream omitted the required structured tool call, strict usage, or [DONE]"); + if (content !== "READY" || !terminalStop || !usageSeen || !doneSeen) { + fail("canary_invalid", "stream omitted the exact READY response, terminal stop, strict usage, or [DONE]"); } return { schema: CANARY_SCHEMA, @@ -1791,7 +2261,7 @@ export async function runQwenCanary( runtime: identity.runtime, authenticated: true, anonymous_refused: true, - capabilities: { streaming_sse: true, tool_calls: true, structured_output: "tool-arguments-json", usage_chunk: true }, + capabilities: { streaming_sse: true, tool_calls: false, structured_output: "not-exercised", usage_chunk: true, tool_free: true }, latency_ms: Date.now() - started, authority_role: AUTHORITY_ROLE, fallback_policy: FALLBACK_POLICY, @@ -1884,7 +2354,7 @@ export function buildQwenQualificationRequest(env: NodeJS.ProcessEnv = process.e capabilities: (canary as Record).capabilities, limits: { total_timeout_ms: Number(env.VINCI_QWEN_TOTAL_TIMEOUT_MS ?? "120000"), - max_retries: Number(env.VINCI_QWEN_MAX_RETRIES ?? "1"), + max_retries: Number(env.VINCI_QWEN_MAX_RETRIES ?? "0"), max_retry_delay_ms: Number(env.VINCI_QWEN_MAX_RETRY_DELAY_MS ?? "5000"), max_concurrency: Number(env.VINCI_QWEN_MAX_CONCURRENCY ?? "1"), advertised_max_concurrency: numberEnv("VINCI_QWEN_ADVERTISED_MAX_CONCURRENCY"), @@ -1926,6 +2396,7 @@ export function qwenProviderHeaders(config: QwenRuntimeConfig, requestId: string "x-vinci-qwen-fleet-permit-id": config.fleetPermit.permitId, "x-vinci-qwen-output-authority": "non-authoritative", "x-vinci-qwen-qualification-sha256": config.qualificationSha256, + ...permitRequestHeaders(config), }; } diff --git a/vinci/extensions/vinci-qwen-provider.ts b/vinci/extensions/vinci-qwen-provider.ts index c553cdcb1..65127bccd 100644 --- a/vinci/extensions/vinci-qwen-provider.ts +++ b/vinci/extensions/vinci-qwen-provider.ts @@ -9,7 +9,6 @@ import { import { streamSimpleOpenAICompletions } from "@earendil-works/pi-ai/compat"; import { Value } from "typebox/value"; import { - acquireQwenFleetPermit, assertQwenCircuitClosed, assertQwenContextBindings, createQwenInferenceFetch, @@ -24,6 +23,7 @@ import { type QwenAttemptRecord, type QwenFetch, type QwenRuntimeConfig, + type QwenReservationAuthority, type QwenSemanticSettlement, settleQwenSemanticOutcome, validateQwenOutboundPayload, @@ -94,6 +94,7 @@ export function qwenProviderConfig( streamOpenAI = streamSimpleOpenAICompletions, onAttempt: (record: QwenAttemptRecord) => void = () => {}, injectedFetch?: QwenFetch, + reservationAuthority?: QwenReservationAuthority, ) { let inFlight = 0; let requestOrdinal = 0; @@ -110,17 +111,12 @@ export function qwenProviderConfig( if (inFlight >= runtime.qualification.limits.max_concurrency) { throw new Error("qwen_concurrency_exceeded: the qualified single-request bound is already in use"); } - const releaseFleetPermit = acquireQwenFleetPermit(runtime); inFlight += 1; let released = false; const release = () => { if (released) return; released = true; - try { - releaseFleetPermit(); - } finally { - inFlight -= 1; - } + inFlight -= 1; }; requestOrdinal += 1; const requestId = qwenSha256(`${runtime.attribution.workOrderId}\0${runtime.attribution.runId}\0${runtime.attribution.attemptId}\0${requestOrdinal}`); @@ -136,7 +132,7 @@ export function qwenProviderConfig( headers: { ...options?.headers, ...qwenProviderHeaders(runtime, requestId) }, timeoutMs: runtime.qualification.limits.total_timeout_ms, maxRetries: 0, - fetch: createQwenInferenceFetch(runtime, requestId, onAttempt, injectedFetch, semanticSettlement), + fetch: createQwenInferenceFetch(runtime, requestId, onAttempt, injectedFetch, semanticSettlement, reservationAuthority), onPayload: (payload) => { validateQwenOutboundPayload(runtime, payload); return payload; @@ -166,9 +162,6 @@ export function qwenProviderConfig( ) { throw new Error("qwen_semantic_invalid: terminal response failed exact identity, usage, finish, or tool semantics"); } - // A permit-release failure is itself a failed request. Never persist semantic - // success until the external concurrency authority has been released cleanly. - release(); settleQwenSemanticOutcome(runtime, semanticSettlement, true, "success", onAttempt); } // Release before the terminal event can resolve result() or become observable to a @@ -242,6 +235,13 @@ export default async function (pi: ExtensionAPI) { if (process.env.VINCI_QWEN_SELECTED !== "1") { throw new Error("qwen_not_selected: the Qwen extension may load only for a Worker-selected Qwen attempt"); } + // Deliberate production NO-GO. The reservation callback used by the tests is not an authority + // implementation, and an environment flag would let the Worker nominate its own authority. + // Replace this stop only when the separately deployed, separate-UID credential-owning dispatcher + // has a concrete client that returns authenticated server-minted reservations and reconciliations. + throw new Error("qwen_dispatcher_unavailable: joined Qwen WorkOrders are disabled until the separate-UID VGC dispatcher is integrated"); + + /* c8 ignore start -- retained integration path, unreachable until the upstream dispatcher lands */ let runtime: QwenRuntimeConfig; try { runtime = await ensureQwenReady(); @@ -287,4 +287,5 @@ export default async function (pi: ExtensionAPI) { session_id: ctx.sessionManager.getSessionId(), }); }); + /* c8 ignore stop */ } diff --git a/vinci/test/worker-qwen-provider.mjs b/vinci/test/worker-qwen-provider.mjs index 878d7cc01..86600eb38 100644 --- a/vinci/test/worker-qwen-provider.mjs +++ b/vinci/test/worker-qwen-provider.mjs @@ -7,7 +7,7 @@ import { join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { streamSimple as sourceStreamSimpleOpenAICompletions } from "../../packages/ai/src/api/openai-completions.ts"; import * as runtime from "../extensions/lib/qwen-runtime.ts"; -import { qwenProviderConfig } from "../extensions/vinci-qwen-provider.ts"; +import qwenExtension, { qwenProviderConfig as rawQwenProviderConfig } from "../extensions/vinci-qwen-provider.ts"; import * as cleanroom from "../worker/cleanroom.mjs"; import * as digest from "../worker/contracts/digest.mjs"; import * as economics from "../worker/economics.mjs"; @@ -25,6 +25,7 @@ const burnInFile = join(temp, "burn-in.json"); const qualificationFile = join(temp, "qualification.json"); const publicKeyFile = join(temp, "qualification-key.pem"); const permitLockDirectory = join(temp, "permit-locks"); +const reconciliationDirectory = join(temp, "reconciliations"); const endpointIdentity = "12".repeat(32); const revision = "ab".repeat(20); const runtimeTuple = { @@ -79,35 +80,6 @@ recordQwenCircuitOutcome({ circuitFile, circuitThreshold: 100, circuitOpenMs: 60 }))); } -async function exerciseCrossProcessFleetPermit(config) { - const release = runtime.acquireQwenFleetPermit(config); - const workerFile = join(temp, "permit-worker.mjs"); - const runtimeUrl = pathToFileURL(join(root, "vinci/extensions/lib/qwen-runtime.ts")).href; - writeFileSync(workerFile, `import { acquireQwenFleetPermit } from ${JSON.stringify(runtimeUrl)}; -const [lockDirectory, permitId, expiresAt] = process.argv.slice(2); -try { - acquireQwenFleetPermit({ fleetPermit: { lockDirectory, permitId, expiresAt }, qualification: { limits: { total_timeout_ms: 1_000 } } }); - process.exitCode = 2; -} catch (error) { - if (error?.code !== "concurrency_exceeded") throw error; -} -`, { mode: 0o600 }); - try { - const child = spawn(process.execPath, [...process.execArgv, workerFile, config.fleetPermit.lockDirectory, config.fleetPermit.permitId, config.fleetPermit.expiresAt], { - stdio: ["ignore", "pipe", "pipe"], - }); - await new Promise((resolveExit, rejectExit) => { - let stderr = ""; - child.stderr.setEncoding("utf8"); - child.stderr.on("data", (chunk) => { stderr += chunk; }); - child.on("error", rejectExit); - child.on("exit", (code) => code === 0 ? resolveExit() : rejectExit(new Error(`permit worker exited ${code}: ${stderr}`))); - }); - } finally { - release(); - } -} - function freshenPermit(config) { config.fleetPermit.expiresAt = new Date(Date.now() + 60_000).toISOString(); return config; @@ -115,6 +87,7 @@ function freshenPermit(config) { writeFileSync(secretFile, "synthetic-test-secret\n", { mode: 0o600 }); mkdirSync(permitLockDirectory, { mode: 0o700 }); +mkdirSync(reconciliationDirectory, { mode: 0o700 }); writeFileSync(promptFile, workOrderPrompt, { mode: 0o400 }); writeFileSync(systemPromptFile, systemPrompt, { mode: 0o400 }); writeFileSync(toolSchemasFile, `${JSON.stringify(tools)}\n`, { mode: 0o400 }); @@ -286,6 +259,14 @@ const authorityBoundary = { attemptId: "task-test/1", maxConcurrency: 1, lockDirectory: permitLockDirectory, + reconciliationDirectory, + leaseId: "lease-authority-test", + fencingGeneration: 7, + sessionId: "run-test", + workerPrincipal: "worker:test", + workerBuildSha256: "31".repeat(32), + contractDigest: "32".repeat(32), + executionSpecDigest: "33".repeat(32), issuedAt: "2026-09-04T17:59:00.000Z", expiresAt: "2026-09-04T18:04:00.000Z", }, @@ -296,12 +277,58 @@ function runtimeEnv(overrides = {}) { } function loadConfig(overrides = {}) { - return runtime.loadQwenRuntimeConfig(runtimeEnv(overrides), nowMs, authorityBoundary); + return freshenPermit(runtime.loadQwenRuntimeConfig(runtimeEnv(overrides), nowMs, authorityBoundary)); +} + +const reservationRequests = []; +function reservationAuthority(request) { + reservationRequests.push(structuredClone(request)); + const issuedAt = new Date().toISOString(); + return { + ...request, + schema: "vinci.qwen-point-of-use-reservation.v1", + authority: "vgc-fleet-permit-authority", + scope: "endpoint-global", + reservation_id: `reservation:${request.invocation_id}`, + issued_at: issuedAt, + expires_at: new Date(Date.now() + 60_000).toISOString(), + }; } -function identityHeaders(overrides = {}) { +function inferenceFetch(config, requestId, onAttempt, fetchImpl, settlement) { + return runtime.createQwenInferenceFetch(config, requestId, onAttempt, fetchImpl, settlement, reservationAuthority); +} + +function qwenProviderConfig(config, streamOpenAI, onAttempt, fetchImpl) { + return rawQwenProviderConfig(config, streamOpenAI, onAttempt, fetchImpl, reservationAuthority); +} + +function invocationHeaders(init = {}, overrides = {}) { + const requestHeaders = new Headers(init.headers); + return { + "x-vinci-qwen-invocation-id": requestHeaders.get("x-vinci-qwen-invocation-id"), + "x-vinci-qwen-request-sha256": requestHeaders.get("x-vinci-qwen-request-sha256"), + "x-vinci-qwen-reservation-id": requestHeaders.get("x-vinci-qwen-reservation-id"), + "x-vinci-model-revision": revision, + "x-vinci-qwen-fleet-permit-id": requestHeaders.get("x-vinci-qwen-fleet-permit-id"), + "x-vinci-lease-id": requestHeaders.get("x-vinci-lease-id"), + "x-vinci-fencing-generation": requestHeaders.get("x-vinci-fencing-generation"), + "x-vinci-session-id": requestHeaders.get("x-vinci-session-id"), + "x-vinci-worker-principal": requestHeaders.get("x-vinci-worker-principal"), + "x-vinci-worker-build-sha256": requestHeaders.get("x-vinci-worker-build-sha256"), + "x-vinci-contract-digest": requestHeaders.get("x-vinci-contract-digest"), + "x-vinci-execution-spec-digest": requestHeaders.get("x-vinci-execution-spec-digest"), + "x-vinci-qwen-permit-authority": "vgc-fleet-permit-authority", + "x-vinci-qwen-permit-scope": "endpoint-global", + "x-vinci-qwen-permit-state": "held", + ...overrides, + }; +} + +function identityHeaders(overrides = {}, init = {}) { return { "content-type": "text/event-stream", + ...invocationHeaders(init), "x-vinci-model-id": runtime.QWEN_MODEL, "x-vinci-model-revision": revision, "x-vinci-endpoint-identity-sha256": endpointIdentity, @@ -313,6 +340,42 @@ function identityHeaders(overrides = {}) { }; } +function reconcileActive(config) { + const active = JSON.parse(readFileSync(join(config.fleetPermit.lockDirectory, "qwen-h200-concurrency-1"), "utf8")); + const ledger = JSON.parse(readFileSync(active.ledger_path, "utf8")); + const status = ledger.status ?? 599; + process.env.VINCI_QWEN_TEST_RECONCILIATION = "1"; + try { + runtime.reconcileQwenInvocationFromAuthority(config, { + schema: "vinci.qwen-invocation-reconciliation.v1", + authority: "vgc-fleet-permit-authority", + invocation_id: ledger.invocation_id, + reservation_id: ledger.reservation_id, + request_sha256: ledger.request_sha256, + permit_id: ledger.permit_id, + work_order_id: ledger.work_order_id, + run_id: ledger.run_id, + attempt_id: ledger.attempt_id, + lease_id: ledger.lease_id, + fencing_generation: ledger.fencing_generation, + session_id: ledger.session_id, + worker_principal: ledger.worker_principal, + worker_build_sha256: ledger.worker_build_sha256, + contract_digest: ledger.contract_digest, + execution_spec_digest: ledger.execution_spec_digest, + endpoint_sha256: ledger.endpoint_sha256, + endpoint_identity_sha256: ledger.endpoint_identity_sha256, + deployment_revision: ledger.deployment_revision, + status, + response_id: ledger.response_id ?? (status >= 200 && status < 300 ? "authority-reconciled" : null), + usage: ledger.usage ?? (status >= 200 && status < 300 ? { input: 0, output: 0 } : null), + }); + } finally { + delete process.env.VINCI_QWEN_TEST_RECONCILIATION; + } + return ledger; +} + const usage = { prompt_tokens: 10, completion_tokens: 2, @@ -335,12 +398,24 @@ const requestBody = JSON.stringify({ }); try { + process.env.VINCI_QWEN_SELECTED = "1"; + try { + await assert.rejects(qwenExtension({}), /qwen_dispatcher_unavailable/, "joined WorkOrders must remain NO-GO without the upstream dispatcher"); + } finally { + delete process.env.VINCI_QWEN_SELECTED; + } + assert.equal( + runtime.buildQwenQualificationRequest({ ...requestEnv, VINCI_QWEN_MAX_RETRIES: undefined }).candidate.limits.max_retries, + 0, + "automatic Qwen retries must default to zero", + ); assert.equal(qualification.safe_resume, false); assert.equal(qualification.limits.max_concurrency, 1); assert.equal(qualification.limits.advertised_max_concurrency, 32); assert.equal(qualification.bindings.request_encoding_sha256, runtime.qwenSha256(runtime.qwenCanonical(runtime.QWEN_REQUEST_ENCODING))); const config = loadConfig(); assert.equal(config.secret, "synthetic-test-secret"); + assert.equal(config.fleetPermit.leaseId, "lease-authority-test", "Worker environment must not choose the authoritative lease"); assert.equal(config.qualification.provenance.authority, "independent-never-builder-review"); assert.throws( () => runtime.loadQwenRuntimeConfig(runtimeEnv(), nowMs), @@ -434,47 +509,48 @@ try { const readyFetch = async (url, init = {}) => { const headers = new Headers(init.headers); observed.push({ url: String(url), authorization: headers.get("authorization") }); - if (String(url).endsWith("/health") && !headers.has("authorization")) return Response.json({}, { status: 401 }); - if (String(url).endsWith("/health")) return Response.json({ status: "ready" }); - if (String(url).endsWith("/v1/models") && !headers.has("authorization")) return Response.json({}, { status: 401 }); + if (String(url).endsWith("/health") && !headers.has("authorization")) return Response.json({}, { status: 401, headers: invocationHeaders(init) }); + if (String(url).endsWith("/health")) return Response.json({ status: "ready" }, { headers: invocationHeaders(init) }); + if (String(url).endsWith("/v1/models") && !headers.has("authorization")) return Response.json({}, { status: 401, headers: invocationHeaders(init) }); if (String(url).endsWith("/v1/models")) { return Response.json({ object: "list", data: [{ id: runtime.QWEN_MODEL, revision, runtime: runtimeTuple, endpoint_identity_sha256: endpointIdentity }], - }); + }, { headers: invocationHeaders(init) }); } throw new Error(`unexpected fake URL ${url}`); }; const lookupPublic = async () => [{ address: "93.184.216.34", family: 4 }]; const readyConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "ready-circuit.json") }); - const identity = await runtime.probeQwenReadiness(readyConfig, { fetchImpl: readyFetch, lookupImpl: lookupPublic, nowMs }); + const identity = await runtime.probeQwenReadiness(readyConfig, { fetchImpl: readyFetch, lookupImpl: lookupPublic, nowMs, reservationAuthority }); assert.equal(identity.endpointIdentity, endpointIdentity); assert.deepEqual(observed.map((entry) => entry.authorization), ["Bearer synthetic-test-secret", "Bearer synthetic-test-secret", null, null]); const canarySse = [ `data: ${JSON.stringify({ - id: "canary-tool", + id: "canary-ready", object: "chat.completion.chunk", created: 1, model: runtime.QWEN_MODEL, - choices: [{ index: 0, delta: { tool_calls: [{ index: 0, function: { name: "report_ready", arguments: '{"status":"ready"}' } }] } }], + choices: [{ index: 0, delta: { content: "READY" }, finish_reason: null }], })}`, - `data: ${JSON.stringify({ id: "canary-tool", object: "chat.completion.chunk", created: 2, model: runtime.QWEN_MODEL, choices: [], usage })}`, + `data: ${JSON.stringify({ id: "canary-ready", object: "chat.completion.chunk", created: 2, model: runtime.QWEN_MODEL, choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })}`, + `data: ${JSON.stringify({ id: "canary-ready", object: "chat.completion.chunk", created: 3, model: runtime.QWEN_MODEL, choices: [], usage })}`, "data: [DONE]", "", ].join("\n\n"); const canaryFetch = async (url, init = {}) => { const target = String(url); const authenticated = new Headers(init.headers).has("authorization"); - if (!authenticated && (target.endsWith("/health") || target.endsWith("/v1/models"))) return Response.json({}, { status: 401 }); - if (target.endsWith("/health")) return Response.json({ status: "ready" }); + if (!authenticated && (target.endsWith("/health") || target.endsWith("/v1/models"))) return Response.json({}, { status: 401, headers: invocationHeaders(init) }); + if (target.endsWith("/health")) return Response.json({ status: "ready" }, { headers: invocationHeaders(init) }); if (target.endsWith("/v1/models")) { return Response.json({ object: "list", data: [{ id: runtime.QWEN_MODEL, revision, runtime: runtimeTuple, endpoint_identity_sha256: endpointIdentity }], - }); + }, { headers: invocationHeaders(init) }); } - if (target.endsWith("/v1/chat/completions")) return new Response(canarySse, { headers: identityHeaders() }); + if (target.endsWith("/v1/chat/completions")) return new Response(canarySse, { headers: identityHeaders({}, init) }); throw new Error(`unexpected canary URL ${target}`); }; const canaryReport = await runtime.runQwenCanary( @@ -482,9 +558,31 @@ try { VINCI_QWEN_BASE_URL: requestEnv.VINCI_QWEN_BASE_URL, VINCI_QWEN_SECRET_REF: `file:${secretFile}`, VINCI_QWEN_CANARY_TIMEOUT_MS: "1000", + VINCI_QWEN_CANARY_WORK_ORDER_ID: "qwen-canary", + VINCI_QWEN_CANARY_RUN_ID: "canary-run", + VINCI_QWEN_CANARY_ATTEMPT_ID: "canary-attempt/1", + VINCI_QWEN_SERVED_REVISION: revision, + VINCI_QWEN_RUNTIME_ENGINE: runtimeTuple.engine, + VINCI_QWEN_RUNTIME_VERSION: runtimeTuple.version, + VINCI_QWEN_RUNTIME_ARTIFACT_SHA256: runtimeTuple.artifact_sha256, + VINCI_QWEN_RUNTIME_ARGUMENTS_SHA256: runtimeTuple.arguments_sha256, + VINCI_QWEN_ENDPOINT_IDENTITY_SHA256: endpointIdentity, }, canaryFetch, lookupPublic, + { + ...authorityBoundary, + fleetPermit: { + ...authorityBoundary.fleetPermit, + workOrderId: "qwen-canary", + runId: "canary-run", + sessionId: "canary-run", + attemptId: "canary-attempt/1", + issuedAt: new Date(Date.now() - 1_000).toISOString(), + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }, + }, + reservationAuthority, ); assert.equal(canaryReport.safe_resume, false); assert.equal(canaryReport.endpoint_identity_sha256, endpointIdentity); @@ -493,6 +591,119 @@ try { const privateConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "private-circuit.json") }); await assert.rejects(runtime.pinQwenEndpoint(privateConfig, async () => [{ address: "169.254.169.254", family: 4 }]), /ssrf_forbidden/); + const unavailableDispatcherConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "dispatcher-unavailable.json") }); + unavailableDispatcherConfig.endpointAddresses = ["93.184.216.34"]; + let unauthorizedTransportCalls = 0; + await assert.rejects( + runtime.createQwenInferenceFetch(unavailableDispatcherConfig, "request-no-dispatcher", () => {}, async () => { + unauthorizedTransportCalls += 1; + throw new Error("must not dispatch"); + })(unavailableDispatcherConfig.chatUrl, { method: "POST", body: requestBody }), + /dispatcher_unavailable/, + ); + assert.equal(unauthorizedTransportCalls, 0, "missing separate-UID dispatcher authority must fail before transport"); + assert.equal(existsSync(join(permitLockDirectory, "qwen-h200-concurrency-1")), false, "an undispatched reservation refusal must not retain occupancy"); + + const forgedReservationConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "forged-reservation.json") }); + forgedReservationConfig.endpointAddresses = ["93.184.216.34"]; + await assert.rejects( + runtime.createQwenInferenceFetch( + forgedReservationConfig, + "request-forged-reservation", + () => {}, + async () => { throw new Error("must not dispatch"); }, + undefined, + (request) => reservationAuthority({ ...request, lease_id: "forged-lease" }), + )(forgedReservationConfig.chatUrl, { method: "POST", body: requestBody }), + /reservation_invalid/, + ); + assert.equal(existsSync(join(permitLockDirectory, "qwen-h200-concurrency-1")), false); + + const restartConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "restart-reconcile.json") }); + restartConfig.endpointAddresses = ["93.184.216.34"]; + restartConfig.qualification.limits.max_retries = 0; + let lostConnectionCalls = 0; + await assert.rejects( + inferenceFetch(restartConfig, "request-connection-loss", () => {}, async () => { + lostConnectionCalls += 1; + throw new Error("connection lost after dispatch"); + })(restartConfig.chatUrl, { method: "POST", body: requestBody }), + /connection lost/, + ); + assert.equal(lostConnectionCalls, 1); + const activeAfterLoss = JSON.parse(readFileSync(join(permitLockDirectory, "qwen-h200-concurrency-1"), "utf8")); + const lossLedger = JSON.parse(readFileSync(activeAfterLoss.ledger_path, "utf8")); + assert.equal(lossLedger.state, "DISPATCHED"); + assert.deepEqual(lossLedger.history.map((event) => event.state), ["PREPARED", "DISPATCHED"]); + assert.equal(lossLedger.invocation_id, reservationRequests.at(-1).invocation_id); + assert.equal(lossLedger.request_sha256, runtime.qwenSha256(requestBody)); + assert.equal(lossLedger.lease_id, authorityBoundary.fleetPermit.leaseId); + assert.equal(lossLedger.fencing_generation, authorityBoundary.fleetPermit.fencingGeneration); + assert.equal(lossLedger.contract_digest, authorityBoundary.fleetPermit.contractDigest); + + let restartTransportCalls = 0; + await assert.rejects( + inferenceFetch(restartConfig, "request-after-restart", () => {}, async () => { + restartTransportCalls += 1; + return new Response(validSse); + })(restartConfig.chatUrl, { method: "POST", body: requestBody }), + /invocation_unresolved/, + ); + assert.equal(restartTransportCalls, 0, "restart must retain ambiguous occupancy before any new dispatch"); + const validReconciliation = { + schema: "vinci.qwen-invocation-reconciliation.v1", + authority: "vgc-fleet-permit-authority", + invocation_id: lossLedger.invocation_id, + reservation_id: lossLedger.reservation_id, + request_sha256: lossLedger.request_sha256, + permit_id: lossLedger.permit_id, + work_order_id: lossLedger.work_order_id, + run_id: lossLedger.run_id, + attempt_id: lossLedger.attempt_id, + lease_id: lossLedger.lease_id, + fencing_generation: lossLedger.fencing_generation, + session_id: lossLedger.session_id, + worker_principal: lossLedger.worker_principal, + worker_build_sha256: lossLedger.worker_build_sha256, + contract_digest: lossLedger.contract_digest, + execution_spec_digest: lossLedger.execution_spec_digest, + endpoint_sha256: lossLedger.endpoint_sha256, + endpoint_identity_sha256: lossLedger.endpoint_identity_sha256, + deployment_revision: lossLedger.deployment_revision, + status: 599, + response_id: null, + usage: null, + }; + assert.throws( + () => runtime.reconcileQwenInvocationFromAuthority(restartConfig, validReconciliation), + /authority_forbidden/, + "Worker memory must not self-authorize reconciliation", + ); + process.env.VINCI_QWEN_TEST_RECONCILIATION = "1"; + try { + assert.throws( + () => runtime.reconcileQwenInvocationFromAuthority(restartConfig, { ...validReconciliation, status: "599" }), + /invocation_reconciliation_invalid/, + ); + } finally { + delete process.env.VINCI_QWEN_TEST_RECONCILIATION; + } + assert.equal(existsSync(join(permitLockDirectory, "qwen-h200-concurrency-1")), true, "invalid reconciliation must retain occupancy"); + reconcileActive(restartConfig); + + const recoveryResponse = await inferenceFetch( + restartConfig, + "request-after-trusted-reconciliation", + () => {}, + async (_url, init = {}) => new Response(validSse, { headers: identityHeaders({}, init) }), + )(restartConfig.chatUrl, { method: "POST", body: requestBody }); + assert.equal(await recoveryResponse.text(), validSse); + const recoveredInvocationId = reservationRequests.at(-1).invocation_id; + const recoveryLedger = JSON.parse(readFileSync(join(permitLockDirectory, `qwen-invocation-${recoveredInvocationId}.json`), "utf8")); + assert.equal(recoveryLedger.state, "RECONCILED"); + assert.deepEqual(recoveryLedger.history.map((event) => event.state), ["PREPARED", "DISPATCHED", "RESPONSE_OBSERVED", "RECONCILED"]); + assert.equal(existsSync(join(permitLockDirectory, "qwen-h200-concurrency-1")), false); + const records = []; const breakerConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "breaker.json") }); breakerConfig.endpointAddresses = ["93.184.216.34"]; @@ -500,7 +711,7 @@ try { const retryKeys = []; const requestDigests = []; const fleetPermitIds = []; - const failingTransport = runtime.createQwenInferenceFetch( + const failingTransport = inferenceFetch( breakerConfig, "request-500", (record) => records.push(record), @@ -510,7 +721,7 @@ try { retryKeys.push(headers.get("x-vinci-idempotency-key")); requestDigests.push(headers.get("x-vinci-qwen-request-sha256")); fleetPermitIds.push(headers.get("x-vinci-qwen-fleet-permit-id")); - return new Response("failure", { status: 500 }); + return new Response("failure", { status: 500, headers: invocationHeaders(init) }); }, ); await assert.rejects( @@ -519,7 +730,9 @@ try { ); assert.equal(failedCalls, 2, "threshold two must count both real HTTP 500 responses"); assert.equal(records.length, 2); - assert.deepEqual(retryKeys, ["request-500/0", "request-500/1"]); + assert.ok(retryKeys.every((key) => typeof key === "string" && /^[0-9a-f]{64}$/.test(key))); + assert.equal(new Set(retryKeys).size, 2, "each retry must use its durable invocation id as the idempotency key"); + assert.deepEqual(retryKeys, records.map((record) => record.invocation_id)); assert.deepEqual(requestDigests, [runtime.qwenSha256(requestBody), runtime.qwenSha256(requestBody)]); assert.deepEqual(fleetPermitIds, [authorityBoundary.fleetPermit.permitId, authorityBoundary.fleetPermit.permitId]); assert.deepEqual(records.map((record) => record.transport_attempt), [0, 1]); @@ -535,46 +748,48 @@ try { const mismatchConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "mismatch.json") }); mismatchConfig.endpointAddresses = ["93.184.216.34"]; - const mismatchTransport = runtime.createQwenInferenceFetch( + const mismatchTransport = inferenceFetch( mismatchConfig, "request-mismatch", () => {}, - async () => new Response(validSse, { headers: identityHeaders({ "x-vinci-model-id": "Qwen/wrong" }) }), + async (_url, init = {}) => new Response(validSse, { headers: identityHeaders({ "x-vinci-model-id": "Qwen/wrong" }, init) }), ); await assert.rejects(mismatchTransport(mismatchConfig.chatUrl, { method: "POST", body: requestBody }), /response_identity_mismatch/); + reconcileActive(mismatchConfig); const runtimeMismatchRecords = []; const runtimeMismatchConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "runtime-mismatch.json") }); runtimeMismatchConfig.endpointAddresses = ["93.184.216.34"]; await assert.rejects( - runtime.createQwenInferenceFetch( + inferenceFetch( runtimeMismatchConfig, "request-runtime-mismatch", (record) => runtimeMismatchRecords.push(record), - async () => new Response(validSse, { headers: identityHeaders({ "x-vinci-runtime-version": "wrong" }) }), + async (_url, init = {}) => new Response(validSse, { headers: identityHeaders({ "x-vinci-runtime-version": "wrong" }, init) }), )(runtimeMismatchConfig.chatUrl, { method: "POST", body: requestBody }), /response_identity_mismatch/, ); assert.equal(runtimeMismatchRecords[0].outcome, "response_identity_mismatch"); + reconcileActive(runtimeMismatchConfig); const redirectConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "redirect.json") }); redirectConfig.endpointAddresses = ["93.184.216.34"]; await assert.rejects( - runtime.createQwenInferenceFetch(redirectConfig, "request-redirect", () => {}, async () => new Response("", { status: 302 }))( + inferenceFetch(redirectConfig, "request-redirect", () => {}, async (_url, init = {}) => new Response("", { status: 302, headers: invocationHeaders(init) }))( redirectConfig.chatUrl, { method: "POST", body: requestBody }, ), /redirect_forbidden/, ); await assert.rejects( - runtime.createQwenInferenceFetch(redirectConfig, "request-ssrf", () => {}, async () => new Response(validSse, { headers: identityHeaders() }))( + inferenceFetch(redirectConfig, "request-ssrf", () => {}, async (_url, init = {}) => new Response(validSse, { headers: identityHeaders({}, init) }))( "http://169.254.169.254/latest/meta-data", { method: "POST", body: requestBody }, ), /ssrf_forbidden/, ); await assert.rejects( - runtime.createQwenInferenceFetch(redirectConfig, "request-too-large", () => {}, async () => new Response(validSse, { headers: identityHeaders() }))( + inferenceFetch(redirectConfig, "request-too-large", () => {}, async (_url, init = {}) => new Response(validSse, { headers: identityHeaders({}, init) }))( redirectConfig.chatUrl, { method: "POST", body: "x".repeat(5_000) }, ), @@ -582,7 +797,7 @@ try { ); const mutatedBody = JSON.stringify({ ...JSON.parse(requestBody), messages: [{ role: "system", content: systemPrompt }, { role: "user", content: "post-hook mutation" }] }); await assert.rejects( - runtime.createQwenInferenceFetch(redirectConfig, "request-mutated", () => {}, async () => new Response(validSse, { headers: identityHeaders() }))( + inferenceFetch(redirectConfig, "request-mutated", () => {}, async (_url, init = {}) => new Response(validSse, { headers: identityHeaders({}, init) }))( redirectConfig.chatUrl, { method: "POST", body: mutatedBody }, ), @@ -596,10 +811,10 @@ try { const retryAbort = new AbortController(); let cancelledCalls = 0; await assert.rejects( - runtime.createQwenInferenceFetch(cancelledConfig, "request-cancelled", () => {}, async () => { + inferenceFetch(cancelledConfig, "request-cancelled", () => {}, async (_url, init = {}) => { cancelledCalls += 1; queueMicrotask(() => retryAbort.abort("operator_stop")); - return new Response("retry", { status: 429, headers: { "retry-after": "1" } }); + return new Response("retry", { status: 429, headers: invocationHeaders(init, { "retry-after": "1" }) }); })(cancelledConfig.chatUrl, { method: "POST", body: requestBody, signal: retryAbort.signal }), /cancelled/, ); @@ -608,23 +823,24 @@ try { const oversizedConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "oversized.json") }); oversizedConfig.endpointAddresses = ["93.184.216.34"]; await assert.rejects( - runtime.createQwenInferenceFetch(oversizedConfig, "request-oversized", () => {}, async () => new Response("x".repeat(300), { status: 500 }))( + inferenceFetch(oversizedConfig, "request-oversized", () => {}, async (_url, init = {}) => new Response("x".repeat(300), { status: 500, headers: invocationHeaders(init) }))( oversizedConfig.chatUrl, { method: "POST", body: requestBody }, ), /response_oversized/, ); + reconcileActive(oversizedConfig); const successRecords = []; const successConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "success.json") }); successConfig.endpointAddresses = ["93.184.216.34"]; const successSettlement = { transportFailed: false, settled: false }; const recordSuccess = (record) => successRecords.push(record); - const successResponse = await runtime.createQwenInferenceFetch( + const successResponse = await inferenceFetch( successConfig, "request-success", recordSuccess, - async () => new Response(validSse, { headers: identityHeaders() }), + async (_url, init = {}) => new Response(validSse, { headers: identityHeaders({}, init) }), successSettlement, )(successConfig.chatUrl, { method: "POST", body: requestBody }); assert.equal(await successResponse.text(), validSse); @@ -645,15 +861,16 @@ try { const conflictingIdRecords = []; const conflictingIdConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "conflicting-id.json"), VINCI_QWEN_CIRCUIT_THRESHOLD: "1" }); conflictingIdConfig.endpointAddresses = ["93.184.216.34"]; - const conflictingIdResponse = await runtime.createQwenInferenceFetch( + const conflictingIdResponse = await inferenceFetch( conflictingIdConfig, "request-conflicting-id", (record) => conflictingIdRecords.push(record), - async () => new Response(conflictingIdSse, { headers: identityHeaders() }), + async (_url, init = {}) => new Response(conflictingIdSse, { headers: identityHeaders({}, init) }), )(conflictingIdConfig.chatUrl, { method: "POST", body: requestBody }); await assert.rejects(conflictingIdResponse.text(), /response id/); assert.equal(conflictingIdRecords[0].outcome, "response_identity_mismatch"); assert.throws(() => runtime.assertQwenCircuitClosed(conflictingIdConfig), /qwen_circuit_open/); + reconcileActive(conflictingIdConfig); const invalidUsage = { ...usage, total_tokens: 99 }; const invalidUsageSse = [ @@ -663,27 +880,29 @@ try { ].join("\n\n"); const invalidUsageConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "invalid-usage.json") }); invalidUsageConfig.endpointAddresses = ["93.184.216.34"]; - const invalidUsageResponse = await runtime.createQwenInferenceFetch( + const invalidUsageResponse = await inferenceFetch( invalidUsageConfig, "request-invalid-usage", () => {}, - async () => new Response(invalidUsageSse, { headers: identityHeaders() }), + async (_url, init = {}) => new Response(invalidUsageSse, { headers: identityHeaders({}, init) }), )(invalidUsageConfig.chatUrl, { method: "POST", body: requestBody }); await assert.rejects(invalidUsageResponse.text(), /usage_invalid/); + reconcileActive(invalidUsageConfig); const oversizedSuccessConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "oversized-success.json") }); oversizedSuccessConfig.endpointAddresses = ["93.184.216.34"]; - const oversizedSuccessResponse = await runtime.createQwenInferenceFetch( + const oversizedSuccessResponse = await inferenceFetch( oversizedSuccessConfig, "request-oversized-success", () => {}, - async () => new Response(`data: ${"x".repeat(5_000)}\n`, { headers: identityHeaders() }), + async (_url, init = {}) => new Response(`data: ${"x".repeat(5_000)}\n`, { headers: identityHeaders({}, init) }), )(oversizedSuccessConfig.chatUrl, { method: "POST", body: requestBody }); await assert.rejects(oversizedSuccessResponse.text(), /response_oversized/); + reconcileActive(oversizedSuccessConfig); const timeoutConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "timeout.json") }); timeoutConfig.endpointAddresses = ["93.184.216.34"]; - const timeoutResponse = await runtime.createQwenInferenceFetch( + const timeoutResponse = await inferenceFetch( timeoutConfig, "request-timeout", () => {}, @@ -691,9 +910,11 @@ try { start(controller) { init.signal.addEventListener("abort", () => controller.error(new DOMException("aborted", "AbortError")), { once: true }); }, - }), { headers: identityHeaders() }), + }), { headers: identityHeaders({}, init) }), )(timeoutConfig.chatUrl, { method: "POST", body: requestBody }); await assert.rejects(timeoutResponse.text(), /AbortError|aborted/); + const timedOutLedger = reconcileActive(timeoutConfig); + assert.equal(timedOutLedger.state, "RESPONSE_OBSERVED"); const context = { systemPrompt, messages: [{ role: "user", content: workOrderPrompt, timestamp: Date.now() }], tools }; const parserValidSse = [ @@ -704,27 +925,43 @@ try { "", ].join("\n\n"); let parserTransportCalls = 0; - const parserTransport = async () => { + const parserTransport = async (_url, init = {}) => { parserTransportCalls += 1; - return new Response(parserValidSse, { headers: identityHeaders() }); + return new Response(parserValidSse, { headers: identityHeaders({}, init) }); }; const permitConfig = freshenPermit(loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, "permit.json") })); permitConfig.endpointAddresses = ["93.184.216.34"]; - assert.throws( - () => runtime.acquireQwenFleetPermit({ ...permitConfig, fleetPermit: { ...permitConfig.fleetPermit, expiresAt: new Date(Date.now() - 1).toISOString() } }), + await assert.rejects( + inferenceFetch( + { ...permitConfig, fleetPermit: { ...permitConfig.fleetPermit, expiresAt: new Date(Date.now() - 1).toISOString() } }, + "expired-permit", + () => {}, + parserTransport, + )(permitConfig.chatUrl, { method: "POST", body: requestBody }), /fleet_permit_expired/, ); - await exerciseCrossProcessFleetPermit(permitConfig); const semanticRecords = []; - const provider = qwenProviderConfig(permitConfig, sourceStreamSimpleOpenAICompletions, (record) => semanticRecords.push(record), parserTransport); + let unblockFirstTransport; + let firstTransportEntered; + let gatedTransportCalls = 0; + const firstTransportReady = new Promise((resolveReady) => { firstTransportEntered = resolveReady; }); + const gatedTransport = async (_url, init = {}) => { + gatedTransportCalls += 1; + if (gatedTransportCalls === 1) { + firstTransportEntered(); + await new Promise((resolveTransport) => { unblockFirstTransport = resolveTransport; }); + } + return new Response(parserValidSse, { headers: identityHeaders({}, init) }); + }; + const provider = qwenProviderConfig(permitConfig, sourceStreamSimpleOpenAICompletions, (record) => semanticRecords.push(record), gatedTransport); const model = { ...provider.models[0], provider: runtime.QWEN_PROVIDER, api: runtime.QWEN_API, baseUrl: provider.baseUrl }; const firstPermitStream = provider.streamSimple(model, context); + await firstTransportReady; const competingProvider = qwenProviderConfig(permitConfig, sourceStreamSimpleOpenAICompletions, () => {}, parserTransport); - assert.throws( - () => competingProvider.streamSimple(model, context), - /concurrency_exceeded/, - "two provider instances sharing an external permit must not run concurrently", - ); + const competing = await competingProvider.streamSimple(model, context).result(); + assert.equal(competing.stopReason, "error"); + assert.equal(parserTransportCalls, 0, "endpoint-global occupancy must block before a competing dispatch reaches transport"); + unblockFirstTransport(); let secondPermitStream; let firstPermitError; for await (const event of firstPermitStream) { @@ -753,7 +990,7 @@ try { truncatedConfig, sourceStreamSimpleOpenAICompletions, (record) => truncatedRecords.push(record), - async () => new Response(missingFinishSse, { headers: identityHeaders() }), + async (_url, init = {}) => new Response(missingFinishSse, { headers: identityHeaders({}, init) }), ); const truncated = await truncatedProvider.streamSimple({ ...truncatedProvider.models[0], provider: runtime.QWEN_PROVIDER, api: runtime.QWEN_API, baseUrl: truncatedProvider.baseUrl }, context).result(); assert.equal(truncated.stopReason, "error"); @@ -775,7 +1012,7 @@ try { toolConfig, sourceStreamSimpleOpenAICompletions, (record) => toolRecords.push(record), - async () => new Response(unknownToolSse, { headers: identityHeaders() }), + async (_url, init = {}) => new Response(unknownToolSse, { headers: identityHeaders({}, init) }), ); const badTool = await toolProvider.streamSimple({ ...toolProvider.models[0], provider: runtime.QWEN_PROVIDER, api: runtime.QWEN_API, baseUrl: toolProvider.baseUrl }, context).result(); assert.equal(badTool.stopReason, "error"); @@ -797,7 +1034,7 @@ try { argumentConfig, sourceStreamSimpleOpenAICompletions, (record) => argumentRecords.push(record), - async () => new Response(invalidArgumentsSse, { headers: identityHeaders() }), + async (_url, init = {}) => new Response(invalidArgumentsSse, { headers: identityHeaders({}, init) }), ); const badArguments = await argumentProvider.streamSimple( { ...argumentProvider.models[0], provider: runtime.QWEN_PROVIDER, api: runtime.QWEN_API, baseUrl: argumentProvider.baseUrl }, diff --git a/vinci/worker/README.md b/vinci/worker/README.md index c289fcdc2..ec243cca5 100644 --- a/vinci/worker/README.md +++ b/vinci/worker/README.md @@ -298,9 +298,9 @@ id (`auto` is deliberately never a class — a contract names a class, not "what resolves to"). A spec-level `provider` pin must EQUAL the configured provider for the class (`provider_mismatch` otherwise); it never overrides it. -### Qualified direct Qwen H200 lane +### Qwen H200 lane (NO-GO pending upstream authority) -`qwen-h200` is an internal OpenAI-compatible provider for the exact model +`qwen-h200` is a local candidate client for the exact model `Qwen/Qwen3.8-27B`. It is not an inference service and never manages a GPU: the operator supplies Ayush's already-running endpoint. A digest-form class entry is: @@ -308,21 +308,27 @@ Ayush's already-running endpoint. A digest-form class entry is: {"qwen-38-27b":{"provider":"qwen-h200","model":"Qwen/Qwen3.8-27B"}} ``` -Enable it only in the Worker process that should admit the lane by including `qwen-h200` in +Configuration does not make this lane admissible. If the upstream dependencies are later supplied, +select it only in the Worker process intended for the lane by including `qwen-h200` in `VINCI_WORKER_ALLOWED_PROVIDERS` and the entry above in `VINCI_WORKER_MODEL_CLASSES`. Qwen is digest-WorkOrder-only; legacy prose handoffs are refused because they cannot carry the validated acceptance criteria and immutable contract binding required by this lane. +The current authority ruling is **NO-GO for joined WorkOrders**. Production has no integrated +separate-UID, credential-owning dispatcher and no independently deployed endpoint-global VGC +reservation service. The client therefore has no production reservation authority and fails before +transport. The process-local test adapter is adversarial-test scaffolding only; it cannot admit a +real request, activate the lane, or count as canary/burn-in evidence. + The lane is fail-closed. Runtime registration requires all of the following: - `VINCI_QWEN_BASE_URL`: the operator HTTPS endpoint. Credentials, query strings, fragments, redirects, IPv6, and private/local/reserved DNS answers are refused; loopback HTTP exists only for tests. Every connection uses the public IPv4 addresses resolved and pinned during readiness. -- `VINCI_QWEN_SECRET_REF=file:/absolute/private/path`: the worker opens a private, regular, - non-symlinked credential file and passes only inherited descriptor 3 to the child. The reference - is removed before spawn, the child consumes and closes the descriptor during provider bootstrap, - and neither the secret nor its reference is put in general child environment, argv, logs, or a - generated file. The WorkOrder prompt is sent on stdin, never argv. +- `VINCI_QWEN_SECRET_REF=file:/absolute/private/path` is retained only for local candidate tests. + It is not an acceptable production credential boundary: a joined WorkOrder must use the + separate-UID dispatcher, which owns the endpoint credential and never exposes it to the Worker or + model process. The WorkOrder prompt is sent on stdin, never argv. - `VINCI_QWEN_QUALIFICATION_FILE` plus its exact `VINCI_QWEN_QUALIFICATION_SHA256`. Trust never comes from Worker environment: a root-owned, non-writable record under `/run/vinci/qwen-authority` pins the independent issuer, Ed25519 key path and digest, and a live @@ -330,8 +336,11 @@ The lane is fail-closed. Runtime registration requires all of the following: that authority root. An unsigned record, a Worker-selected key, or a self-admitted record is invalid. - a deterministic Governor lease and worker-derived WorkOrder, Run, and Attempt identities. Model - output supplies none of them. Each Worker attempt has its own session; every bounded transport - retry is recorded beneath that Attempt with a distinct idempotency key. + output supplies none of them. Before every endpoint request, the dispatcher must return a fresh, + server-minted, endpoint-global reservation binding the exact invocation/request digest, permit, + WorkOrder, Run, Attempt, lease id, fencing generation, session, worker principal/build, contract, + execution spec, endpoint identity/digest, and deployment revision. Missing, stale, extra, or + mismatched reservation fields fail before dispatch. The signed v2 envelope has a maximum seven-day lifetime, cites a canary no more than 24 hours old, and binds the independent review message/body digest and burn-in report digest. Its closed payload @@ -355,11 +364,12 @@ schema; they count toward the atomically persisted circuit just like HTTP and tr failures. Three failures open it for 60 seconds by default. `VINCI_QWEN_CIRCUIT_THRESHOLD` and `VINCI_QWEN_CIRCUIT_OPEN_MS` are bounded operator overrides. -Concurrency is fixed at 1. The external VGC record carries a five-minute-or-shorter permit for the -exact WorkOrder/Run/Attempt and names the shared authority lock directory. Every provider instance -and process must atomically hold the same fleet lock before it can start inference; the local -closure counter is defense in depth. A crash leaves the lock closed until the external authority -reclaims it, rather than permitting overlapping work. The signed schema understands only the ladder +Concurrency is fixed at 1. The filesystem occupancy record and provider closure counter are only +same-host defense in depth; neither is endpoint-global authority. The external VGC service must +atomically reserve the endpoint before generation and bind that reservation to the invocation. The +client durably records `PREPARED → DISPATCHED → RESPONSE_OBSERVED → RECONCILED`; a crash or ambiguous +transport result retains occupancy across restart until a trusted VGC reconciliation proves the +remote outcome. The signed schema understands only the ladder `1 → 2 → 4 → 8 → 16 → 24 → 32`, never an intermediate value, and never above Ayush's advertised ceiling. Any stage above 1 must cite the immediately prior stage with at least 168 continuous hours and 1,000 WorkOrders, 100% acceptance pass and usage coverage, at most 0.5% transport errors, and @@ -376,8 +386,12 @@ verification, review, and no-merge boundaries remain authoritative. There is no switching. OpenRouter fallback means a new, separately authorized attempt whose envelope explicitly selects `openrouter` and whose operator allowlist permits it. -Post-launch canary (readiness/auth GETs plus one bounded inference request; no deployment, GPU, -credential, or remote-state mutation): +Do not run a canary while the dispatcher, endpoint-global reservation service, or independent +reconciliation producer is absent. Once all three are independently deployed and an operator +authorizes it, the only admissible canary is retries-zero and tool-free, uses the same dispatcher and +reservation path as a future WorkOrder, requires the exact text `READY`, strict usage and `[DONE]`, +and remains non-countable evidence. The command below is a future operator procedure, not an +activation instruction: ``` VINCI_QWEN_BASE_URL=https://operator-endpoint.example \ @@ -385,9 +399,7 @@ VINCI_QWEN_SECRET_REF=file:/run/secrets/vinci-qwen-token \ node --experimental-strip-types vinci/extensions/lib/qwen-runtime.ts --canary ``` -The canary requests one streaming `report_ready` tool call, requires exact arguments -`{ "status": "ready" }` plus strict usage, and prints a non-authoritative JSON report. It cannot -write or admit qualification. +The canary prints a non-authoritative JSON report. It cannot write or admit qualification. After reviewing the canary and numeric burn-in report, generate an **unsigned request** for the never-builder reviewer. The command prints JSON only. The full system-prompt, tool-schema, canary, @@ -686,9 +698,10 @@ configured nothing changes (no downgrade), so soak boxes may run without it. - `VINCI_DECLARATION_REFRESH_S`: how often (seconds) a governed daemon re-posts its capability declaration; default `21600` (6h), and anything that is not a positive number falls back to the default. It must stay comfortably below the Governor's `VGC_DECLARATION_MAX_AGE_S` (default 86400), which is when a declaration expires and admission starts answering `eligible: false, reason: stale_declaration`. **The default is chosen against row retention, not liveness** (gpu-control §32): the Governor's `worker_declarations` table is append-only with a DELETE trigger and every refresh writes an audit row, so the volume cannot be pruned later. 6h keeps four refreshes inside the 24h window — three consecutive failed re-posts can be absorbed before one goes stale — at a quarter the rows of hourly, which buys no liveness at all - `GH_TOKEN`: (optional) GitHub machine user token for cloning/pushing private repos and creating PRs - `OPENROUTER_API_KEY`: (or provider-specific key) via vinci's standard configuration -- `VINCI_QWEN_BASE_URL` + `VINCI_QWEN_SECRET_REF=file:/absolute/private/path`: direct Qwen endpoint - and worker-only credential-file reference; the worker converts the reference to child descriptor - 3 and removes it before spawn. Never place the credential value in worker configuration +- `VINCI_QWEN_BASE_URL` + `VINCI_QWEN_SECRET_REF=file:/absolute/private/path`: candidate-test-only + direct endpoint settings. They do not authorize production use. Joined WorkOrders remain refused + until a separate-UID dispatcher owns the endpoint credential and supplies the server-minted + point-of-use reservation and reconciliation contracts described above Never hardcode. Use systemd SecureString parameters, AWS Secrets Manager, or similar. @@ -848,9 +861,11 @@ endpoint/worker-only secret-reference/qualification/circuit settings above — t `vinci/bin/vinci` and the install shim read to find the backend and the run mode. Plus **only** the key the envelope's `provider:` authenticates with: `OPENROUTER_API_KEY` for `openrouter`, `VINCI_API_KEY` for `vinci`, `VINCI_INTERNAL_DEEPINFRA_API_KEY` for `deepinfra` (an unknown provider -gets no key and the launcher refuses it, as today). `qwen-h200` carries no credential value through -the allowlist; immediately before spawn the worker replaces its narrow file reference with inherited -descriptor 3, and the extension consumes that descriptor during bootstrap. Set by the daemon, never copied: `HOME` and +gets no key and the launcher refuses it, as today). The current candidate `qwen-h200` path carries no +credential value through the allowlist; immediately before spawn the worker replaces its narrow file +reference with inherited descriptor 3, and the extension consumes that descriptor during bootstrap. +That is test scaffolding, not the required separate-UID production boundary, so it cannot admit a +joined WorkOrder. Set by the daemon, never copied: `HOME` and `TMPDIR` (per attempt), `VINCI_CODING_AGENT_DIR` and `PI_CODING_AGENT_DIR` (both spellings, both `.home/agent` — the daemon's own slot is **not** passed through: it holds `auth.json` for every provider, every prior session and `bin/`), `VINCI_HOME` (the launcher's install root — the diff --git a/vinci/worker/economics.mjs b/vinci/worker/economics.mjs index 2ad08888a..e25ce5b19 100644 --- a/vinci/worker/economics.mjs +++ b/vinci/worker/economics.mjs @@ -84,6 +84,8 @@ function rollupUsage(entries, flags) { // skipped (calls, tokens and cost), otherwise cost double-counts while calls do not. cost_basis: null, cost_confidence: null, + invocations: [], + invocationIds: new Set(), }; rollup.set(key, group); } @@ -92,6 +94,27 @@ function rollupUsage(entries, flags) { if (seenResponseIds.has(entry.responseId)) continue; seenResponseIds.add(entry.responseId); } + const invocationId = str(entry.invocation_id); + if (entry.invocation_id != null && invocationId === null) flags.malformed = true; + if (invocationId && !group.invocationIds.has(invocationId)) { + const stringFields = [ + "reservation_id", "work_order_id", "run_id", "attempt_id", "lease_id", "session_id", "worker_principal", + "worker_build_sha256", "contract_digest", "execution_spec_digest", + ]; + const invocation = { invocation_id: invocationId }; + for (const field of stringFields) { + const value = str(entry[field]); + if (entry[field] != null && value === null) flags.malformed = true; + if (value !== null) invocation[field] = value; + } + if (Number.isSafeInteger(entry.fencing_generation) && entry.fencing_generation >= 1) { + invocation.fencing_generation = entry.fencing_generation; + } else if (entry.fencing_generation != null) { + flags.malformed = true; + } + group.invocationIds.add(invocationId); + group.invocations.push(invocation); + } if (typeof entry.model_calls === "number" && entry.model_calls > 0) group.model_calls += entry.model_calls; if (typeof entry.input_tokens === "number") group.input_tokens += entry.input_tokens; if (typeof entry.cached_read_tokens === "number") group.cached_read_tokens += entry.cached_read_tokens; @@ -106,7 +129,7 @@ function rollupUsage(entries, flags) { const result = []; for (const group of rollup.values()) { - result.push({ + const row = { phase: group.phase, cost_category: group.cost_category, provider: group.provider, @@ -121,7 +144,9 @@ function rollupUsage(entries, flags) { cost_microusd: group.cost_microusd, cost_basis: group.cost_basis ?? "estimated", cost_confidence: group.cost_confidence ?? "estimated", - }); + }; + if (group.invocations.length > 0) row.invocations = group.invocations; + result.push(row); } return result; } diff --git a/vinci/worker/session-read.mjs b/vinci/worker/session-read.mjs index d8f615166..621e565ac 100644 --- a/vinci/worker/session-read.mjs +++ b/vinci/worker/session-read.mjs @@ -153,10 +153,29 @@ function usageEntryToRecord(entry) { // economics summary can roll up real usage instead of an empty array. function usageEntries(entries) { const result = []; + const qwenAttempts = new Map(); + for (const entry of entries) { + if (entry?.type !== "custom" || entry.customType !== "vinci-qwen-transport-attempt") continue; + const data = entry.data; + if (!data || typeof data !== "object" || data.outcome !== "success" || typeof data.response_id !== "string" || !data.response_id) continue; + qwenAttempts.set(data.response_id, data); + } for (const entry of entries) { if (entry?.type !== "custom" || entry.customType !== "vinci-task-usage") continue; const record = usageEntryToRecord(entry); - if (record) result.push(record); + if (!record) continue; + const responseKey = record.responseId; + const responseId = typeof responseKey === "string" ? responseKey.slice(responseKey.lastIndexOf("\0") + 1) : null; + const attempt = responseId ? qwenAttempts.get(responseId) : null; + if (attempt) { + for (const key of [ + "invocation_id", "reservation_id", "work_order_id", "run_id", "attempt_id", "lease_id", "fencing_generation", + "session_id", "worker_principal", "worker_build_sha256", "contract_digest", "execution_spec_digest", + ]) { + if (attempt[key] !== undefined) record[key] = attempt[key]; + } + } + result.push(record); } return result; } diff --git a/vinci/worker/test/economics-session.test.mjs b/vinci/worker/test/economics-session.test.mjs index 206ca511c..9f894e055 100644 --- a/vinci/worker/test/economics-session.test.mjs +++ b/vinci/worker/test/economics-session.test.mjs @@ -32,6 +32,26 @@ const receiptEntry = { data: { schemaVersion: 1, taskId: SESSION_ID, state: "DONE", changedFiles: ["a.ts"], verificationStatus: "passed", verificationCommand: "npm test", usage: { modelCalls: 3, estimatedCostUsd: 0.03 } }, }; const crewEntry = { type: "custom", customType: "vinci-crew-helper", data: { agentId: "helper-1", task: "x" } }; +const qwenAttemptEntry = { + type: "custom", + customType: "vinci-qwen-transport-attempt", + data: { + outcome: "success", + response_id: "qwen-response-1", + invocation_id: "invocation-1", + reservation_id: "reservation-1", + work_order_id: "wo-1", + run_id: "run-1", + attempt_id: "task-1/1", + lease_id: "lease-1", + fencing_generation: 7, + session_id: "run-1", + worker_principal: "worker:test", + worker_build_sha256: "31".repeat(32), + contract_digest: "32".repeat(32), + execution_spec_digest: "33".repeat(32), + }, +}; function withSession(entries, fn) { const dir = mkdtempSync(join(tmpdir(), "econ-session-")); @@ -79,6 +99,35 @@ test("session-read: usage entries are mapped and a duplicated responseKey is one ); }); +test("Qwen transport authority joins its invocation tuple into economics by response id", () => { + withSession( + [ + qwenAttemptEntry, + usageEntry({ id: "qwen-call", responseKey: "qwen-h200\0qwen-response-1", provider: "qwen-h200", model: "Qwen/Qwen3.8-27B", input: 10, output: 2, cost: 0.000004 }), + receiptEntry, + ], + (state) => { + assert.equal(state.usageEntries[0].invocation_id, "invocation-1"); + assert.equal(state.usageEntries[0].reservation_id, "reservation-1"); + const summary = buildEconomicsSummary(base(state)); + assert.deepEqual(summary.usage[0].invocations, [{ + invocation_id: "invocation-1", + reservation_id: "reservation-1", + work_order_id: "wo-1", + run_id: "run-1", + attempt_id: "task-1/1", + lease_id: "lease-1", + session_id: "run-1", + worker_principal: "worker:test", + worker_build_sha256: "31".repeat(32), + contract_digest: "32".repeat(32), + execution_spec_digest: "33".repeat(32), + fencing_generation: 7, + }]); + }, + ); +}); + test("receipt present: verification_state comes from the receipt and killed_before_outcome is absent", () => { withSession([usageEntry({ id: "c1", provider: "anthropic", model: "m-a" }), receiptEntry], (state) => { const summary = buildEconomicsSummary(base(state)); From 2a50640e58870058be3f0fc8df04de50f1855097 Mon Sep 17 00:00:00 2001 From: George Pu <19347973+thegeorgepu@users.noreply.github.com> Date: Sat, 5 Sep 2026 03:29:33 -0400 Subject: [PATCH 7/9] fix(worker): bind Qwen launcher and lease identities --- vinci/test/worker-lease-loop.mjs | 54 +++++++++++++++++++++++ vinci/test/worker-qwen-provider.mjs | 66 ++++++++++++++++++++++++++++- vinci/worker/README.md | 2 +- vinci/worker/lease.mjs | 10 ++--- vinci/worker/run.mjs | 5 +++ 5 files changed, 130 insertions(+), 7 deletions(-) diff --git a/vinci/test/worker-lease-loop.mjs b/vinci/test/worker-lease-loop.mjs index 105c52f70..f4f704e86 100644 --- a/vinci/test/worker-lease-loop.mjs +++ b/vinci/test/worker-lease-loop.mjs @@ -10,6 +10,7 @@ import { tmpdir } from 'node:os'; import { join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { CLAIM_PATHS_DEPLOYED, CLAIM_PATHS_WITH_TTL, FakeGovernor, WorkerTestFixture } from './lib/worker-fixture.mjs'; +import { executionSpecDigest, workOrderDigest } from '../worker/contracts/digest.mjs'; import { CAPABILITY_MATRIX, DECLARATION_REFRESH_DEFAULT_S, GOVERNOR_DECLARATION_MAX_AGE_S, LEASE_TIMEOUT_MS, LeaseClient, REFRESH_HEADROOM_FACTOR, buildDeclaration, declarationDigest, startHeartbeat } from '../worker/lease.mjs'; import { prBodyFooter, publish } from '../worker/publisher.mjs'; @@ -27,6 +28,7 @@ const hasTerminalFailure = (posts) => const ROOT = join(dirname(fileURLToPath(import.meta.url)), '../..'); const TOOLS = join(ROOT, 'vinci/test/fixtures/worker-test-tools'); const WORKER = join(ROOT, 'vinci/worker/worker.mjs'); +const CONTRACT_VECTORS = join(ROOT, 'vinci/test/fixtures/contract-vectors'); let passed = 0; let failed = 0; @@ -158,6 +160,58 @@ if (failed > 0) { process.exit(1); } +await test('digest handoff leases the immutable WorkOrder id, not the bus message id', async () => { + const fixture = new WorkerTestFixture('digest-lease-subject'); + const governor = new FakeGovernor({ mode: 'leased' }); + await governor.start(); + try { + fixture.linkTools(TOOLS); + const workOrder = { + ...JSON.parse(readFileSync(join(CONTRACT_VECTORS, 'work-order-1-minimal/input.json'), 'utf8')), + id: 'wo-lease-subject', + expiresAt: new Date(Date.now() + 7_200_000).toISOString(), + }; + const contractDigest = workOrderDigest(workOrder); + const executionSpec = { + ...JSON.parse(readFileSync(join(CONTRACT_VECTORS, 'execution-spec-1-minimal/input.json'), 'utf8')), + workOrderId: workOrder.id, + workOrderDigest: contractDigest, + targetBranch: 'feat/lease-subject', + requiredCapabilities: [], + resourceBounds: { + budgetMicrousd: 1_000_000, + maxRuntimeS: 60, + deadline: new Date(Date.now() + 3_600_000).toISOString(), + }, + }; + const specDigest = executionSpecDigest(executionSpec); + const messageId = 'bus-message-not-work-order'; + await fixture.startBus([ + handoff(messageId, 'w1', JSON.stringify({ + work_order_id: workOrder.id, + contract_digest: contractDigest, + execution_spec_digest: specDigest, + })), + ], { + [workOrder.id]: { work_order: workOrder, execution_spec: executionSpec }, + }); + + const { code, stderr } = await runWorker(fixture, 'w1', ['--governor', governor.url], { + VINCI_WORKER_ALLOWED_PROVIDERS: 'openrouter,vinci', + }); + assert.equal(code, 0, stderr); + assert.equal(governor.acquires.length, 1); + assert.equal(governor.acquires[0].work_order_id, workOrder.id); + assert.notEqual(governor.acquires[0].work_order_id, messageId); + assert.equal(state(fixture, messageId).state, 'BLOCKED'); + assert.equal(existsSync(join(fixture.tempDir, 'repos')), false, 'a refused lease must stop before clone'); + assert.equal(fixture.getVinciCalls().length, 0, 'a refused lease must stop before provider spawn'); + } finally { + await governor.close(); + await fixture.cleanup(); + } +}); + await test('happy path: acquire before clone, renew at ttl/3, fenced push+PR, release completed', async () => { const fixture = new WorkerTestFixture('lease-happy'); const governor = new FakeGovernor({ claim: CLAIM_PATHS_WITH_TTL, ttlS: 1 }); diff --git a/vinci/test/worker-qwen-provider.mjs b/vinci/test/worker-qwen-provider.mjs index 86600eb38..d7216036b 100644 --- a/vinci/test/worker-qwen-provider.mjs +++ b/vinci/test/worker-qwen-provider.mjs @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import { generateKeyPairSync, sign } from "node:crypto"; +import { getEventListeners } from "node:events"; import { chmodSync, existsSync, mkdirSync, mkdtempSync, openSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; @@ -526,6 +527,65 @@ try { assert.equal(identity.endpointIdentity, endpointIdentity); assert.deepEqual(observed.map((entry) => entry.authorization), ["Bearer synthetic-test-secret", "Bearer synthetic-test-secret", null, null]); + // Readiness must retain its deadline and caller cancellation after response headers. A + // separate watchdog makes removing either control fail deterministically instead of hanging CI. + for (const path of ["/health", "/v1/models"]) { + for (const interruption of ["timeout", "cancel"]) { + const label = `readiness-body-${path.split("/").at(-1)}-${interruption}`; + const boundedConfig = loadConfig({ VINCI_QWEN_CIRCUIT_FILE: join(temp, `${label}.json`) }); + const caller = new AbortController(); + let watchdog; + let cancelTimer; + let bodyRead = false; + let bodyAborted = false; + let requestSignal; + let calls = 0; + const started = Date.now(); + try { + await assert.rejects(runtime.probeQwenReadiness(boundedConfig, { + lookupImpl: lookupPublic, + nowMs, + signal: caller.signal, + reservationAuthority, + fetchImpl: async (url, init = {}) => { + calls += 1; + if (!String(url).endsWith(path)) return readyFetch(url, init); + requestSignal = init.signal; + return new Response(new ReadableStream({ + start(controller) { + init.signal.addEventListener("abort", () => { + bodyAborted = true; + controller.error(new DOMException("body aborted", "AbortError")); + }, { once: true }); + watchdog = setTimeout(() => controller.error(new Error("readiness body watchdog fired")), 3_000); + }, + pull() { + bodyRead = true; + if (interruption === "cancel") cancelTimer = setTimeout(() => caller.abort("operator_stop"), 10); + }, + }), { headers: invocationHeaders(init) }); + }, + }), (error) => error.code === (interruption === "cancel" ? "cancelled" : "request_timeout"), label); + assert.equal(bodyRead, true, `${label}: interruption must occur during body consumption`); + assert.equal(bodyAborted, true, `${label}: transport must receive abort`); + assert.equal(requestSignal.aborted, true); + assert.equal(requestSignal.reason, interruption === "cancel" ? "operator_stop" : "total_timeout", label); + assert.ok(Date.now() - started < (interruption === "cancel" ? 500 : 2_500), `${label}: must finish at the intended interruption`); + assert.equal(calls, path === "/health" ? 1 : 2, `${label}: no subsequent probe or fallback`); + assert.equal(getEventListeners(caller.signal, "abort").length, 0, `${label}: caller listener released`); + const circuit = JSON.parse(readFileSync(boundedConfig.circuitFile, "utf8")); + assert.equal(circuit.failures, 1, `${label}: failed readiness cannot become ready`); + const ledger = reconcileActive(boundedConfig); + assert.equal(ledger.state, "RESPONSE_OBSERVED", `${label}: retain attribution until authority reconciliation`); + assert.equal(ledger.work_order_id, "wo-test"); + assert.equal(ledger.contract_digest, authorityBoundary.fleetPermit.contractDigest); + } finally { + clearTimeout(watchdog); + clearTimeout(cancelTimer); + } + } + } + const canarySse = [ `data: ${JSON.stringify({ id: "canary-ready", @@ -1066,7 +1126,7 @@ let stdin = ""; process.stdin.setEncoding("utf8"); for await (const chunk of process.stdin) stdin += chunk; const secret = readFileSync(3); -writeFileSync(process.env.QWEN_TEST_SPAWN_RECORD, JSON.stringify({ argv: process.argv.slice(2), stdin, qwenEnvKeys: Object.keys(process.env).filter((key) => key.includes("QWEN_SECRET")), clientBuild: process.env.VINCI_QWEN_CLIENT_BUILD_SHA256, secretBytes: fstatSync(3).size, secretReadBytes: secret.length })); +writeFileSync(process.env.QWEN_TEST_SPAWN_RECORD, JSON.stringify({ argv: process.argv.slice(2), stdin, qwenEnvKeys: Object.keys(process.env).filter((key) => key.includes("QWEN_SECRET")), launcherProvider: process.env.VINCI_PROVIDER, launcherModel: process.env.VINCI_MODEL, clientBuild: process.env.VINCI_QWEN_CLIENT_BUILD_SHA256, secretBytes: fstatSync(3).size, secretReadBytes: secret.length })); `, { mode: 0o700 }); chmodSync(fakeVinci, 0o700); const stateDir = join(temp, "run-state"); @@ -1104,6 +1164,8 @@ writeFileSync(process.env.QWEN_TEST_SPAWN_RECORD, JSON.stringify({ argv: process env: { PATH: `${fakeBin}:${originalPath}`, QWEN_TEST_SPAWN_RECORD: spawnRecord, + VINCI_PROVIDER: "openrouter", + VINCI_MODEL: "inherited/wrong-model", VINCI_QWEN_SECRET_REF: `file:${secretFile}`, }, envDelta: {}, @@ -1118,6 +1180,8 @@ writeFileSync(process.env.QWEN_TEST_SPAWN_RECORD, JSON.stringify({ argv: process assert.equal(spawned.stdin, "synthetic prompt must not be argv"); assert.equal(spawned.stdin.includes("synthetic-test-secret"), false); assert.deepEqual(spawned.qwenEnvKeys, ["VINCI_QWEN_SECRET_FD"]); + assert.equal(spawned.launcherProvider, runtime.QWEN_PROVIDER, "the validated envelope must override an inherited launcher provider selector"); + assert.equal(spawned.launcherModel, runtime.QWEN_MODEL, "the validated envelope must override an inherited launcher model selector"); assert.match(spawned.clientBuild, /^[0-9a-f]{64}$/); assert.equal(spawned.clientBuild, expectedClientBuild); assert.notEqual(spawned.clientBuild, runtime.qwenSha256(readFileSync(fakeVinci)), "client build must include executed parser and coding-agent dependencies, not only the launcher"); diff --git a/vinci/worker/README.md b/vinci/worker/README.md index ec243cca5..713a39978 100644 --- a/vinci/worker/README.md +++ b/vinci/worker/README.md @@ -963,7 +963,7 @@ model is a granted lease.** There is no fail-open path. Exactly these rules appl With `--governor` set, every task holds a **work-order lease** (`vinci/worker/lease.mjs`) from before the path claim until its terminal state; without `--governor` none of this runs and the daemon is byte-identical to before. All lease calls go to `/v1/governor/leases…` with `Authorization: Session `, except `check`, which uses the bus token. Every lease request is bounded by a 10 s timeout (`LEASE_TIMEOUT_MS`): a Governor that accepts the connection and never answers is `Governor connection failed: timeout after 10000 ms` — the same class as a refused connection, so a hang can never hold a heartbeat, a fence or a release open. -1. **Acquire (L1)** — BEFORE the path claim and before the clone: `POST /v1/governor/leases` with `work_order_id` (the envelope `ref`, else the task id — `leaseSubject(task)` is the one place to change when digest handoffs carry the real id), `attempt_id` (`/`), `worker_build_digest` (the daemon's commit), `adapter_version` (identity.json) and `capability_declaration_digest`. **Any 2xx** with a string `lease_id`, a `fencing_generation` and a numeric `ttl_s` of at least 1 second is a lease and is recorded on the task (`lease.lease_id`, `fencing_generation`, `expires_at`, `ttl_s`); `ttl_s` below 1 (`0`, `0.3`, negative, a string, missing) ⇒ **BLOCKED** `lease_unavailable: Governor lease invalid: ttl_s=`. `409 {reason:"leased"}` — or the older server's `403 {reason:"leased_by_other_attempt"}`, read as the same decision ⇒ **BLOCKED** `leased_by until ` (`outcome.governor = "leased"`; the task is not ours — not a failure). Anything else ⇒ **BLOCKED** `lease_unavailable: ` (fail closed). +1. **Acquire (L1)** — BEFORE the path claim and before the clone: `POST /v1/governor/leases` with `work_order_id` selected by `leaseSubject(task)` in this order: the validated digest envelope's immutable `work_order_id`, then a prose envelope's `ref`, then the bus task id. A transport message id does not name the WorkOrder held by the Governor session, so using it for a digest task would make an otherwise valid lease request fail closed at the server; both prose fallbacks remain unchanged. The request also carries `attempt_id` (`/`), `worker_build_digest` (the daemon's commit), `adapter_version` (identity.json) and `capability_declaration_digest`. **Any 2xx** with a string `lease_id`, a `fencing_generation` and a numeric `ttl_s` of at least 1 second is a lease and is recorded on the task (`lease.lease_id`, `fencing_generation`, `expires_at`, `ttl_s`); `ttl_s` below 1 (`0`, `0.3`, negative, a string, missing) ⇒ **BLOCKED** `lease_unavailable: Governor lease invalid: ttl_s=`. `409 {reason:"leased"}` — or the older server's `403 {reason:"leased_by_other_attempt"}`, read as the same decision ⇒ **BLOCKED** `leased_by until ` (`outcome.governor = "leased"`; the task is not ours — not a failure). Anything else ⇒ **BLOCKED** `lease_unavailable: ` (fail closed). The two repos deploy independently, so this client assumes neither the Governor's status codes nor its rollout state. Concretely: (a) **any 2xx** with a well-formed body is a granted lease — the server's acquire is moving from `201` to `200` and a client pinned to one of them would refuse a lease it was actually granted, leaving the Governor holding a lease nobody renews or releases; (b) **every `403` or `409` on a lease route is a decision** — final, never retried — carrying the server's reason verbatim, at acquire (`leased` when it names another holder, otherwise `refused`) as well as at renew. The classification is by STATUS, per CONTRACT §29.1; the reason is payload, never a predicate. It used to be a hand-maintained list of reason strings, and that list was wrong twice: first in snake_case the server never emits (inert — nothing matched), then rebuilt from the server source and still only 1-of-15 live, which the #201 integration caught as `403 {"reason":"session does not hold this work order"}` being filed as a transport fault. A list assembled by reading code cannot keep up with a reason set the server owns. Note this is deliberately **not** a blanket 4xx: `408` and `429` are transient and keep their retry; (c) **anything else non-2xx** is a transport/unknown failure and fails closed on its own path (renew retries once, then loss of authority). (d) `fencing_generation` must be **an integer ≥ 1** — no string form. `app.py::_generation_from` requires `type is int and >= 1` and 400s otherwise, so any other value is one this client could never hand back on a renew, release or check. An earlier version accepted a non-empty string, reasoning that refusing one would make a future token change "a total, silent inadmissibility" — but that argument refuted itself: accepting a string does not avoid the failure, it relocates it to mid-run (acquire succeeds, the clone runs, the child spawns, the first renew 400s, is filed `unreachable`, authority is lost and the child is SIGTERMed), which is the exact outcome the *number* half was narrowed to prevent. One rule, one direction: refuse at acquire, where a task is BLOCKED cleanly with the offending value in the reason, before a clone and before any spend. Everything in a lease response is now type-checked hard. A refused or unavailable lease leaves no path claim held (the Governor cannot release one); a path claim refused after the lease was granted releases the lease `blocked`. No git runs before a lease is held. 2. **Heartbeat (L2)** — renew every `ttl_s/3` (unref'd timer) from acquire until release; a renew that serves `ttl_s` below 1 keeps the previous ttl. The first renew refused (`409 stale_generation|expired|revoked`) or unreachable after one retry (a timeout counts as a miss) is **loss of authority**: the child is SIGTERMed (SIGKILL after 10 s, `VINCI_WORKER_LEASE_KILL_GRACE_MS`), the task is **BLOCKED** `lease_lost:`, nothing is published — not even a branch push — and the evidence bundle is still attempted with `authority: "lost"` in result.json (its ledger POST is fenced out). A loss during the clone skips the spawn entirely. diff --git a/vinci/worker/lease.mjs b/vinci/worker/lease.mjs index dee27647d..8c64c8950 100644 --- a/vinci/worker/lease.mjs +++ b/vinci/worker/lease.mjs @@ -43,12 +43,12 @@ export const LEASE_TIMEOUT_MS = 10_000; export const MIN_TTL_S = 1; export const RELEASE_OUTCOMES = Object.freeze(["completed", "failed", "blocked", "unverified", "abandoned"]); -// The lease subject: what the Governor keys the lease on. Today the envelope's `ref` (a ledger -// job/experiment id) when present, else the bus task id. Digest handoffs will pass the real -// work-order id later; the daemon takes this as an injectable function so that change is one -// argument, not a rewrite. +// The lease subject: what the Governor keys the lease on. A validated digest handoff carries the +// immutable WorkOrder id; prose handoffs retain the ledger ref and then bus-task fallback. +// A transport message id does not name the WorkOrder held by the Governor session, so using it +// for a digest task would make the otherwise valid lease request fail closed at the server. export function leaseSubject(task) { - return task?.envelope?.ref ?? task?.id ?? null; + return task?.envelope?.work_order_id ?? task?.envelope?.ref ?? task?.id ?? null; } function validTtl(value) { diff --git a/vinci/worker/run.mjs b/vinci/worker/run.mjs index 5ca2c1764..b9cc3d613 100644 --- a/vinci/worker/run.mjs +++ b/vinci/worker/run.mjs @@ -1296,6 +1296,11 @@ export function runVinci({ envelope, repoDir, stateDir, taskId, sessionId, env, } const taskEnvironment = applyEnvDelta(env ?? process.env, envDelta); taskEnvironment.VINCI_UPDATE_DISABLED = "1"; + // The launcher selects and validates provider-specific extensions from these variables before + // it forwards argv to Pi. The envelope is already validated and is the task's authority, so an + // inherited daemon selector must never choose a different launcher path than --provider/--model. + taskEnvironment.VINCI_PROVIDER = envelope.provider; + taskEnvironment.VINCI_MODEL = envelope.model; let qwenSecretReference; // The direct H200 lane is one exact, pre-qualified provider. These values are derived by the // worker, not accepted from the model or repository, and bind the provider extension to the From 5bf190e237ffcc6e03105959e379243a0770ae91 Mon Sep 17 00:00:00 2001 From: George Pu <19347973+thegeorgepu@users.noreply.github.com> Date: Sat, 5 Sep 2026 03:31:41 -0400 Subject: [PATCH 8/9] test(worker): compose Qwen fail-closed path --- .../test/worker-qwen-refusal-composition.mjs | 185 ++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 vinci/test/worker-qwen-refusal-composition.mjs diff --git a/vinci/test/worker-qwen-refusal-composition.mjs b/vinci/test/worker-qwen-refusal-composition.mjs new file mode 100644 index 000000000..e1fbb1e38 --- /dev/null +++ b/vinci/test/worker-qwen-refusal-composition.mjs @@ -0,0 +1,185 @@ +// Production negative composition only: the real Qwen extension currently refuses before +// readiness/registration. Fixture services and upload transport are not provider evidence. +import assert from "node:assert/strict"; +import { execFileSync, spawn } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, rmSync, unlinkSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { CLAIM_PATHS_WITH_TTL, FakeGovernor, WorkerTestFixture } from "./lib/worker-fixture.mjs"; +import { provisionWorkerDebrisAuthority } from "./lib/worker-debris-authority-fixture.mjs"; +import { executionSpecDigest, workOrderDigest } from "../worker/contracts/digest.mjs"; + +const root = resolve(import.meta.dirname, "../.."); +const launcher = join(root, "vinci/bin/vinci"); +for (const explicitLauncherSelection of [false, true]) { +const fixture = new WorkerTestFixture(`qwen-refusal-composition-${explicitLauncherSelection ? "explicit" : "default"}`); +const governor = new FakeGovernor({ claim: CLAIM_PATHS_WITH_TTL }); +const authority = provisionWorkerDebrisAuthority(fixture.tempDir, "6".repeat(64)); +const taskId = "qwen-refusal"; +const workOrderId = "job_qwen_refusal"; +const records = join(fixture.tempDir, "actual-launches.jsonl"); +const denied = join(fixture.tempDir, "network-denied.jsonl"); +const awsRecords = join(fixture.tempDir, "aws.jsonl"); +const jsonLines = (path) => existsSync(path) ? readFileSync(path, "utf8").trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)) : []; +let bundlePath; +try { + fixture.linkTools(join(root, "vinci/test/fixtures/worker-test-tools")); + fixture.recordGit(); + // Unlink the synthetic producer before writing a recording exec shim. Never write through it. + unlinkSync(join(fixture.toolsDir, "vinci")); + const recorder = join(fixture.tempDir, "record-launch.cjs"); + writeFileSync(recorder, `const fs = require("node:fs"); +fs.appendFileSync(${JSON.stringify(records)}, JSON.stringify({at:Date.now(), argv:process.argv.slice(2), selected:process.env.VINCI_QWEN_SELECTED, workOrder:process.env.VINCI_QWEN_WORK_ORDER_ID, run:process.env.VINCI_QWEN_RUN_ID, attempt:process.env.VINCI_QWEN_ATTEMPT_ID, secretFd:process.env.VINCI_QWEN_SECRET_FD}) + "\\n"); +`); + const quote = (value) => `'${value.replaceAll("'", "'\\''")}'`; + writeFileSync(join(fixture.toolsDir, "vinci"), `#!/bin/bash\n${quote(process.execPath)} ${quote(recorder)} "$@"\nexec /bin/bash ${quote(launcher)} "$@" --offline --no-extensions --no-skills --no-prompt-templates --no-themes\n`, { mode: 0o700 }); + // Node network guard is inherited by daemon, CLI, extensions, and fixture tools. Git is + // separately restricted to the file protocol. No ambient provider credentials are inherited. + const guard = join(fixture.tempDir, "network-guard.cjs"); + writeFileSync(guard, `const fs = require("node:fs"); +const net = require("node:net"); +const dns = require("node:dns"); +const dgram = require("node:dgram"); +const reject = (host) => { fs.appendFileSync(${JSON.stringify(denied)}, JSON.stringify({host:String(host)}) + "\\n"); throw new Error("unexpected network destination: " + host); }; +const local = (host) => ["127.0.0.1", "::1", "localhost"].includes(host); +const connect = net.Socket.prototype.connect; +net.Socket.prototype.connect = function(...args) { + const first = Array.isArray(args[0]) ? args[0][0] : args[0]; + const options = typeof first === "object" ? first : {port:first, host:typeof args[1] === "string" ? args[1] : "localhost"}; + if (!options.path && !local(options.host ?? "localhost")) reject(options.host); + return connect.apply(this,args); +}; +const lookup = dns.lookup; +dns.lookup = function(host,...args) { if (!local(host)) reject(host); return lookup.call(this,host,...args); }; +const promiseLookup = dns.promises.lookup; +dns.promises.lookup = async function(host,...args) { if (!local(host)) reject(host); return promiseLookup.call(this,host,...args); }; +for (const name of ["resolve", "resolve4", "resolve6", "resolveAny", "resolveTxt", "reverse"]) { + dns[name] = (host) => reject(host); + dns.promises[name] = async (host) => reject(host); +} +dgram.createSocket = () => reject("UDP"); +`); + const { origin } = fixture.createRepo("getsimpledirect", "vinci-contracts"); + const baseCommit = execFileSync("git", ["--git-dir", origin, "rev-parse", "main"], { encoding: "utf8" }).trim(); + const vectors = join(root, "vinci/test/fixtures/contract-vectors"); + const goldenOrder = JSON.parse(readFileSync(join(vectors, "work-order-1-minimal/input.json"), "utf8")); + const goldenSpec = JSON.parse(readFileSync(join(vectors, "execution-spec-1-minimal/input.json"), "utf8")); + assert.equal(workOrderDigest(goldenOrder), goldenSpec.workOrderDigest); + const order = { ...goldenOrder, id: workOrderId, expiresAt: new Date(Date.now() + 7_200_000).toISOString() }; + const contractDigest = workOrderDigest(order); + const spec = { + ...goldenSpec, workOrderId, workOrderDigest: contractDigest, baseCommit, + modelClass: "qwen-refusal", requiredCapabilities: [], inputArtifacts: [], output: "none", promotion: "none", + resourceBounds: { ...goldenSpec.resourceBounds, maxRuntimeS: 45, deadline: new Date(Date.now() + 3_600_000).toISOString() }, + }; + const specDigest = executionSpecDigest(spec); + authority.reserveTask(taskId); + await governor.start(); + await fixture.startBus([{ + message_id: taskId, to_agent: "worker:w1", kind: "handoff", subject: "Qwen production refusal", + body: JSON.stringify({ work_order_id: workOrderId, contract_digest: contractDigest, execution_spec_digest: specDigest }), + ts: new Date().toISOString(), posted_by: "fixture:scheduler", + }], { [workOrderId]: { work_order: order, execution_spec: spec } }); + const home = join(fixture.tempDir, "home"); + mkdirSync(home, { recursive: true }); + const secret = join(fixture.tempDir, "synthetic-secret"); + writeFileSync(secret, "synthetic-refusal-test-only\n", { mode: 0o600 }); + const env = { + PATH: `${fixture.toolsDir}:${dirname(process.execPath)}:/usr/bin:/bin:/usr/sbin:/sbin`, HOME: home, + LANG: "C", LC_ALL: "C", NODE_OPTIONS: `--require=${guard}`, + PI_OFFLINE: "1", PI_SKIP_VERSION_CHECK: "1", VINCI_SOURCE_CLI: "1", + VINCI_NO_BOOTSTRAP_HEAL: "1", VINCI_UPDATE_DISABLED: "1", VINCI_NO_RESUME: "1", + VINCI_NO_VERIFY: "1", VINCI_TOOL_BOOTSTRAP: "0", + GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", GIT_ALLOW_PROTOCOL: "file", + VINCI_BUS_TOKEN: "test-token", VINCI_GOVERNOR_TOKEN: "gov-token", + VINCI_WORKER_GIT_BASE: `file://${fixture.reposDir}/`, + VINCI_WORKER_ALLOWED_PROVIDERS: "qwen-h200", + VINCI_WORKER_MODEL_CLASSES: JSON.stringify({ "qwen-refusal": { provider: "qwen-h200", model: "Qwen/Qwen3.8-27B" } }), + VINCI_QWEN_SECRET_REF: `file:${secret}`, + VINCI_EVIDENCE_URI_PREFIX: "s3://fixture-evidence/worker/", FAKE_AWS_RECORD: awsRecords, + FAKE_GH_RECORD: join(fixture.tempDir, "gh-calls.txt"), + }; + // Both an empty environment and stale daemon selectors must use the validated envelope. + // The launcher chooses its extension from env before forwarding the worker's CLI selectors. + if (explicitLauncherSelection) { env.VINCI_PROVIDER = "openrouter"; env.VINCI_MODEL = "inherited/wrong-model"; } + for (const [name, value] of Object.entries(process.env)) if (name.startsWith("VINCI_WORKER_DEBRIS_")) env[name] = value; + env.VINCI_WORKER_DEBRIS_AUTHORITY_CAPABILITY_FD = "3"; + const result = await new Promise((resolveRun, rejectRun) => { + const child = spawn("/bin/bash", [launcher, "worker", "start", "--id", "w1", "--server", fixture.busUrl(), "--governor", governor.url, "--once", "--state-dir", fixture.tempDir], { + env, stdio: ["ignore", "pipe", "pipe", authority.capabilityFd], + }); + authority.releaseCapabilityToChild(); + let output = ""; + child.stdout.on("data", (chunk) => { output += chunk; }); + child.stderr.on("data", (chunk) => { output += chunk; }); + const timer = setTimeout(() => child.kill("SIGKILL"), 60_000); + child.once("error", rejectRun); + child.once("close", (code, signal) => { clearTimeout(timer); resolveRun({ code, signal, output }); }); + }); + const state = JSON.parse(readFileSync(join(fixture.tempDir, "tasks", `${taskId}.json`), "utf8")); + console.log(JSON.stringify({ explicitLauncherSelection, run: result, state, acquires: governor.acquires, releases: governor.releases, evidence: fixture.evidencePosts, posts: fixture.postedMessages, launches: jsonLines(records), denied: jsonLines(denied) }, null, 2)); + assert.equal(result.code, 0, result.output); + assert.equal(result.signal, null); + assert.deepEqual(jsonLines(denied), [], "unexpected network attempt fails the composition"); + assert.match(result.output, /qwen_dispatcher_unavailable/, "both launcher environments must reach the production extension refusal"); + assert.match(result.output, /Unknown provider "qwen-h200"/); + assert.equal(fixture.contractRequests.length, 1); + assert.equal(governor.acquires.length, 1); + assert.equal(governor.claims.length, 1); + assert.equal(governor.releases.length, 1); + assert.equal(governor.releases[0].outcome, "failed"); + assert.equal(governor.holderAttemptId, null); + // The immutable WorkOrder, not its transport message, is the lease subject. + assert.equal(governor.acquires[0].work_order_id, workOrderId); + assert.notEqual(governor.acquires[0].work_order_id, taskId); + assert.equal(governor.acquires[0].attempt_id, `${taskId}/1`); + assert.equal(governor.claims[0].attempt_id, `${taskId}/1`); + const launches = jsonLines(records).filter((record) => record.argv.includes("-p")); + assert.equal(launches.length, 1, "exactly one actual attempt and no fallback"); + assert.equal(launches[0].selected, "1"); + assert.equal(launches[0].workOrder, workOrderId); + assert.equal(launches[0].attempt, `${taskId}/1`); + assert.equal(launches[0].run, `${taskId}-qwen-attempt-1`); + assert.equal(launches[0].secretFd, "3"); + assert(governor.hits.find((hit) => hit.url === "/v1/governor/claim-paths").at <= launches[0].at); + assert.equal(state.state, "FAILED", JSON.stringify(state.outcome)); + assert.equal(state.terminal, true); + assert.equal(state.cost_usd, 0); + assert.equal(state.exit_code, 1); + assert.equal(state.head, baseCommit); + assert.equal(state.outcome.no_commit, true); + assert.match(state.outcome.reason, /^no_commit:/); + const terminals = fixture.postedMessages.filter((post) => post.in_reply_to === taskId && post.outcome); + assert.equal(terminals.length, 1, "one attributable terminal"); + assert.equal(terminals[0].outcome, "FAILED"); + assert.match(terminals[0].body, new RegExp(`contract=${workOrderId}@${contractDigest.slice(0, 8)}`)); + // Real evidence bundle creation runs, but digest handoffs currently omit the ledger ref + // required by uploadEvidence's bus POST. This is a proof gap, not a synthetic accepted POST. + assert.equal(fixture.evidencePosts.length, 0); + const uploads = jsonLines(awsRecords); + assert.equal(uploads.length, 1); + bundlePath = uploads[0].argv[3]; + const bundleResult = JSON.parse(execFileSync("tar", ["xzOf", bundlePath, "./result.json"], { encoding: "utf8" })); + assert.equal(bundleResult.state, state.state); + assert.equal(bundleResult.terminal, false); + assert.equal(bundleResult.contract_digest, contractDigest); + assert.equal(bundleResult.execution_spec_digest, specDigest); + assert.equal(bundleResult.work_order_id, workOrderId); + assert.equal(bundleResult.session_id, launches[0].run); + const economics = JSON.parse(execFileSync("tar", ["xzOf", bundlePath, "./economics-summary.json"], { encoding: "utf8" })); + assert.equal(economics.work_order_id, workOrderId); + assert.equal(economics.attempt_label, `${taskId}/1`); + assert.equal(economics.session_id, launches[0].run); + assert.equal((economics.usage ?? []).length, 0); + assert.equal(economics.route.initial_provider, null); + assert.deepEqual(economics.route.escalations, []); + assert.equal(execFileSync("tar", ["xzOf", bundlePath, "./session.jsonl"], { encoding: "utf8" }), ""); + assert.equal(fixture.gitTransferCalls().filter((args) => args.includes("push")).length, 0); + assert.equal(fixture.getVinciCalls().length, 0, "synthetic terminal producer never ran"); + console.log(`PASS worker-qwen-refusal-composition ${explicitLauncherSelection ? "stale-env overridden" : "default-env selected"}: real dispatcher refusal, correct WorkOrder lease, FAILED terminal and local evidence; no accepted Qwen outcome or evidence POST proved`); +} finally { + authority.cleanup(); + await governor.close(); + await fixture.cleanup(); + if (bundlePath) { rmSync(bundlePath, { force: true }); rmSync(bundlePath.replace(/\.tgz$/, ""), { recursive: true, force: true }); } +} +} From 490eee9ee34bdf58534abf19cdff8a9dc7e80867 Mon Sep 17 00:00:00 2001 From: George Pu <19347973+thegeorgepu@users.noreply.github.com> Date: Sat, 5 Sep 2026 03:34:50 -0400 Subject: [PATCH 9/9] test(worker): decouple Qwen refusal from evidence identity --- vinci/test/worker-qwen-refusal-composition.mjs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/vinci/test/worker-qwen-refusal-composition.mjs b/vinci/test/worker-qwen-refusal-composition.mjs index e1fbb1e38..fe5aab213 100644 --- a/vinci/test/worker-qwen-refusal-composition.mjs +++ b/vinci/test/worker-qwen-refusal-composition.mjs @@ -152,9 +152,6 @@ dgram.createSocket = () => reject("UDP"); assert.equal(terminals.length, 1, "one attributable terminal"); assert.equal(terminals[0].outcome, "FAILED"); assert.match(terminals[0].body, new RegExp(`contract=${workOrderId}@${contractDigest.slice(0, 8)}`)); - // Real evidence bundle creation runs, but digest handoffs currently omit the ledger ref - // required by uploadEvidence's bus POST. This is a proof gap, not a synthetic accepted POST. - assert.equal(fixture.evidencePosts.length, 0); const uploads = jsonLines(awsRecords); assert.equal(uploads.length, 1); bundlePath = uploads[0].argv[3]; @@ -175,7 +172,7 @@ dgram.createSocket = () => reject("UDP"); assert.equal(execFileSync("tar", ["xzOf", bundlePath, "./session.jsonl"], { encoding: "utf8" }), ""); assert.equal(fixture.gitTransferCalls().filter((args) => args.includes("push")).length, 0); assert.equal(fixture.getVinciCalls().length, 0, "synthetic terminal producer never ran"); - console.log(`PASS worker-qwen-refusal-composition ${explicitLauncherSelection ? "stale-env overridden" : "default-env selected"}: real dispatcher refusal, correct WorkOrder lease, FAILED terminal and local evidence; no accepted Qwen outcome or evidence POST proved`); + console.log(`PASS worker-qwen-refusal-composition ${explicitLauncherSelection ? "stale-env overridden" : "default-env selected"}: real dispatcher refusal, correct WorkOrder lease, FAILED terminal and local evidence; no accepted Qwen outcome or provider network attempt`); } finally { authority.cleanup(); await governor.close();