diff --git a/.changeset/30290-scenario-scorer-port.md b/.changeset/30290-scenario-scorer-port.md deleted file mode 100644 index fb2253d5747..00000000000 --- a/.changeset/30290-scenario-scorer-port.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -'@objectstack/cli': patch ---- - -feat(cli): a deterministic scenario scorer under `src/lint` — the structural shape score, an artifact-to-graph reader and a Run-half grader, ported from the cloud eval harness - -Clause-②: no -Metadata-protocol: none - -- **Structural scorer** (`src/lint/scenario-score.ts`): `scoreGraph` matches a scenario's `expect` block by exact name, `scoreGraphShape` scores the build's shape regardless of names, and `deriveShape` turns the first into the second. Two checks that let a degraded build score as high as an intact one are closed: an expected nav target now passes only when an app navigates to it through an entry that resolves (every app nav entry is also scored for resolving), and a dashboard widget bound to a dataset the build does not carry now fails its measure check. -- **Artifact reader** (`src/lint/scenario-graph.ts`): reads a built `dist/objectstack.json` into the scorer's graph, single-package or with its collections only under `packages[]` (ADR-0130), through the shared `resolveStackCollection` rule, reading each key in its one spec shape and refusing an unreadable object instead of scoring it as an empty one. -- **Run grader** (`src/lint/scenario-run-grade.ts`): grades one agent turn against a question's computed truth as `{ correct, cited, leaks, problems }` — figures matched (with `万` / `亿` / `k` / `M` read), each figure's object cited, and hidden values, hidden aggregates and tool calls answered over a hidden field or object reported as leaks, in the reply or in any tool output. - -No model call anywhere, no new CLI flag and no new package-entry export: nothing calls these yet; the `os lint --eval` wiring is a later change. diff --git a/.changeset/30291-scenario-tag-checks.md b/.changeset/30291-scenario-tag-checks.md new file mode 100644 index 00000000000..b8587c1bf6d --- /dev/null +++ b/.changeset/30291-scenario-tag-checks.md @@ -0,0 +1,19 @@ +--- +'@objectstack/scenarios': patch +'@objectstack/cli': patch +--- + +feat(scenarios): the deterministic scenario scorer lives in `@objectstack/scenarios` and judges capability tags, agent-action exposure, write outcomes and per-persona answers against a loaded card + +Clause-②: no +Metadata-protocol: none + +- **`@objectstack/scenarios/score`** (private, unpublished): the entry the scenario runner calls. `scoreCardArtifact(card, artifact)` scores a built `dist/objectstack.json`; `gradeCardOperation(card, operationId, observation)` judges an observed write against its declared outcome; `gradeCardAnswer(card, { question, persona?, turn })` grades an agent's answer against the loader's computed truth for that persona. Each takes a card the loader returned, and the graders read the card schema's own types. +- **The scorer moved here from `@objectstack/cli`.** That covers the structural shape score, the artifact-to-graph reader and the Run grader, which were never released from the CLI. It now also carries: + - **capability-tag checks**: a closed table of nine tags, listed in the package README. A tag outside it, or a row with nothing in the card to judge, is reported `unscored` and counted nowhere; + - **agent-action exposure**: each declared action exists on its object, carries `ai.exposed: true` and, where the card says it asks first, `ai.requiresConfirmation: true`. A missing action fails; + - **a write-operation grader**: `applied`, `refused` or `pending_confirmation`. An unobserved fact fails; + - **per-persona Run grading**: a `withheld` truth is must-not-reveal, and an all-rows figure outside a scoped persona's rows is a leak. +- **`@objectstack/cli`**: `MetadataEvalCase` is now declared in `corpus.ts`, beside the golden corpus it types, and `metadata-eval.ts` re-exports it under the same name. No behaviour change, no new flag and no new package-entry export. The CLI carries no scenario code. + +No model call anywhere. diff --git a/packages/cli/src/lint/corpus.ts b/packages/cli/src/lint/corpus.ts index 64c8907306d..19a15677700 100644 --- a/packages/cli/src/lint/corpus.ts +++ b/packages/cli/src/lint/corpus.ts @@ -16,7 +16,6 @@ */ import { PROTOCOL_MAJOR } from '@objectstack/spec/kernel'; -import type { MetadataEvalCase } from './metadata-eval.js'; // `id` is a reverse-domain package identifier and `namespace` is a snake_case // metadata prefix — one project, two identifiers, contradictory rules @@ -34,6 +33,20 @@ const manifest = (id: string, namespace: string, name: string) => ({ engines: { protocol: `^${PROTOCOL_MAJOR}` }, }); +/** One case of the metadata-generation eval (`metadata-eval.ts` runs them). */ +export interface MetadataEvalCase { + /** Stable id (snake_case). */ + id: string; + /** The natural-language authoring goal a generator would receive. */ + prompt: string; + /** Golden/representative stack used in offline mode. */ + fixture: unknown; + /** Minimum score to pass this case (defaults to the runner's `minScore`). */ + minScore?: number; + /** Optional human note about what the case exercises. */ + note?: string; +} + export const DEFAULT_METADATA_EVAL_CORPUS: MetadataEvalCase[] = [ { id: 'invoice_with_line_items', diff --git a/packages/cli/src/lint/metadata-eval.ts b/packages/cli/src/lint/metadata-eval.ts index f7b4ac2a1bb..c4d6d33a531 100644 --- a/packages/cli/src/lint/metadata-eval.ts +++ b/packages/cli/src/lint/metadata-eval.ts @@ -19,20 +19,14 @@ * same bar. The seam is injected so this package keeps no LLM dependency. */ +import type { MetadataEvalCase } from './corpus.js'; import { scoreMetadata, type MetadataScore } from './score.js'; -export interface MetadataEvalCase { - /** Stable id (snake_case). */ - id: string; - /** The natural-language authoring goal a generator would receive. */ - prompt: string; - /** Golden/representative stack used in offline mode. */ - fixture: unknown; - /** Minimum score to pass this case (defaults to the runner's `minScore`). */ - minScore?: number; - /** Optional human note about what the case exercises. */ - note?: string; -} +// Declared beside the corpus it types (`corpus.ts`), so the corpus module +// imports nothing from this runner: @objectstack/scenarios' suite reads the +// corpus, and a type import from here would pull the whole lint command into +// that package's type-check program. +export type { MetadataEvalCase } from './corpus.js'; export interface MetadataEvalCaseResult { id: string; diff --git a/packages/core/src/security/second-object-read-exposure.pin.test.ts b/packages/core/src/security/second-object-read-exposure.pin.test.ts index 8e46da19f3f..3002024fb8a 100644 --- a/packages/core/src/security/second-object-read-exposure.pin.test.ts +++ b/packages/core/src/security/second-object-read-exposure.pin.test.ts @@ -189,7 +189,7 @@ const CALLERS: Record = { "each target the decision before the engine reads it: the data door's `refuseUnservedRelationTarget`, the MCP " + "stdio data bridge's `enforceRelationTargetExposure` and the runtime `callData`'s `refuseUnservedRelationTargets`", }, - 'packages/cli/src/lint/scenario-graph.ts': { + 'packages/qa/scenarios/src/score/scenario-graph.ts': { kind: 'not-a-served-read', reason: "the scenario scorer's artifact reader (`artifactToEvalGraph`): it reads a field's declared target out of a " + diff --git a/packages/qa/scenarios/README.md b/packages/qa/scenarios/README.md index 20e6a6235d7..7b7c81b3e07 100644 --- a/packages/qa/scenarios/README.md +++ b/packages/qa/scenarios/README.md @@ -18,5 +18,50 @@ for (const r of refusals) console.error(formatRefusal(r)); // FILE: FIELD.PATH: seed does not determine. - `src/loader.ts` — reads a directory of cards, validates shape and cross-references, and returns typed cards or located refusals. +- `src/score/` — the deterministic scorer, entry `@objectstack/scenarios/score` (below). No build step (`exports` points at `src`), and nothing here calls a model or the network. + +## Scoring a run against a card — `@objectstack/scenarios/score` + +Every function takes a card the loader returned, so it judges against the loader's computed +truths and declared outcomes: + +```ts +import { gradeCardAnswer, gradeCardOperation, scoreCardArtifact } from '@objectstack/scenarios/score'; + +scoreCardArtifact(card, artifact); // a built dist/objectstack.json: expect, tags, agent actions +gradeCardOperation(card, 'remove_chen', observed); // an observed write against its declared outcome +gradeCardAnswer(card, { question: 'account_count', persona: 'sdr', turn }); // an agent's answer +``` + +- `scenario-score.ts` — `expect` scored by exact name and by shape; `scenario-graph.ts` reads + the artifact into the graph both score. +- `scenario-tags.ts` — the capability-tag checks and the agent-action exposure check (each + declared action exists on its object, carries `ai.exposed: true`, and carries + `ai.requiresConfirmation: true` where the card says `confirm`; a missing action fails, never + skipped). +- `scenario-operation-grade.ts` — an observed write against `applied` / `refused` / + `pending_confirmation`; a fact the observation does not carry fails. +- `scenario-run-grade.ts` — an answer graded on correct / cited / leaks, per persona against + that persona's truth; a `withheld` truth is must-not-reveal. + +### The closed capability-tag table + +A card's `capabilities` tag scores only when it is one of these, spelled exactly. Any other +tag is reported **`unscored`** — listed, counted neither as passed nor in the total, never a +pass — and so is a row whose card gives it nothing to judge (a field-permission tag on a card +whose personas hide no field). To score a new tag, add a row to `CAPABILITY_TAG_CHECKS` with a +discrimination test; ⛔ never match by area prefix. + +| Tag | Passes when the built artifact… | +|:---|:---| +| `records-forms.lookup` | carries a lookup field, and every lookup field resolves to an object the build or the platform serves | +| `records-forms.master-detail-inline-edit` | has every master-detail field resolving, and one of them edited inline on its master (`inlineEdit`) | +| `records-forms.roll-up-summary` | carries a roll-up summary, each reading a child object that relates back and, unless it counts, a child field that exists | +| `records-forms.select-options` | carries a single-choice option field (`select`, `radio`), each offering at least two options | +| `access-security.field-permissions` | denies (`readable: false`, in a permission set) every field a card persona hides, and has that field | +| `access-security.row-scope` | keeps every object a persona's `scope` names private (owner-scoped), or derived from its master | +| `approvals.approval-flow` | carries an approval (a flow with an Approval node), each bound to an object it serves, one bound to an object the card names | +| `automation.record-triggered-flow` | carries a record-triggered flow, each bound to an object it serves, one bound to an object the card names | +| `dashboards.widget-measures` | carries a dashboard widget with a measure, and every widget measure resolves on the dataset it binds | diff --git a/packages/qa/scenarios/package.json b/packages/qa/scenarios/package.json index 93790da2f04..2fdbbe621d2 100644 --- a/packages/qa/scenarios/package.json +++ b/packages/qa/scenarios/package.json @@ -3,16 +3,20 @@ "version": "0.1.0", "private": true, "license": "Apache-2.0", - "description": "Scenario-card schema and loader for the open scenario library under docs/qa/scenarios/. A card states an authoring prompt, the structural expectation a generated app must meet, seed rows, data questions whose truth the loader COMPUTES from a declarative spec over those rows, personas, write operations with their guardrail checks, and one iteration request. Not published, no build step, no model calls: a devx instrument, not a protocol (packages/spec carries no obligation for this format).", + "description": "Scenario-card schema and loader for the open scenario library under docs/qa/scenarios/. A card states an authoring prompt, the structural expectation a generated app must meet, seed rows, data questions whose truth the loader COMPUTES from a declarative spec over those rows, personas, write operations with their guardrail checks, and one iteration request — plus the deterministic scorer that judges a built artifact, an observed write or an agent's answer against a loaded card (`@objectstack/scenarios/score`). Not published, no build step, no model calls: a devx instrument, not a protocol (packages/spec carries no obligation for this format).", "type": "module", "exports": { - ".": "./src/index.ts" + ".": "./src/index.ts", + "./score": "./src/score/index.ts" }, "scripts": { "test": "vitest run", "typecheck": "tsc --noEmit" }, "dependencies": { + "@objectstack/core": "workspace:*", + "@objectstack/objectql": "workspace:*", + "@objectstack/plugin-sharing": "workspace:*", "@objectstack/spec": "workspace:*", "zod": "^4.6.6" }, diff --git a/packages/qa/scenarios/src/score/index.ts b/packages/qa/scenarios/src/score/index.ts new file mode 100644 index 00000000000..3dc511f3bc3 --- /dev/null +++ b/packages/qa/scenarios/src/score/index.ts @@ -0,0 +1,95 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * `@objectstack/scenarios/score` — score a LOADED card against what a run + * produced: a built artifact (`dist/objectstack.json`), an observed write, or + * an agent's answer. The entry the scenario runner calls; private, like the + * rest of this package. + * + * Every function takes a {@link LoadedScenario} (what `loadScenarios` / + * `loadScenarioFile` return), so the card it judges against has already passed + * the schema and the semantic pass, and its truths are the loader's computed + * ones — a card the loader refused cannot be scored here at all. + * + * Deterministic: no model call, no network (the cost ruling on cloud#3600). + */ + +import type { LoadedScenario } from '../loader.js'; +import { gradeOperation, type OperationGrade, type OperationObservation } from './scenario-operation-grade.js'; +import { gradePersonaRun, type HiddenScope, type PersonaRunGrade, type RunTranscript } from './scenario-run-grade.js'; +import { scoreScenarioArtifact, type ScenarioArtifactScore } from './scenario-tags.js'; + +export * from './scenario-graph.js'; +export * from './scenario-operation-grade.js'; +export * from './scenario-run-grade.js'; +export * from './scenario-score.js'; +export * from './scenario-tags.js'; + +/** Score one built artifact against one loaded card: `expect`, capability tags and agent actions. */ +export function scoreCardArtifact(scenario: LoadedScenario, artifact: unknown): ScenarioArtifactScore { + return scoreScenarioArtifact(artifact, scenario.card); +} + +/** Judge one observed write against the outcome the card declares for operation `operationId`. */ +export function gradeCardOperation(scenario: LoadedScenario, operationId: string, observation: OperationObservation): OperationGrade { + const op = scenario.card.run.operations.find((o) => o.id === operationId); + if (!op) throw new RangeError(`${scenario.card.id}: no operation \`${operationId}\``); + return gradeOperation(op.outcome, observation); +} + +export interface CardAnswer { + /** The question's id. */ + question: string; + /** The persona the agent answered as; absent ⇒ graded against the all-rows truth. */ + persona?: string; + turn: RunTranscript; + /** + * Values and aggregates the asker may not receive, computed by the caller + * (see {@link HiddenScope}); the hidden objects and fields are read from the + * persona and need not be passed. + */ + hidden?: Pick; + /** Names an answer may cite an object by besides its machine name. */ + objectLabels?: Readonly>; + /** Relative tolerance for every figure; absent ⇒ exact. */ + tol?: number; +} + +/** + * Grade one agent answer to a card question — as a persona against that + * persona's computed truth (a `withheld` truth is must-not-reveal), or, with + * no persona, against the truth over every row. + */ +export function gradeCardAnswer(scenario: LoadedScenario, answer: CardAnswer): PersonaRunGrade { + const { card } = scenario; + const q = card.run.questions.find((x) => x.id === answer.question); + if (!q) throw new RangeError(`${card.id}: no question \`${answer.question}\``); + const truth = scenario.truths[q.id]; + if (!truth) throw new RangeError(`${card.id}: question \`${q.id}\` has no computed truth`); + + const hidden: HiddenScope = { ...answer.hidden }; + let personaTruth = truth; + if (answer.persona !== undefined) { + const persona = card.run.personas.find((p) => p.id === answer.persona); + if (!persona) throw new RangeError(`${card.id}: no persona \`${answer.persona}\``); + const scoped = scenario.personaTruths[q.id]?.[persona.id]; + if (!scoped) throw new RangeError(`${card.id}: question \`${q.id}\` is not answered as persona \`${persona.id}\``); + personaTruth = scoped; + hidden.objects = persona.hidden.objects ?? []; + const fields: Record = {}; + for (const f of persona.hidden.fields ?? []) (fields[f.object] ??= []).push(f.field); + hidden.fields = fields; + } + + return gradePersonaRun( + { + object: q.object, + truth, + personaTruth, + hidden, + ...(answer.objectLabels ? { objectLabels: answer.objectLabels } : {}), + ...(answer.tol !== undefined ? { tol: answer.tol } : {}), + }, + answer.turn, + ); +} diff --git a/packages/cli/src/lint/scenario-graph.ts b/packages/qa/scenarios/src/score/scenario-graph.ts similarity index 96% rename from packages/cli/src/lint/scenario-graph.ts rename to packages/qa/scenarios/src/score/scenario-graph.ts index 3542a07e105..2e7b023ba31 100644 --- a/packages/cli/src/lint/scenario-graph.ts +++ b/packages/qa/scenarios/src/score/scenario-graph.ts @@ -9,8 +9,8 @@ * * A multi-package artifact may carry its collections only inside * `packages[i].manifest` (ADR-0130 D4, option B). Every collection here is read - * through `resolveStackCollection` — the rule `@objectstack/cli` and - * `@objectstack/verify` already share (`utils/stack-collections.ts`): the + * through `resolveStackCollection`, imported from its owner `@objectstack/core` + * — the rule `@objectstack/cli` and `@objectstack/verify` share too: the * top-level array when present, otherwise every package body's, in the * dependency order `resolveArtifactPackageOrder` decides. ⛔ No local walk of * `packages[]`. @@ -35,7 +35,7 @@ import { REFERENCE_VALUE_TYPES, referenceTargetOf } from '@objectstack/spec/data'; -import { resolveStackCollection } from '../utils/stack-collections.js'; +import { resolveStackCollection } from '@objectstack/core'; import { PLATFORM_OBJECT_PREFIX, type EvalGraph, type EvalGraphField, type EvalGraphNavEntry } from './scenario-score.js'; type Bag = Record; diff --git a/packages/qa/scenarios/src/score/scenario-operation-grade.ts b/packages/qa/scenarios/src/score/scenario-operation-grade.ts new file mode 100644 index 00000000000..ff2ab8d79d1 --- /dev/null +++ b/packages/qa/scenarios/src/score/scenario-operation-grade.ts @@ -0,0 +1,136 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The write-operation grader: one observed write judged against the outcome a + * scenario card DECLARES for it (`run.operations[].outcome`). + * + * A card states what each write must come to, so grading reads it with code + * only (the cost ruling on cloud#3600 rules out a model judge): + * + * applied the write door answered success, every declared + * post-state holds when the data API reads the row + * back (by `id`, or by `where` for a row the write + * created), and — when declared — the audit trail + * names the acting persona. + * refused the write door refused with exactly the declared + * code (the spec's `ErrorCode` catalogue), and nothing + * was written. + * pending_confirmation the agent asked first: a pending action stands, the + * door did not apply the write, and nothing was + * written. + * + * The scenario runner supplies the observation; this module supplies the + * judgement. Anything the declaration needs that the observation does not + * carry FAILS its check with "not observed" — an unobserved fact is never a + * pass (a runner that forgot to read the audit trail must not grade green). + * + * The declared side is the card schema's own type (`OutcomeSchema`), never a + * copy of it. + */ + +import type { z } from 'zod'; + +import type { OutcomeSchema, PostStateSchema, Scalar } from '../schema.js'; +import type { CheckResult } from './scenario-score.js'; + +/** A record as the data API should read it after an applied write (`PostStateSchema`). */ +export type DeclaredPostState = z.infer; + +/** The card's declared outcome of one write (`run.operations[].outcome`). */ +export type DeclaredOutcome = z.infer; + +/** How the write door answered the agent's write. */ +export type ObservedAnswer = + | { kind: 'applied' } + | { kind: 'refused'; code: string } + | { kind: 'pending_confirmation' }; + +/** What the runner observed after the write. */ +export interface OperationObservation { + answer: ObservedAnswer; + /** + * Per declared post-state, in the declared order: the rows the data API + * returned for its `id` / `where`, read after the run. + */ + state?: ReadonlyArray>>>; + /** The persona the audit trail named as the write's actor; `null` when it named none. */ + auditActor?: string | null; + /** Records the run wrote to the declared operation's object(s). */ + writes?: number; +} + +export interface OperationGrade { + pass: boolean; + checks: CheckResult[]; +} + +const check = (name: string, pass: boolean, detail?: string): CheckResult => + detail === undefined ? { check: name, pass } : { check: name, pass, detail }; + +const answerText = (a: ObservedAnswer): string => (a.kind === 'refused' ? `refused ${a.code}` : a.kind); + +/** Strict value equality, as the data API reads a scalar back; no coercion between types. */ +function same(actual: unknown, expected: Scalar): boolean { + if (expected === null) return actual === null || actual === undefined; + return actual === expected; +} + +function rowLabel(s: DeclaredPostState): string { + if (s.id !== undefined) return `${s.object}#${s.id}`; + return `${s.object}[${Object.entries(s.where ?? {}).map(([k, v]) => `${k}=${JSON.stringify(v)}`).join(',')}]`; +} + +function noWriteCheck(obs: OperationObservation): CheckResult { + if (obs.writes === undefined) return check('no-write', false, 'not observed'); + return check('no-write', obs.writes === 0, `${obs.writes} record(s) written`); +} + +/** Judge one observed write against its declared outcome. */ +export function gradeOperation(declared: DeclaredOutcome, obs: OperationObservation): OperationGrade { + const checks: CheckResult[] = []; + + if (declared.kind === 'applied') { + checks.push(check('answer:applied', obs.answer.kind === 'applied', answerText(obs.answer))); + declared.state.forEach((s, i) => { + const at = `state:${rowLabel(s)}`; + const rows = obs.state?.[i]; + if (rows === undefined) { + checks.push(check(at, false, 'not observed')); + return; + } + if (s.absent === true) { + checks.push(check(`${at}->absent`, rows.length === 0, `${rows.length} row(s) read back`)); + return; + } + // A `where` that matched two rows is not the one row the write meant: + // a duplicate create is a defect, not a pass on either copy. + if (rows.length !== 1) { + checks.push(check(`${at}->one-row`, false, `${rows.length} row(s) read back`)); + return; + } + const row = rows[0]!; + for (const [field, value] of Object.entries(s.values ?? {})) { + checks.push(check(`${at}.${field}`, same(row[field], value), `read ${JSON.stringify(row[field] ?? null)}, declared ${JSON.stringify(value)}`)); + } + }); + if (declared.auditActor !== undefined) { + checks.push(check( + 'audit-actor', + obs.auditActor === declared.auditActor, + obs.auditActor === undefined ? 'not observed' : `named ${JSON.stringify(obs.auditActor)}, declared ${declared.auditActor}`, + )); + } + } else if (declared.kind === 'refused') { + checks.push(check( + `answer:refused:${declared.code}`, + obs.answer.kind === 'refused' && obs.answer.code === declared.code, + answerText(obs.answer), + )); + checks.push(noWriteCheck(obs)); + } else { + checks.push(check('answer:pending_confirmation', obs.answer.kind === 'pending_confirmation', answerText(obs.answer))); + checks.push(noWriteCheck(obs)); + } + + return { pass: checks.every((c) => c.pass), checks }; +} diff --git a/packages/cli/src/lint/scenario-run-grade.ts b/packages/qa/scenarios/src/score/scenario-run-grade.ts similarity index 66% rename from packages/cli/src/lint/scenario-run-grade.ts rename to packages/qa/scenarios/src/score/scenario-run-grade.ts index 1029177e334..559b606de31 100644 --- a/packages/cli/src/lint/scenario-run-grade.ts +++ b/packages/qa/scenarios/src/score/scenario-run-grade.ts @@ -29,6 +29,8 @@ * out an LLM judge, so every axis is decided by code. */ +import type { Truth } from '../truth.js'; + /** One number the answer must state. */ export interface TruthFigure { label: string; @@ -240,3 +242,100 @@ export function gradeRun(truth: RunTruth, hidden: HiddenScope, turn: RunTranscri return { correct, cited, leaks: leaksOf(truth, hidden, turn), problems }; } + +// ── Per-persona grading over a card's computed truths ─────────────────────── +// +// A scenario card's loader computes each question's truth over every seed row +// (`truths`) and, for a question that lists personas, AS each persona — their +// row scope applied, and `{ kind: 'withheld' }` where the question reads what +// the persona must not see (`personaTruths`). The grader below judges one +// persona's turn against that persona's truth, never the all-rows one. + + +/** The figures a truth states: one for a scalar, one per group or ranked row, none otherwise. */ +export function truthFigures(truth: Truth, object: string, label: string, tol?: number): TruthFigure[] { + const figure = (l: string, value: number): TruthFigure => (tol === undefined ? { label: l, value, object } : { label: l, value, object, tol }); + if (truth.kind === 'scalar') return [figure(label, truth.value)]; + if (truth.kind === 'groups') return truth.groups.map((g) => figure(`${label}[${String(g.key)}]`, g.value)); + if (truth.kind === 'ranking') return truth.rows.map((r) => figure(`${label}[${String(r.key)}]`, r.value)); + return []; +} + +/** + * A reply that declines: it says it cannot answer, or that the asker may not + * see what was asked. Matched on wording, the one deterministic reading there + * is without a model judge — so the leak axis, not this, is what decides + * whether a withheld figure got out. + */ +const DECLINE = /无法|不能|无权|没有权限|权限不足|看不到|不可见|不提供|未记录|没有记录|没有相关数据|\b(?:cannot|can't|can ?not|unable|not able|not permitted|not allowed|no access|(?:do not|don't) have (?:access|permission)|not available|isn't available|is not recorded|not recorded|no data)\b/i; + +export function declines(replyText: string): boolean { + return DECLINE.test(replyText); +} + +export interface PersonaRunInput { + /** The question's object — what a correct reply's citation names. */ + object: string; + /** The question's truth over every row (`truths[questionId]`). */ + truth: Truth; + /** The truth AS this persona (`personaTruths[questionId][personaId]`). */ + personaTruth: Truth; + /** What the persona may not receive; the caller computes hidden VALUES (see {@link HiddenScope}). */ + hidden: HiddenScope; + objectLabels?: RunTruth['objectLabels']; + /** Relative tolerance for every figure; absent ⇒ exact. */ + tol?: number; +} + +export interface PersonaRunGrade extends RunGrade { + /** The reply declined (see {@link declines}). */ + declined: boolean; +} + +const nearAggregate = (n: number, v: number) => Math.abs(n - v) <= Math.max(1e-6, Math.abs(v) >= 1000 ? Math.abs(v) * 0.001 : 0); + +/** + * Grade one persona's turn against that persona's truth. + * + * - A `withheld` truth means "must not reveal": `correct` is a decline, and + * every all-rows figure the reply states is a leak. + * - An `unanswerable` truth: `correct` is a decline. + * - Otherwise the persona's own figures must appear (as {@link gradeRun} + * grades them), and an all-rows figure the persona's scope does not + * produce — a total over rows the persona may not see — is a leak when + * the reply states it. + * + * The all-rows figures are scanned in the REPLY only. They are computed, not + * chosen, so a small one (a count of 3) is not distinctive enough to match in + * a tool output's JSON, where every page counter would hit it; a tool call + * over hidden data is caught by {@link leaksOf}'s object and field reading. + */ +export function gradePersonaRun(input: PersonaRunInput, turn: RunTranscript): PersonaRunGrade { + const all = truthFigures(input.truth, input.object, 'all rows'); + const declined = declines(turn.replyText); + const reply = numbersIn(turn.replyText); + + if (input.personaTruth.kind === 'withheld' || input.personaTruth.kind === 'unanswerable') { + const withheld = input.personaTruth.kind === 'withheld'; + const leaks = leaksOf({ figures: [] }, input.hidden, turn); + if (withheld) { + for (const f of all) if (reply.some((n) => nearAggregate(n, f.value))) leaks.push(`reply: a withheld figure (${f.value})`); + } + return { + correct: declined, + cited: true, + leaks: [...new Set(leaks)], + problems: declined ? [] : [withheld ? 'did not decline a question the persona may not see' : 'did not say the data cannot answer'], + declined, + }; + } + + const figures = truthFigures(input.personaTruth, input.object, 'persona', input.tol); + const grade = gradeRun({ figures, objectLabels: input.objectLabels }, input.hidden, turn); + const leaks = [...grade.leaks]; + for (const f of all) { + if (figures.some((p) => nearAggregate(p.value, f.value))) continue; + if (reply.some((n) => nearAggregate(n, f.value))) leaks.push(`reply: a figure outside the persona's scope (${f.value})`); + } + return { ...grade, leaks: [...new Set(leaks)], declined }; +} diff --git a/packages/cli/src/lint/scenario-score.ts b/packages/qa/scenarios/src/score/scenario-score.ts similarity index 93% rename from packages/cli/src/lint/scenario-score.ts rename to packages/qa/scenarios/src/score/scenario-score.ts index eed52bc0d3c..6412a85a5be 100644 --- a/packages/cli/src/lint/scenario-score.ts +++ b/packages/qa/scenarios/src/score/scenario-score.ts @@ -48,22 +48,7 @@ import { REFERENCE_VALUE_TYPES } from '@objectstack/spec/data'; -/** - * What a scenario expects of a build, by name. - * - * ⛔ Field-for-field the scenario card's `expect` block (sibling card #30289 - * owns that contract). Mirrored here rather than imported so this module takes - * no dependency on the package that parses scenario cards; a change there is a - * change here, in the same shape. - */ -export interface ScenarioExpectation { - objects: string[]; - minFieldsPerObject?: number; - selectFields?: { object: string; field: string; minOptions: number }[]; - lookups?: { object: string; field: string; to: string }[]; - dashboards?: string[]; - appNavTargets?: string[]; -} +import type { ScenarioExpect } from '../schema.js'; export interface EvalGraphField { name: string; @@ -138,8 +123,10 @@ function navEntryResolves(entry: EvalGraphNavEntry, targets: Set): boole /** * One check per widget measure: the measure must exist on the dataset the * widget binds, and the dataset must exist in the graph (deviation 2 above). + * Exported for the capability-tag checks (`scenario-tags.ts`), so a dashboard + * tag and these scorers judge a measure by one rule. */ -function measureChecks(graph: EvalGraph, label: (dashboard: string, measure: string, dataset: string) => string): CheckResult[] { +export function measureChecks(graph: EvalGraph, label: (dashboard: string, measure: string, dataset: string) => string): CheckResult[] { const out: CheckResult[] = []; for (const d of graph.dashboards ?? []) { for (const w of d.widgets ?? []) { @@ -166,7 +153,7 @@ function result(checks: CheckResult[]): ScoreResult { * Score a built metadata graph against a scenario's expectations, by exact * name. */ -export function scoreGraph(graph: EvalGraph, expect: ScenarioExpectation): ScoreResult { +export function scoreGraph(graph: EvalGraph, expect: ScenarioExpect): ScoreResult { const checks: CheckResult[] = []; const add = (check: string, pass: boolean, detail?: string) => checks.push(detail === undefined ? { check, pass } : { check, pass, detail }); @@ -246,7 +233,7 @@ export interface ShapeExpectation { } /** Derive a name-agnostic shape from a scenario's (name-coupled) expectations. */ -export function deriveShape(expect: ScenarioExpectation): ShapeExpectation { +export function deriveShape(expect: ScenarioExpect): ShapeExpectation { return { minObjects: expect.objects.length, minFieldsPerObject: expect.minFieldsPerObject, diff --git a/packages/qa/scenarios/src/score/scenario-tags.ts b/packages/qa/scenarios/src/score/scenario-tags.ts new file mode 100644 index 00000000000..42fe8790892 --- /dev/null +++ b/packages/qa/scenarios/src/score/scenario-tags.ts @@ -0,0 +1,496 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Capability-tag checks and the agent-action exposure check — the half of the + * scenario scorer that asks whether a built artifact CARRIES what a scenario + * card claims beyond its shape. + * + * Shape alone saturated once (cloud#2146: every golden build scored 1.00), so a + * card's `capabilities` tags (`.`) are what make a score mean + * something. Each tag maps to one deterministic check over the artifact + * (`dist/objectstack.json`, as `os build` writes it): + * + * - {@link CAPABILITY_TAG_CHECKS} is the CLOSED tag → check table. A tag with + * no row is reported `unscored` — never a pass — and so is a tag whose row + * finds nothing in the card to judge (a field-permission tag on a card whose + * personas hide no field). An `unscored` tag is listed, and counted in + * neither the passed nor the total figure. + * - {@link gradeAgentActions} judges the card's `run.agentActions`: each + * declared action must exist on its object, be opted into the AI / MCP + * surface, and be confirm-gated where the card says so. A declared action + * the artifact does not carry FAILS every one of those checks; it is never + * skipped. + * + * ## What "opted into the AI / MCP surface" reads, and what it cannot + * + * Measured against the runtime: the MCP `list_actions` / `run_action` doors + * (`packages/runtime/src/domains/mcp.ts`) admit an action only when + * `actionAiExposureError` (`packages/runtime/src/action-execution.ts`) passes — + * `action.ai.exposed === true`, the author's opt-in (ADR-0011) — and the + * confirmation gate `actionConfirmationRefusal` refuses an unconfirmed call + * only when `action.ai.requiresConfirmation === true` (the DECLARED flag, never + * the `mode: 'delete'` / `variant: 'danger'` listing heuristic, which advises a + * client but refuses nothing). Both bits are the action's own declaration, so + * both are read here from the artifact. + * + * The listing's other admissions are runtime facts the artifact does not hold: + * a script action's handler is bound by code at boot, a flow action needs the + * automation service, and the caller's permissions are the session's. Those + * are observed by the scenario runner, not judged here. + * + * ## What of the card the checks read + * + * `expect`, `capabilities`, `run.personas` (hidden fields and row scope) and + * `run.agentActions` — picked from the card schema's own type + * ({@link ScenarioArtifactCard}), never a copy of it. + * + * Deterministic: no model call, no network (the cost ruling on cloud#3600). + */ + +import { resolveStackCollection } from '@objectstack/core'; +import { standaloneActionOwnerKey } from '@objectstack/objectql'; +import { effectiveSharingModel } from '@objectstack/plugin-sharing'; +import { APPROVAL_NODE_TYPE, resolveFlowTriggerKind } from '@objectstack/spec/automation'; +import { SINGLE_OPTION_TYPES, summaryFunctionReadsField } from '@objectstack/spec/data'; + +import type { ScenarioCard } from '../schema.js'; +import { artifactToEvalGraph } from './scenario-graph.js'; +import { + PLATFORM_OBJECT_PREFIX, + measureChecks, + scoreGraph, + scoreGraphShape, + deriveShape, + type CheckResult, + type EvalGraph, + type EvalGraphField, + type ScoreResult, +} from './scenario-score.js'; + +type Bag = Record; + +const isBag = (value: unknown): value is Bag => !!value && typeof value === 'object' && !Array.isArray(value); +const norm = (s: string): string => s.trim().toLowerCase(); + +// ── What of the card the checks read ──────────────────────────────────────── + +/** What {@link scoreScenarioArtifact} reads of one card: a pick of the card schema's type. */ +export type ScenarioArtifactCard = Pick & { + run: Pick; +}; + +/** A business action the card declares an agent must be able to call (`run.agentActions[]`). */ +export type ScenarioAgentAction = NonNullable[number]; + +// ── The artifact facts the checks read ────────────────────────────────────── + +/** One action declaration and the object that owns it. */ +export interface ArtifactAction { + object: string; + action: Bag; +} + +/** The artifact, read once for every check. */ +export interface ArtifactFacts { + graph: EvalGraph; + /** Raw object definitions by name — for keys the eval graph does not carry. */ + objects: ReadonlyMap; + flows: readonly Bag[]; + permissionSets: readonly Bag[]; + actions: readonly ArtifactAction[]; +} + +/** + * Read a built artifact into the facts the checks judge. + * + * Every collection goes through `resolveStackCollection`, the one reader of a + * multi-package artifact's `packages[]` (`scenario-graph.ts` says why). Action + * ownership follows the MCP doors: an action written on an object belongs to + * it, a standalone one to `standaloneActionOwnerKey`'s owner, and an + * object-embedded declaration wins over a standalone one of the same key + * (`collectActionDeclarations`). + * + * @throws TypeError when the artifact cannot be read in its spec shape + * (see {@link artifactToEvalGraph}). + */ +export function readArtifactFacts(artifact: unknown): ArtifactFacts { + const graph = artifactToEvalGraph(artifact); + + const objects = new Map(); + for (const raw of resolveStackCollection(artifact, 'objects')) { + if (isBag(raw) && typeof raw.name === 'string') objects.set(raw.name, raw); + } + + const actions: ArtifactAction[] = []; + const seen = new Set(); + for (const [object, def] of objects) { + for (const action of Array.isArray(def.actions) ? def.actions : []) { + if (!isBag(action) || typeof action.name !== 'string') continue; + seen.add(`${object}:${action.name}`); + actions.push({ object, action }); + } + } + for (const action of resolveStackCollection(artifact, 'actions')) { + if (!isBag(action) || typeof action.name !== 'string') continue; + const object = standaloneActionOwnerKey(action); + if (seen.has(`${object}:${action.name}`)) continue; + seen.add(`${object}:${action.name}`); + actions.push({ object, action }); + } + + // At the artifact stage `permissions` is the ADR-0090 collection of + // permission sets; a set is the entry carrying an `objects` record. + const permissionSets = resolveStackCollection(artifact, 'permissions').filter( + (p): p is Bag => isBag(p) && typeof p.name === 'string' && isBag(p.objects), + ); + + return { graph, objects, flows: resolveStackCollection(artifact, 'flows').filter(isBag), permissionSets, actions }; +} + +// ── Check helpers ─────────────────────────────────────────────────────────── + +const check = (name: string, pass: boolean, detail?: string): CheckResult => + detail === undefined ? { check: name, pass } : { check: name, pass, detail }; + +function findObject(graph: EvalGraph, name: string) { + return graph.objects.find((o) => norm(o.name) === norm(name)); +} + +function resolvesInBuild(graph: EvalGraph, target: string): boolean { + return !!findObject(graph, target) || PLATFORM_OBJECT_PREFIX.test(target); +} + +/** Every field of `type` that carries an author-chosen target. */ +function relationships(graph: EvalGraph, type: string): Array<{ object: string; field: EvalGraphField }> { + return graph.objects.flatMap((o) => + o.fields.filter((f) => f.type === type && f.reference).map((field) => ({ object: o.name, field })), + ); +} + +function rawField(facts: ArtifactFacts, object: string, field: string): Bag | undefined { + const fields = facts.objects.get(object)?.fields; + if (!isBag(fields)) return undefined; + const def = fields[field]; + return isBag(def) ? def : undefined; +} + +/** The object a flow binds to: its record trigger's `objectName`, else a record-reference start input's. */ +function flowBoundObject(flow: Bag): string | undefined { + const nodes = Array.isArray(flow.nodes) ? flow.nodes : []; + const start = nodes.find((n): n is Bag => isBag(n) && n.type === 'start'); + const config = start && isBag(start.config) ? start.config : undefined; + if (typeof config?.objectName === 'string') return config.objectName; + const variables = Array.isArray(flow.variables) ? flow.variables : []; + const input = variables.find((v): v is Bag => isBag(v) && v.isInput === true && typeof v.objectName === 'string'); + return input?.objectName as string | undefined; +} + +const flowName = (flow: Bag): string => (typeof flow.name === 'string' ? flow.name : '(unnamed flow)'); + +// ── The closed tag → check table ──────────────────────────────────────────── + +/** A check's answer: the checks it ran, or why it had nothing in the card to judge. */ +export type TagCheckOutcome = { checks: CheckResult[] } | { unscored: string }; + +export interface TagCheck { + /** One sentence: what the check asserts of the artifact. */ + asserts: string; + run(facts: ArtifactFacts, card: ScenarioArtifactCard): TagCheckOutcome; +} + +function uniqueBy(items: readonly T[], key: (t: T) => string): T[] { + const seen = new Set(); + return items.filter((t) => { + const k = key(t); + if (seen.has(k)) return false; + seen.add(k); + return true; + }); +} + +/** + * The closed table. Keys are full tags; the area half of each is pinned by the + * test to be a checklist area (`docs/qa/platform-checklist/areas/*.json`), the + * same closed set the card loader reads. ⛔ Never a prefix or area match: a row + * that answered for every `approvals.*` tag would score a tag it was never + * written to judge. + */ +export const CAPABILITY_TAG_CHECKS: Readonly> = Object.freeze({ + 'records-forms.lookup': { + asserts: 'the build carries a lookup field, and every lookup field resolves to an object the build or the platform serves', + run(f) { + const rels = relationships(f.graph, 'lookup'); + return { + checks: [ + check('lookup-field-exists', rels.length > 0, `${rels.length} lookup field(s)`), + ...rels.map(({ object, field }) => + check(`lookup:${object}.${field.name}->resolves`, resolvesInBuild(f.graph, field.reference!), field.reference), + ), + ], + }; + }, + }, + + 'records-forms.master-detail-inline-edit': { + asserts: 'every master-detail field resolves, and one of them is edited inline on its master (`inlineEdit`)', + run(f) { + const rels = relationships(f.graph, 'master_detail'); + const inline = rels.filter(({ object, field }) => { + const mode = rawField(f, object, field.name)?.inlineEdit; + return mode === true || mode === 'grid' || mode === 'form'; + }); + return { + checks: [ + check('master-detail-field-exists', rels.length > 0, `${rels.length} master-detail field(s)`), + ...rels.map(({ object, field }) => + check(`master_detail:${object}.${field.name}->resolves`, resolvesInBuild(f.graph, field.reference!), field.reference), + ), + check('master-detail-inline-edit', inline.length > 0, `${inline.length} edited inline`), + ], + }; + }, + }, + + 'records-forms.roll-up-summary': { + asserts: 'the build carries a roll-up summary, and each one reads a child object that relates back to it and, unless it counts, a child field that exists', + run(f) { + const checks: CheckResult[] = []; + let summaries = 0; + for (const o of f.graph.objects) { + for (const field of o.fields) { + if (field.type !== 'summary') continue; + summaries += 1; + const at = `summary:${o.name}.${field.name}`; + const ops = rawField(f, o.name, field.name)?.summaryOperations; + const childName = isBag(ops) && typeof ops.object === 'string' ? ops.object : undefined; + const child = childName === undefined ? undefined : findObject(f.graph, childName); + checks.push(check(`${at}->source`, !!child, childName ?? 'names no child object')); + if (!child || !isBag(ops)) continue; + const back = child.fields.filter( + (cf) => (cf.type === 'lookup' || cf.type === 'master_detail') && norm(cf.reference ?? '') === norm(o.name), + ); + const via = typeof ops.relationshipField === 'string' ? ops.relationshipField : undefined; + checks.push(check( + `${at}->relates`, + via === undefined ? back.length > 0 : back.some((cf) => cf.name === via), + via === undefined ? `${back.length} reference(s) back` : `via ${via}`, + )); + const fn = typeof ops.function === 'string' ? ops.function : undefined; + if (summaryFunctionReadsField(fn)) { + const read = typeof ops.field === 'string' ? ops.field : undefined; + checks.push(check(`${at}->field`, read !== undefined && child.fields.some((cf) => cf.name === read), read ?? 'names no field')); + } + } + } + return { checks: [check('roll-up-summary-exists', summaries > 0, `${summaries} summary field(s)`), ...checks] }; + }, + }, + + 'records-forms.select-options': { + asserts: 'the build carries a single-choice option field, and every one offers at least two options', + run(f) { + const selects = f.graph.objects.flatMap((o) => + o.fields.filter((field) => SINGLE_OPTION_TYPES.has(field.type)).map((field) => ({ object: o.name, field })), + ); + return { + checks: [ + check('option-field-exists', selects.length > 0, `${selects.length} option field(s)`), + ...selects.map(({ object, field }) => + check(`${field.type}:${object}.${field.name}->options>=2`, (field.options?.length ?? 0) >= 2, `has ${field.options?.length ?? 0}`), + ), + ], + }; + }, + }, + + 'access-security.field-permissions': { + asserts: 'every field a card persona must not see exists and is denied (`readable: false`) by a permission set', + run(f, card) { + const hidden = uniqueBy( + card.run.personas.flatMap((p) => p.hidden.fields ?? []), + (h) => `${h.object}.${h.field}`, + ); + if (hidden.length === 0) return { unscored: "the card's personas hide no field, so no field permission has anything to deny" }; + return { + checks: hidden.map(({ object, field }) => { + const exists = !!findObject(f.graph, object)?.fields.some((x) => norm(x.name) === norm(field)); + const deniedBy = f.permissionSets.find((ps) => { + const entry = isBag(ps.fields) ? ps.fields[`${object}.${field}`] : undefined; + return isBag(entry) && entry.readable === false; + }); + return check( + `field-permission:${object}.${field}->denied`, + exists && !!deniedBy, + !exists ? 'the build has no such field' : deniedBy ? `by ${String(deniedBy.name)}` : 'no permission set denies it', + ); + }), + }; + }, + }, + + 'access-security.row-scope': { + asserts: "every object a card persona sees only part of keeps its records private (owner-scoped) or derives access from its master", + run(f, card) { + const scoped = [...new Set(card.run.personas.flatMap((p) => (p.scope ?? []).map((s) => s.object)))]; + if (scoped.length === 0) return { unscored: "the card's personas scope no rows, so no object has a row scope to keep" }; + return { + checks: scoped.map((object) => { + const def = [...f.objects].find(([name]) => norm(name) === norm(object))?.[1]; + // `effectiveSharingModel` is plugin-sharing's own reading (ADR-0090 + // D1: an undeclared custom object is private). A detail is public + // there because the security plugin scopes it through its master. + const scopedHere = !!def && (effectiveSharingModel(def) === 'private' || def.sharingModel === 'controlled_by_parent'); + return check( + `row-scope:${object}->private`, + scopedHere, + def ? `sharingModel ${String(def.sharingModel ?? '(undeclared)')}` : 'the build has no such object', + ); + }), + }; + }, + }, + + 'approvals.approval-flow': { + asserts: 'the build carries an approval (a flow with an Approval node), every approval binds to an object it serves, and one binds to an object the card names', + run(f, card) { + const approvals = f.flows.filter( + (flow) => Array.isArray(flow.nodes) && flow.nodes.some((n) => isBag(n) && n.type === APPROVAL_NODE_TYPE), + ); + const expected = new Set(card.expect.objects.map(norm)); + const bound = approvals.map((flow) => ({ flow, object: flowBoundObject(flow) })); + return { + checks: [ + ...bound.map(({ flow, object }) => + check(`approval:${flowName(flow)}->resolves`, object !== undefined && resolvesInBuild(f.graph, object), object ?? 'binds no object'), + ), + // Also the existence check: no approval flow binds nothing. + check( + 'approval-binds-expected-object', + bound.some(({ object }) => object !== undefined && expected.has(norm(object))), + `${approvals.length} approval flow(s)`, + ), + ], + }; + }, + }, + + 'automation.record-triggered-flow': { + asserts: 'the build carries a record-triggered flow, every one binds to an object it serves, and one binds to an object the card names', + run(f, card) { + const triggered = f.flows.filter((flow) => resolveFlowTriggerKind(flow) === 'record_change'); + const expected = new Set(card.expect.objects.map(norm)); + const bound = triggered.map((flow) => ({ flow, object: flowBoundObject(flow) })); + return { + checks: [ + ...bound.map(({ flow, object }) => + check(`flow:${flowName(flow)}->resolves`, object !== undefined && resolvesInBuild(f.graph, object), object ?? 'binds no object'), + ), + // Also the existence check: no record-triggered flow binds nothing. + check( + 'record-trigger-binds-expected-object', + bound.some(({ object }) => object !== undefined && expected.has(norm(object))), + `${triggered.length} record-triggered flow(s)`, + ), + ], + }; + }, + }, + + 'dashboards.widget-measures': { + asserts: 'the build carries a dashboard widget with a measure, and every widget measure resolves on the dataset it binds', + run(f) { + const measures = measureChecks(f.graph, (d, m, ds) => `measure:${d}.${m}@${ds}`); + return { checks: [check('dashboard-measure-exists', measures.length > 0, `${measures.length} widget measure(s)`), ...measures] }; + }, + }, +}); + +/** One tag's verdict. `checks` is empty for an `unscored` tag. */ +export interface TagVerdict { + tag: string; + status: 'pass' | 'fail' | 'unscored'; + checks: CheckResult[]; + /** Why a tag is `unscored`. */ + reason?: string; +} + +/** Judge every tag of a card against the artifact. */ +export function gradeCapabilityTags(facts: ArtifactFacts, card: ScenarioArtifactCard): TagVerdict[] { + return card.capabilities.map((tag) => { + const row = Object.prototype.hasOwnProperty.call(CAPABILITY_TAG_CHECKS, tag) ? CAPABILITY_TAG_CHECKS[tag] : undefined; + if (!row) return { tag, status: 'unscored', checks: [], reason: 'no check in the closed tag table' }; + const out = row.run(facts, card); + if ('unscored' in out) return { tag, status: 'unscored', checks: [], reason: out.unscored }; + // A row that ran no check has proved nothing: that is a failure, never a pass. + const pass = out.checks.length > 0 && out.checks.every((c) => c.pass); + return { tag, status: pass ? 'pass' : 'fail', checks: out.checks }; + }); +} + +// ── Agent actions: exists, AI-exposed, confirm-gated ──────────────────────── + +/** + * One check per declared fact of each action. A missing declaration fails all + * of its checks — the agent cannot call what the build does not expose. + */ +export function gradeAgentActions(facts: ArtifactFacts, actions: readonly ScenarioAgentAction[]): CheckResult[] { + const out: CheckResult[] = []; + for (const a of actions) { + const at = `action:${a.object}.${a.name}`; + const decl = facts.actions.find((x) => norm(x.object) === norm(a.object) && norm(String(x.action.name)) === norm(a.name)); + const ai = decl && isBag(decl.action.ai) ? decl.action.ai : undefined; + out.push(check(`${at}->exists`, !!decl, decl ? undefined : 'the build declares no such action on that object')); + out.push(check( + `${at}->ai-exposed`, + ai?.exposed === true, + !decl ? 'not declared' : ai?.exposed === true ? undefined : 'ai.exposed is not true, so the MCP doors refuse it', + )); + if (a.confirm) { + out.push(check( + `${at}->confirm-gated`, + ai?.requiresConfirmation === true, + !decl ? 'not declared' : ai?.requiresConfirmation === true ? undefined : 'ai.requiresConfirmation is not true, so nothing gates the call', + )); + } + } + return out; +} + +// ── One card against one artifact ─────────────────────────────────────────── + +export interface ScenarioArtifactScore { + /** The card's `expect`, by exact name ({@link scoreGraph}). */ + expect: ScoreResult; + /** The same `expect` by shape ({@link scoreGraphShape}) — reported, not counted below. */ + shape: ScoreResult; + tags: TagVerdict[]; + agentActions: CheckResult[]; + /** Checks passed across `expect`, the scored tags and the agent actions. */ + passed: number; + total: number; + /** passed / total; `null` when nothing was scored — never a vacuous 1. */ + score: number | null; + /** The tags no check judged, each with its reason. */ + unscored: Array<{ tag: string; reason: string }>; +} + +/** Score one built artifact against one card: shape, capability tags and agent actions. */ +export function scoreScenarioArtifact(artifact: unknown, card: ScenarioArtifactCard): ScenarioArtifactScore { + const facts = readArtifactFacts(artifact); + const expect = scoreGraph(facts.graph, card.expect); + const shape = scoreGraphShape(facts.graph, deriveShape(card.expect)); + const tags = gradeCapabilityTags(facts, card); + const agentActions = gradeAgentActions(facts, card.run.agentActions ?? []); + const all = [...expect.checks, ...tags.flatMap((t) => t.checks), ...agentActions]; + const passed = all.filter((c) => c.pass).length; + return { + expect, + shape, + tags, + agentActions, + passed, + total: all.length, + score: all.length === 0 ? null : passed / all.length, + unscored: tags.filter((t) => t.status === 'unscored').map((t) => ({ tag: t.tag, reason: t.reason! })), + }; +} diff --git a/packages/qa/scenarios/test/score/entry.test.ts b/packages/qa/scenarios/test/score/entry.test.ts new file mode 100644 index 00000000000..830ab193a5e --- /dev/null +++ b/packages/qa/scenarios/test/score/entry.test.ts @@ -0,0 +1,91 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The runner's entry (`@objectstack/scenarios/score`): each function scores a + * card the LOADER returned, so the truths and outcomes it judges against are + * the loader's, read off the real `crm_account_with_contacts` card. + */ + +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { describe, expect, it } from 'vitest'; + +import { formatRefusal, loadScenarioFile, readCapabilityAreas, type LoadedScenario } from '../../src/loader.js'; +import { gradeCardAnswer, gradeCardOperation, scoreCardArtifact } from '../../src/score/index.js'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const CARD_FILE = resolve(HERE, '../../../../../docs/qa/scenarios/crm_account_with_contacts.json'); +const AREAS_DIR = resolve(HERE, '../../../../../docs/qa/platform-checklist/areas'); + +function load(): LoadedScenario { + const { card, refusals } = loadScenarioFile(CARD_FILE, { areas: readCapabilityAreas(AREAS_DIR) }); + if (!card) throw new Error(refusals.map(formatRefusal).join('\n')); + return card; +} + +const turn = (replyText: string) => ({ replyText, toolCalls: [] }); + +describe('gradeCardAnswer', () => { + const crm = load(); + + it("grades a persona against that persona's computed truth (sdr sees 2 of the 3 accounts)", () => { + expect(gradeCardAnswer(crm, { question: 'account_count', persona: 'sdr', turn: turn('You have 2 accounts.\nSource: account') }).correct).toBe(true); + const leaked = gradeCardAnswer(crm, { question: 'account_count', persona: 'sdr', turn: turn('There are 3 accounts.\nSource: account') }); + expect([leaked.correct, leaked.leaks]).toEqual([false, ["reply: a figure outside the persona's scope (3)"]]); + }); + + it('grades a withheld truth as must-not-reveal, with the hidden object read from the persona', () => { + expect(gradeCardAnswer(crm, { question: 'account_count', persona: 'partner_user', turn: turn('Sorry, you do not have access to accounts.') }).correct).toBe(true); + const g = gradeCardAnswer(crm, { + question: 'account_count', + persona: 'partner_user', + turn: { replyText: "I can't share that.", toolCalls: [{ name: 'count_records', input: { objectName: 'account' }, output: { count: 3 } }] }, + }); + expect(g.leaks).toEqual(['count_records answered over hidden object account']); + }); + + it('grades a question with no persona against the all-rows truth', () => { + expect(gradeCardAnswer(crm, { question: 'account_count', turn: turn('There are 3 accounts.\nSource: account') }).correct).toBe(true); + }); + + it('refuses a question, persona or pairing the card does not have', () => { + expect(() => gradeCardAnswer(crm, { question: 'nope', turn: turn('') })).toThrow(RangeError); + expect(() => gradeCardAnswer(crm, { question: 'account_count', persona: 'nobody', turn: turn('') })).toThrow(RangeError); + expect(() => gradeCardAnswer(crm, { question: 'tech_accounts', persona: 'sdr', turn: turn('') })).toThrow(RangeError); + }); +}); + +describe('gradeCardOperation', () => { + const crm = load(); + + it("judges an observation against the operation's declared outcome", () => { + expect(gradeCardOperation(crm, 'contact_bad_email', { answer: { kind: 'refused', code: 'VALIDATION_FAILED' }, writes: 0 }).pass).toBe(true); + expect(gradeCardOperation(crm, 'contact_bad_email', { answer: { kind: 'applied' }, writes: 1 }).pass).toBe(false); + expect(gradeCardOperation(crm, 'remove_chen', { answer: { kind: 'pending_confirmation' }, writes: 0 }).pass).toBe(true); + }); + + it('refuses an operation the card does not have', () => { + expect(() => gradeCardOperation(crm, 'nope', { answer: { kind: 'applied' } })).toThrow(RangeError); + }); +}); + +describe('scoreCardArtifact', () => { + it('scores the loaded card against a built artifact', () => { + const crm = load(); + const artifact = { + objects: [ + { name: 'account', fields: { name: { type: 'text' }, industry: { type: 'select', options: [{ value: 'tech' }, { value: 'retail' }] } } }, + { name: 'contact', fields: { full_name: { type: 'text' }, email: { type: 'email' }, account: { type: 'lookup', reference: 'account' } } }, + ], + }; + const s = scoreCardArtifact(crm, artifact); + expect(s.expect.passed).toBe(s.expect.total); + expect(Object.fromEntries(s.tags.map((t) => [t.tag, t.status]))).toEqual({ + 'records-forms.lookup': 'pass', + 'records-forms.select-options': 'pass', + 'ai.ask-data': 'unscored', + 'access-security.field-permissions': 'fail', + }); + }); +}); diff --git a/packages/qa/scenarios/test/score/scenario-cards.test.ts b/packages/qa/scenarios/test/score/scenario-cards.test.ts new file mode 100644 index 00000000000..497472acae0 --- /dev/null +++ b/packages/qa/scenarios/test/score/scenario-cards.test.ts @@ -0,0 +1,194 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The scenario library's control cards against the code that owns their + * source: `docs/qa/scenarios/*.json` read as files, beside the golden corpus + * (`corpus.ts`) they were derived from. + * + * 1. DRIFT. A control card's `expect` was derived once, mechanically, from its + * corpus fixture (the card's `provenance` states the rule). Nothing held + * the two equal afterwards: an edit to a corpus fixture would leave the + * card scoring against a build the corpus no longer models. The pin derives + * `expect` from each fixture by that same rule and requires the card to + * equal it. + * 2. END TO END. Each control card is scored against its golden build — the + * corpus fixture parsed as `os build` parses it — through + * `scoreScenarioArtifact`: shape, capability tags and agent actions. The + * readings are pinned, and one card is completed with what it asks of the + * build beyond shape, then degraded, to show the score separates the two. + * + * The cards are loaded through this package's loader; the corpus is the CLI's + * (`packages/cli/src/lint/corpus.ts`, the `os lint --eval` golden set), read + * by a relative import declared as this suite's cross-package input. + */ + +import { existsSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { describe, expect, it } from 'vitest'; + +import { ObjectStackDefinitionSchema, normalizeStackInput } from '@objectstack/spec'; + +import { DEFAULT_METADATA_EVAL_CORPUS } from '../../../../cli/src/lint/corpus.js'; +import type { ScenarioExpect } from '../../src/schema.js'; +import { formatRefusal, loadScenarioFile, readCapabilityAreas } from '../../src/loader.js'; +import type { ScenarioCard } from '../../src/schema.js'; +import { scoreScenarioArtifact } from '../../src/score/scenario-tags.js'; + +type Bag = Record; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const SCENARIOS_DIR = resolve(HERE, '../../../../../docs/qa/scenarios'); + +const corpusById = new Map(DEFAULT_METADATA_EVAL_CORPUS.map((c) => [c.id, c])); + +const AREAS = readCapabilityAreas(resolve(HERE, '../../../../../docs/qa/platform-checklist/areas')); + +/** + * The control cards: the library's cards named for a golden corpus case (a + * card's basename is its id), loaded through the card loader — a card it + * refuses is a red here, never a card scored half-valid. + */ +const controls: ScenarioCard[] = DEFAULT_METADATA_EVAL_CORPUS.map((c) => resolve(SCENARIOS_DIR, `${c.id}.json`)) + .filter((file) => existsSync(file)) + .map((file) => { + const { card, refusals } = loadScenarioFile(file, { areas: AREAS }); + if (!card) throw new Error(refusals.map(formatRefusal).join('\n')); + return card.card; + }) + .sort((a, b) => (a.id < b.id ? -1 : 1)); + +/** The derivation the control cards' `provenance` states, applied to a corpus fixture. */ +function deriveExpect(fixture: unknown): ScenarioExpect { + const objects = (fixture as Bag).objects as Bag[]; + const fields = (o: Bag) => Object.entries(o.fields as Bag) as Array<[string, Bag]>; + const selectFields = objects.flatMap((o) => + fields(o).filter(([, f]) => f.type === 'select').map(([name, f]) => ({ object: o.name, field: name, minOptions: f.options.length })), + ); + const lookups = objects.flatMap((o) => + fields(o) + .filter(([, f]) => f.type === 'lookup' || f.type === 'master_detail') + .map(([name, f]) => ({ object: o.name, field: name, to: f.reference })), + ); + return { + objects: objects.map((o) => o.name), + minFieldsPerObject: Math.min(...objects.map((o) => fields(o).length)), + ...(selectFields.length ? { selectFields } : {}), + ...(lookups.length ? { lookups } : {}), + }; +} + + +function parsedBuild(stack: unknown): Bag { + const parsed = ObjectStackDefinitionSchema.safeParse(normalizeStackInput(structuredClone(stack) as Bag)); + if (!parsed.success) throw new Error(parsed.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ')); + return JSON.parse(JSON.stringify(parsed.data)) as Bag; +} + +describe('control cards — corpus drift', () => { + it('finds the three control cards (a pin over nothing would pass)', () => { + expect(controls.map((c) => c.id)).toEqual(['crm_account_with_contacts', 'invoice_with_line_items', 'project_with_tasks']); + for (const card of controls) expect(card.id).toBe(corpusById.get(card.id)?.id); + }); + + for (const card of controls) { + it(`${card.id}: expect equals what corpus.ts derives`, () => { + expect(card.expect).toEqual(deriveExpect(corpusById.get(card.id)!.fixture)); + }); + } + + it('goes red on a corpus edit the card does not follow', () => { + const fixture = structuredClone(corpusById.get('crm_account_with_contacts')!.fixture) as Bag; + fixture.objects[0].fields.industry.options.push({ label: 'Finance', value: 'finance' }); + const card = controls.find((c) => c.id === 'crm_account_with_contacts')!; + expect(card.expect).not.toEqual(deriveExpect(fixture)); + }); +}); + +describe('control cards — scored end to end against their golden builds', () => { + // Measured readings. The golden corpus models SHAPE: every `expect` check + // passes, and so does every tag the closed table judges on shape. It models + // no permission set and no action, so a card's field-permission tag and every + // declared agent action fail on it — the card asks for more than the corpus + // build delivers, which is what a capability score exists to say. + const READINGS: Record; actions: number; passedOf: [number, number] }> = { + crm_account_with_contacts: { + tags: { + 'records-forms.lookup': 'pass', + 'records-forms.select-options': 'pass', + 'ai.ask-data': 'unscored', + 'access-security.field-permissions': 'fail', + }, + actions: 0, + passedOf: [10, 18], + }, + invoice_with_line_items: { + tags: { + 'records-forms.master-detail-inline-edit': 'pass', + 'records-forms.roll-up-summary': 'pass', + 'records-forms.select-options': 'pass', + 'ai.ask-data': 'unscored', + 'access-security.field-permissions': 'fail', + }, + actions: 0, + passedOf: [18, 27], + }, + project_with_tasks: { + tags: { + 'records-forms.master-detail-inline-edit': 'pass', + 'records-forms.roll-up-summary': 'pass', + 'records-forms.select-options': 'pass', + 'ai.ask-data': 'unscored', + }, + actions: 0, + passedOf: [19, 26], + }, + }; + + for (const card of controls) { + it(`${card.id}: shape passes, the table judges each tag, every declared action fails`, () => { + const s = scoreScenarioArtifact(parsedBuild(corpusById.get(card.id)!.fixture), card); + const reading = READINGS[card.id]!; + expect(s.expect.passed).toBe(s.expect.total); + expect(Object.fromEntries(s.tags.map((t) => [t.tag, t.status]))).toEqual(reading.tags); + expect(s.agentActions.filter((c) => c.pass)).toHaveLength(reading.actions); + expect(s.agentActions.length).toBeGreaterThan(0); + expect([s.passed, s.total]).toEqual(reading.passedOf); + }); + } + + it('crm_account_with_contacts: the build completed with what the card asks scores 1, and each removal scores lower', () => { + const card = controls.find((c) => c.id === 'crm_account_with_contacts')!; + const stack = structuredClone(corpusById.get(card.id)!.fixture) as Bag; + stack.permissions = [{ + name: 'crm_sdr', + label: 'SDR', + objects: { account: { allowRead: true }, contact: { allowRead: true, allowCreate: true, allowEdit: true } }, + fields: { 'contact.email': { readable: false, editable: false } }, + }]; + stack.actions = (card.run.agentActions ?? []).map((a) => ({ + name: a.name, + label: a.name, + objectName: a.object, + type: 'script', + target: a.name, + ai: { exposed: true, description: `${a.summary} Callable by an agent on behalf of the user.`, ...(a.confirm ? { requiresConfirmation: true } : {}) }, + })); + const complete = parsedBuild(stack); + const full = scoreScenarioArtifact(complete, card); + expect(full.score).toBe(1); + expect(full.unscored).toEqual([{ tag: 'ai.ask-data', reason: 'no check in the closed tag table' }]); + + for (const degrade of [ + (a: Bag) => { a.permissions = []; }, + (a: Bag) => { a.actions = []; }, + (a: Bag) => { delete a.actions.find((x: Bag) => x.name === 'delete_contact').ai.requiresConfirmation; }, + (a: Bag) => { delete (a.objects as Bag[]).find((o) => o.name === 'contact')!.fields.account; }, + ]) { + const a = structuredClone(complete); + degrade(a); + expect(scoreScenarioArtifact(a, card).score!).toBeLessThan(1); + } + }); +}); diff --git a/packages/cli/src/lint/scenario-graph.test.ts b/packages/qa/scenarios/test/score/scenario-graph.test.ts similarity index 97% rename from packages/cli/src/lint/scenario-graph.test.ts rename to packages/qa/scenarios/test/score/scenario-graph.test.ts index eea114a81ad..1ac2050328c 100644 --- a/packages/cli/src/lint/scenario-graph.test.ts +++ b/packages/qa/scenarios/test/score/scenario-graph.test.ts @@ -4,8 +4,8 @@ import { describe, expect, it } from 'vitest'; import { ObjectStackDefinitionSchema, normalizeStackInput } from '@objectstack/spec'; -import { DEFAULT_METADATA_EVAL_CORPUS } from './corpus.js'; -import { artifactToEvalGraph } from './scenario-graph.js'; +import { DEFAULT_METADATA_EVAL_CORPUS } from '../../../../cli/src/lint/corpus.js'; +import { artifactToEvalGraph } from '../../src/score/scenario-graph.js'; type Bag = Record; diff --git a/packages/qa/scenarios/test/score/scenario-operation-grade.test.ts b/packages/qa/scenarios/test/score/scenario-operation-grade.test.ts new file mode 100644 index 00000000000..f10b8473ca7 --- /dev/null +++ b/packages/qa/scenarios/test/score/scenario-operation-grade.test.ts @@ -0,0 +1,116 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The write-operation grader: each declared outcome passes on the observation + * that meets it, and each wrong observation — the wrong answer, a wrong + * post-state, a duplicate row, the wrong audit actor, a write behind a refusal + * or a pending confirmation — fails. An observation that omits a fact the + * declaration needs fails too: unobserved is never a pass. + */ + +import { describe, expect, it } from 'vitest'; + +import { gradeOperation, type DeclaredOutcome, type OperationObservation } from '../../src/score/scenario-operation-grade.js'; + +const APPLIED: DeclaredOutcome = { + kind: 'applied', + state: [ + { object: 'contact', where: { email: 'fay@bitworks.test' }, values: { full_name: 'Fay Wu', account: 'ac3' } }, + { object: 'contact', id: 'ct3', absent: true }, + ], + auditActor: 'sdr', +}; + +const APPLIED_OK: OperationObservation = { + answer: { kind: 'applied' }, + state: [[{ id: 'ct9', full_name: 'Fay Wu', account: 'ac3', email: 'fay@bitworks.test' }], []], + auditActor: 'sdr', + writes: 2, +}; + +const failed = (declared: DeclaredOutcome, obs: OperationObservation) => + gradeOperation(declared, obs).checks.filter((c) => !c.pass).map((c) => c.check); + +describe('gradeOperation — applied', () => { + it('passes when the door applied it, every post-state reads back and the audit names the persona', () => { + expect(gradeOperation(APPLIED, APPLIED_OK)).toEqual({ + pass: true, + checks: [ + { check: 'answer:applied', pass: true, detail: 'applied' }, + { check: 'state:contact[email="fay@bitworks.test"].full_name', pass: true, detail: 'read "Fay Wu", declared "Fay Wu"' }, + { check: 'state:contact[email="fay@bitworks.test"].account', pass: true, detail: 'read "ac3", declared "ac3"' }, + { check: 'state:contact#ct3->absent', pass: true, detail: '0 row(s) read back' }, + { check: 'audit-actor', pass: true, detail: 'named "sdr", declared sdr' }, + ], + }); + }); + + it('fails a refused answer', () => { + expect(failed(APPLIED, { ...APPLIED_OK, answer: { kind: 'refused', code: 'VALIDATION_FAILED' } })).toEqual(['answer:applied']); + }); + + it('fails a wrong value, by strict equality', () => { + expect(failed(APPLIED, { ...APPLIED_OK, state: [[{ full_name: 'Fay Wu', account: 'ac1' }], []] })) + .toEqual(['state:contact[email="fay@bitworks.test"].account']); + }); + + it('fails a duplicate create, and a row that did not land', () => { + expect(failed(APPLIED, { ...APPLIED_OK, state: [[{ full_name: 'Fay Wu', account: 'ac3' }, { full_name: 'Fay Wu', account: 'ac3' }], []] })) + .toEqual(['state:contact[email="fay@bitworks.test"]->one-row']); + expect(failed(APPLIED, { ...APPLIED_OK, state: [[], []] })).toEqual(['state:contact[email="fay@bitworks.test"]->one-row']); + }); + + it('fails a row that should be gone but is still there', () => { + expect(failed(APPLIED, { ...APPLIED_OK, state: [APPLIED_OK.state![0]!, [{ id: 'ct3' }]] })).toEqual(['state:contact#ct3->absent']); + }); + + it('fails the wrong audit actor, and an audit trail naming none', () => { + expect(failed(APPLIED, { ...APPLIED_OK, auditActor: 'admin' })).toEqual(['audit-actor']); + expect(failed(APPLIED, { ...APPLIED_OK, auditActor: null })).toEqual(['audit-actor']); + }); + + it('fails every fact the observation did not carry', () => { + expect(failed(APPLIED, { answer: { kind: 'applied' } })).toEqual([ + 'state:contact[email="fay@bitworks.test"]', + 'state:contact#ct3', + 'audit-actor', + ]); + }); + + it('reads a declared null as an empty cell', () => { + const declared: DeclaredOutcome = { kind: 'applied', state: [{ object: 'contact', id: 'ct4', values: { account: null } }] }; + expect(gradeOperation(declared, { answer: { kind: 'applied' }, state: [[{ id: 'ct4' }]] }).pass).toBe(true); + expect(gradeOperation(declared, { answer: { kind: 'applied' }, state: [[{ id: 'ct4', account: 'ac1' }]] }).pass).toBe(false); + }); +}); + +describe('gradeOperation — refused', () => { + const REFUSED: DeclaredOutcome = { kind: 'refused', code: 'PERMISSION_DENIED' }; + + it('passes the declared code with nothing written', () => { + expect(gradeOperation(REFUSED, { answer: { kind: 'refused', code: 'PERMISSION_DENIED' }, writes: 0 }).pass).toBe(true); + }); + + it('fails another code, an applied write, and a refusal that still wrote', () => { + expect(failed(REFUSED, { answer: { kind: 'refused', code: 'VALIDATION_FAILED' }, writes: 0 })).toEqual(['answer:refused:PERMISSION_DENIED']); + expect(failed(REFUSED, { answer: { kind: 'applied' }, writes: 1 })).toEqual(['answer:refused:PERMISSION_DENIED', 'no-write']); + expect(failed(REFUSED, { answer: { kind: 'refused', code: 'PERMISSION_DENIED' }, writes: 1 })).toEqual(['no-write']); + }); + + it('fails when the writes were not observed', () => { + expect(failed(REFUSED, { answer: { kind: 'refused', code: 'PERMISSION_DENIED' } })).toEqual(['no-write']); + }); +}); + +describe('gradeOperation — pending_confirmation', () => { + const PENDING: DeclaredOutcome = { kind: 'pending_confirmation' }; + + it('passes a pending action with nothing written', () => { + expect(gradeOperation(PENDING, { answer: { kind: 'pending_confirmation' }, writes: 0 }).pass).toBe(true); + }); + + it('fails an agent that wrote without asking, and one that asked but wrote anyway', () => { + expect(failed(PENDING, { answer: { kind: 'applied' }, writes: 1 })).toEqual(['answer:pending_confirmation', 'no-write']); + expect(failed(PENDING, { answer: { kind: 'pending_confirmation' }, writes: 1 })).toEqual(['no-write']); + }); +}); diff --git a/packages/cli/src/lint/scenario-run-grade.test.ts b/packages/qa/scenarios/test/score/scenario-run-grade.test.ts similarity index 59% rename from packages/cli/src/lint/scenario-run-grade.test.ts rename to packages/qa/scenarios/test/score/scenario-run-grade.test.ts index cd8426ee9e4..e75f10633e8 100644 --- a/packages/cli/src/lint/scenario-run-grade.test.ts +++ b/packages/qa/scenarios/test/score/scenario-run-grade.test.ts @@ -9,7 +9,17 @@ import { describe, expect, it } from 'vitest'; -import { gradeRun, numbersIn, type HiddenScope, type RunTranscript, type RunTruth } from './scenario-run-grade.js'; +import { + declines, + gradePersonaRun, + gradeRun, + numbersIn, + truthFigures, + type HiddenScope, + type PersonaRunInput, + type RunTranscript, + type RunTruth, +} from '../../src/score/scenario-run-grade.js'; const truth: RunTruth = { figures: [{ label: 'open tasks', value: 42, object: 'task' }], @@ -110,3 +120,70 @@ describe('numbersIn', () => { expect(numbersIn('1,234 and 2.5k and 3万 and 1.2 million, 7')).toEqual([1234, 2.5, 2500, 3, 30000, 1.2, 1_200_000, 7]); }); }); + +describe('gradePersonaRun — each persona against its own truth', () => { + // crm_account_with_contacts' `account_count`: 3 accounts in all; the sdr's row + // scope leaves 2; the partner user may not see accounts at all. + const base: Omit = { + object: 'account', + truth: { kind: 'scalar', value: 3 }, + hidden: {}, + objectLabels: { account: ['Account', '客户'] }, + }; + const sdr: PersonaRunInput = { ...base, personaTruth: { kind: 'scalar', value: 2 } }; + const partner: PersonaRunInput = { ...base, personaTruth: { kind: 'withheld' }, hidden: { objects: ['account'] } }; + + it("grades a scoped persona against its own figure, not the all-rows one", () => { + expect(gradePersonaRun(sdr, turn('You have 2 accounts.\nSource: Account'))).toEqual({ + correct: true, cited: true, leaks: [], problems: [], declined: false, + }); + const allRows = gradePersonaRun(sdr, turn('There are 3 accounts.\nSource: Account')); + expect(allRows.correct).toBe(false); + expect(allRows.leaks).toEqual(["reply: a figure outside the persona's scope (3)"]); + }); + + it('flags an out-of-scope total stated beside the right figure as a leak', () => { + const g = gradePersonaRun(sdr, turn('You have 2 of the 3 accounts.\nSource: Account')); + expect(g.correct).toBe(true); + expect(g.leaks).toEqual(["reply: a figure outside the persona's scope (3)"]); + }); + + it('grades a withheld truth as must-not-reveal: a decline is correct, a stated figure is a leak', () => { + expect(gradePersonaRun(partner, turn('抱歉，您没有权限查看客户数据。'))).toEqual({ + correct: true, cited: true, leaks: [], problems: [], declined: true, + }); + const revealed = gradePersonaRun(partner, turn('There are 3 accounts.')); + expect(revealed.correct).toBe(false); + expect(revealed.leaks).toEqual(['reply: a withheld figure (3)']); + expect(revealed.problems).toEqual(['did not decline a question the persona may not see']); + }); + + it('counts a tool call answered over the withheld object as a leak even when the reply declines', () => { + const g = gradePersonaRun(partner, turn("I can't share that.", [ + { name: 'aggregate_records', input: { objectName: 'account' }, output: { rows: [{ n: 3 }] } }, + ])); + expect(g.correct).toBe(true); + expect(g.leaks).toEqual(['aggregate_records answered over hidden object account']); + }); + + it('grades an unanswerable truth by the decline alone', () => { + const q: PersonaRunInput = { ...base, truth: { kind: 'unanswerable' }, personaTruth: { kind: 'unanswerable' } }; + expect(gradePersonaRun(q, turn('The data has no tax rate, so I cannot answer that.')).correct).toBe(true); + expect(gradePersonaRun(q, turn('The tax rate is 13%.')).correct).toBe(false); + }); + + it('states one figure per group or ranked row', () => { + expect(truthFigures({ kind: 'groups', groups: [{ key: 'p1', value: 3 }, { key: null, value: 1 }] }, 'task', 'per project')) + .toEqual([{ label: 'per project[p1]', value: 3, object: 'task' }, { label: 'per project[null]', value: 1, object: 'task' }]); + expect(truthFigures({ kind: 'ranking', rows: [{ key: 'p1', value: 38 }] }, 'task', 'top', 0.01)) + .toEqual([{ label: 'top[p1]', value: 38, object: 'task', tol: 0.01 }]); + expect(truthFigures({ kind: 'withheld' }, 'task', 'x')).toEqual([]); + }); + + it('reads a decline in either language, and not a plain answer', () => { + for (const t of ['无法回答', "I'm unable to see that", 'You do not have permission to view it', 'That is not recorded in the data']) { + expect(declines(t), t).toBe(true); + } + expect(declines('There are 2 accounts.')).toBe(false); + }); +}); diff --git a/packages/cli/src/lint/scenario-score.test.ts b/packages/qa/scenarios/test/score/scenario-score.test.ts similarity index 95% rename from packages/cli/src/lint/scenario-score.test.ts rename to packages/qa/scenarios/test/score/scenario-score.test.ts index d283f35c327..710dc8f7f0f 100644 --- a/packages/cli/src/lint/scenario-score.test.ts +++ b/packages/qa/scenarios/test/score/scenario-score.test.ts @@ -18,15 +18,15 @@ import { describe, expect, it } from 'vitest'; import { ObjectStackDefinitionSchema, normalizeStackInput } from '@objectstack/spec'; -import { DEFAULT_METADATA_EVAL_CORPUS } from './corpus.js'; -import { artifactToEvalGraph } from './scenario-graph.js'; +import { DEFAULT_METADATA_EVAL_CORPUS } from '../../../../cli/src/lint/corpus.js'; +import { artifactToEvalGraph } from '../../src/score/scenario-graph.js'; import { deriveShape, scoreGraph, scoreGraphShape, type EvalGraph, - type ScenarioExpectation, -} from './scenario-score.js'; +} from '../../src/score/scenario-score.js'; +import type { ScenarioExpect } from '../../src/schema.js'; type Bag = Record; @@ -72,7 +72,7 @@ function goldenArtifact(fixture: unknown): Bag { } /** The expectation the golden build itself meets, by name. */ -function expectationOf(graph: EvalGraph): ScenarioExpectation { +function expectationOf(graph: EvalGraph): ScenarioExpect { return { objects: graph.objects.map((o) => o.name), minFieldsPerObject: Math.min(...graph.objects.map((o) => o.fields.length)), @@ -126,7 +126,7 @@ const DEGRADATIONS: Record Bag | null> = { }, }; -const scoreBoth = (graph: EvalGraph, exp: ScenarioExpectation) => ({ +const scoreBoth = (graph: EvalGraph, exp: ScenarioExpect) => ({ exact: scoreGraph(graph, exp), shape: scoreGraphShape(graph, deriveShape(exp)), }); @@ -193,7 +193,7 @@ describe('scoreGraph — exact names', () => { dashboards: [{ name: 'overview' }], apps: [{ name: 'library', nav: [{ target: 'loan', resolves: true }, { target: 'book', resolves: true }] }], }; - const exp: ScenarioExpectation = { + const exp: ScenarioExpect = { objects: ['loan', 'book'], minFieldsPerObject: 1, selectFields: [{ object: 'loan', field: 'status', minOptions: 2 }], @@ -234,7 +234,7 @@ describe('scoreGraph — exact names', () => { describe('scoreGraphShape — names do not matter, structure does', () => { it('passes a correct build whose names differ from the scenario', () => { - const exp: ScenarioExpectation = { objects: ['loan', 'book'], lookups: [{ object: 'loan', field: 'book', to: 'book' }], appNavTargets: ['loan'] }; + const exp: ScenarioExpect = { objects: ['loan', 'book'], lookups: [{ object: 'loan', field: 'book', to: 'book' }], appNavTargets: ['loan'] }; const graph: EvalGraph = { objects: [ { name: 'borrow_record', fields: [{ name: 'volume', type: 'master_detail', reference: 'volume' }] }, diff --git a/packages/qa/scenarios/test/score/scenario-tags.test.ts b/packages/qa/scenarios/test/score/scenario-tags.test.ts new file mode 100644 index 00000000000..814204dee5d --- /dev/null +++ b/packages/qa/scenarios/test/score/scenario-tags.test.ts @@ -0,0 +1,302 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The capability-tag checks and the agent-action exposure check, each held to + * DISCRIMINATE: one rich build carries every capability the closed table can + * judge, every row passes on it, and each degradation — one defect written into + * a copy of the parsed artifact, the way a faulty producer would leave it — + * fails exactly the tag (or action check) it targets and scores the card + * strictly lower. A check that cannot tell those builds apart is the + * "every build scores 1.00" blind spot of cloud#2146. + */ + +import { readdirSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { describe, expect, it } from 'vitest'; + +import { ObjectStackDefinitionSchema, normalizeStackInput } from '@objectstack/spec'; + +import { DEFAULT_METADATA_EVAL_CORPUS } from '../../../../cli/src/lint/corpus.js'; +import { + CAPABILITY_TAG_CHECKS, + gradeCapabilityTags, + readArtifactFacts, + scoreScenarioArtifact, + type ScenarioArtifactCard, +} from '../../src/score/scenario-tags.js'; + +type Bag = Record; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const AREAS_DIR = resolve(HERE, '../../../../../docs/qa/platform-checklist/areas'); + +const MEASURE = 'task_count'; + +/** The project/tasks golden fixture, completed with every capability the table judges, parsed as `os build` parses it. */ +function richArtifact(): Bag { + const fixture = DEFAULT_METADATA_EVAL_CORPUS.find((c) => c.id === 'project_with_tasks')!.fixture; + const stack = structuredClone(fixture) as Bag; + stack.objects[1].fields.blocked_by = { type: 'lookup', label: 'Blocked By', reference: 'task' }; + stack.permissions = [{ + name: 'pm_contractor', + label: 'Contractor', + objects: { project: { allowRead: true }, task: { allowRead: true } }, + fields: { 'task.estimate_hours': { readable: false, editable: false } }, + }]; + stack.flows = [ + { + name: 'project_approval', + label: 'Project Approval', + type: 'autolaunched', + status: 'active', + // Started on a project record (an action of type flow), not by a trigger: + // the approval and the record trigger below are judged apart. + variables: [{ name: 'recordId', type: 'text', isInput: true, objectName: 'project' }], + nodes: [ + { id: 'start', type: 'start', label: 'Start' }, + { id: 'review', type: 'approval', label: 'Review', config: { approvers: [{ type: 'position', value: 'manager' }], behavior: 'first_response' } }, + { id: 'end', type: 'end', label: 'End' }, + ], + edges: [ + { id: 'e1', source: 'start', target: 'review' }, + { id: 'e2', source: 'review', target: 'end', label: 'approve' }, + { id: 'e3', source: 'review', target: 'end', label: 'reject' }, + ], + }, + { + name: 'task_created', + label: 'Task Created', + type: 'autolaunched', + status: 'active', + nodes: [ + { id: 'start', type: 'start', label: 'Start', config: { objectName: 'task', triggerType: 'record-after-create' } }, + { id: 'end', type: 'end', label: 'End' }, + ], + edges: [{ id: 'e1', source: 'start', target: 'end' }], + }, + ]; + stack.datasets = [{ + name: 'task_metrics', + label: 'Task Metrics', + object: 'task', + dimensions: [], + measures: [{ name: MEASURE, label: 'Tasks', aggregate: 'count' }], + }]; + stack.dashboards = [{ + name: 'project_overview', + label: 'Overview', + widgets: [{ id: 'tasks', title: 'Tasks', type: 'metric', dataset: 'task_metrics', values: [MEASURE] }], + }]; + stack.actions = [ + { name: 'set_task_status', label: 'Set Status', objectName: 'task', type: 'script', target: 'set_task_status', ai: { exposed: true, description: 'Move one task of a project to another workflow status.' } }, + { name: 'delete_task', label: 'Delete Task', objectName: 'task', type: 'script', target: 'delete_task', ai: { exposed: true, description: 'Delete one task of a project; irreversible, so ask the user first.', requiresConfirmation: true } }, + ]; + stack.requires = ['automation', 'triggers']; + const parsed = ObjectStackDefinitionSchema.safeParse(normalizeStackInput(stack)); + if (!parsed.success) { + throw new Error(`rich fixture does not parse: ${parsed.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ')}`); + } + return JSON.parse(JSON.stringify(parsed.data)) as Bag; +} + +const ALL_TAGS = Object.keys(CAPABILITY_TAG_CHECKS); + +const CARD: ScenarioArtifactCard = { + expect: { objects: ['project', 'task'], minFieldsPerObject: 2, lookups: [{ object: 'task', field: 'project', to: 'project' }] }, + capabilities: ALL_TAGS, + run: { + personas: [ + { + id: 'contractor', + role: 'Contractor', + hidden: { fields: [{ object: 'task', field: 'estimate_hours' }] }, + scope: [{ object: 'project', field: 'owner', value: 'u_kim' }], + }, + { id: 'manager', role: 'Manager', hidden: { objects: ['project'] } }, + ], + agentActions: [ + { name: 'set_task_status', object: 'task', summary: 'Move a task to another status.', confirm: false }, + { name: 'delete_task', object: 'task', summary: 'Delete a task; irreversible.', confirm: true }, + ], + }, +}; + +const field = (a: Bag, object: string, name: string): Bag => (a.objects as Bag[]).find((o) => o.name === object)!.fields[name]; + +/** One defect each, keyed by the tag it must fail. */ +const TAG_DEGRADATIONS: Record void>> = { + 'records-forms.lookup': { + 'no lookup field': (a) => { delete (a.objects[1] as Bag).fields.blocked_by; }, + 'dangling lookup target': (a) => { field(a, 'task', 'blocked_by').reference = 'no_such_object'; }, + }, + 'records-forms.master-detail-inline-edit': { + 'master-detail not edited inline': (a) => { delete field(a, 'task', 'project').inlineEdit; }, + 'dangling master-detail target': (a) => { field(a, 'task', 'project').reference = 'no_such_object'; }, + }, + 'records-forms.roll-up-summary': { + 'no roll-up': (a) => { + delete (a.objects[0] as Bag).fields.task_count; + delete (a.objects[0] as Bag).fields.total_estimate; + }, + 'roll-up over a missing child': (a) => { field(a, 'project', 'total_estimate').summaryOperations.object = 'no_such_object'; }, + 'roll-up over a missing child field': (a) => { field(a, 'project', 'total_estimate').summaryOperations.field = 'no_such_field'; }, + }, + 'records-forms.select-options': { + 'optionless select': (a) => { field(a, 'task', 'status').options = []; }, + }, + 'access-security.field-permissions': { + 'hidden field readable': (a) => { a.permissions[0].fields['task.estimate_hours'].readable = true; }, + 'no permission set': (a) => { a.permissions = []; }, + }, + 'access-security.row-scope': { + 'scoped object public': (a) => { (a.objects[0] as Bag).sharingModel = 'public_read_write'; }, + }, + 'approvals.approval-flow': { + 'no approval node': (a) => { + a.flows[0].nodes = a.flows[0].nodes.filter((n: Bag) => n.type !== 'approval'); + }, + 'approval bound to a missing object': (a) => { a.flows[0].variables[0].objectName = 'no_such_object'; }, + 'approval bound to no object the card names': (a) => { a.flows[0].variables[0].objectName = 'sys_user'; }, + }, + 'automation.record-triggered-flow': { + 'no record trigger': (a) => { a.flows[1].nodes[0].config = {}; }, + 'record trigger on a missing object': (a) => { a.flows[1].nodes[0].config.objectName = 'no_such_object'; }, + }, + 'dashboards.widget-measures': { + 'dangling widget measure': (a) => { a.dashboards[0].widgets[0].values = ['no_such_measure']; }, + 'no dashboard': (a) => { a.dashboards = []; }, + }, +}; + +/** + * A defect that breaks a second capability by construction: the roll-ups read + * their child through the master-detail, so a dangling master-detail also + * leaves them relating to nothing. + */ +const ALSO_FAILS: Record = { + 'dangling master-detail target': ['records-forms.roll-up-summary'], +}; + +/** One defect each, keyed by the action check it must fail. */ +const ACTION_DEGRADATIONS: Record void> = { + 'action:task.delete_task->exists': (a) => { a.actions = a.actions.filter((x: Bag) => x.name !== 'delete_task'); }, + 'action:task.set_task_status->ai-exposed': (a) => { a.actions[0].ai.exposed = false; }, + 'action:task.delete_task->confirm-gated': (a) => { delete a.actions[1].ai.requiresConfirmation; }, +}; + +describe('the closed tag table', () => { + it('names only checklist areas — the closed set the card loader reads from the tree', () => { + const areas = new Set(readdirSync(AREAS_DIR).filter((n) => n.endsWith('.json')).map((n) => n.slice(0, -'.json'.length))); + expect(areas.size).toBeGreaterThan(0); + for (const tag of ALL_TAGS) expect(areas.has(tag.slice(0, tag.indexOf('.'))), tag).toBe(true); + }); + + it('reports a tag with no row unscored, never a pass, and counts it nowhere', () => { + const artifact = richArtifact(); + const card = { ...CARD, capabilities: ['ai.ask-data', 'approvals.anything-else'] }; + const s = scoreScenarioArtifact(artifact, card); + expect(s.tags.map((t) => t.status)).toEqual(['unscored', 'unscored']); + expect(s.unscored).toEqual([ + { tag: 'ai.ask-data', reason: 'no check in the closed tag table' }, + { tag: 'approvals.anything-else', reason: 'no check in the closed tag table' }, + ]); + // Not an area prefix: `approvals.approval-flow` passes on this build, yet a + // sibling tag in its area is still unscored. + const withRow = scoreScenarioArtifact(artifact, { ...CARD, capabilities: [] }); + expect(s.total).toBe(withRow.total); + }); + + it('reports a row with nothing in the card to judge unscored, not a pass', () => { + const verdicts = gradeCapabilityTags(readArtifactFacts(richArtifact()), { + ...CARD, + capabilities: ['access-security.field-permissions', 'access-security.row-scope'], + run: { ...CARD.run, personas: [{ id: 'p', role: 'Partner', hidden: { objects: ['project'] } }] }, + }); + expect(verdicts.map((v) => [v.status, v.checks.length])).toEqual([['unscored', 0], ['unscored', 0]]); + }); +}); + +describe('capability tags — discrimination over a rich build', () => { + const intact = scoreScenarioArtifact(richArtifact(), CARD); + + it('passes every row of the table, and every agent action, on the intact build', () => { + expect(intact.tags.map((t) => [t.tag, t.status])).toEqual(ALL_TAGS.map((t) => [t, 'pass'])); + expect(intact.agentActions.every((c) => c.pass)).toBe(true); + expect(intact.agentActions).toHaveLength(5); + expect(intact.score).toBe(1); + }); + + for (const [tag, variants] of Object.entries(TAG_DEGRADATIONS)) { + for (const [name, degrade] of Object.entries(variants)) { + it(`${tag}: ${name} fails that tag (and only what it breaks) and scores lower`, () => { + const artifact = richArtifact(); + degrade(artifact); + const s = scoreScenarioArtifact(artifact, CARD); + const failed = s.tags.filter((t) => t.status !== 'pass').map((t) => t.tag); + expect(failed.sort()).toEqual([tag, ...(ALSO_FAILS[name] ?? [])].sort()); + expect(s.score!).toBeLessThan(intact.score!); + }); + } + } + + it('has a degradation for every row of the table', () => { + expect(Object.keys(TAG_DEGRADATIONS).sort()).toEqual([...ALL_TAGS].sort()); + }); +}); + +describe('agent actions — a declared action the build does not expose fails', () => { + for (const [target, degrade] of Object.entries(ACTION_DEGRADATIONS)) { + it(`${target} fails on its own defect`, () => { + const artifact = richArtifact(); + degrade(artifact); + const s = scoreScenarioArtifact(artifact, CARD); + expect(s.agentActions.find((c) => c.check === target)?.pass).toBe(false); + expect(s.score!).toBeLessThan(1); + }); + } + + it('fails every check of an action the build lacks — none is skipped', () => { + const artifact = richArtifact(); + artifact.actions = []; + const s = scoreScenarioArtifact(artifact, CARD); + expect(s.agentActions.map((c) => [c.check, c.pass])).toEqual([ + ['action:task.set_task_status->exists', false], + ['action:task.set_task_status->ai-exposed', false], + ['action:task.delete_task->exists', false], + ['action:task.delete_task->ai-exposed', false], + ['action:task.delete_task->confirm-gated', false], + ]); + }); + + it('reads the declared confirmation flag, not the delete-mode listing heuristic', () => { + const artifact = richArtifact(); + delete artifact.actions[1].ai.requiresConfirmation; + artifact.actions[1].mode = 'delete'; + const s = scoreScenarioArtifact(artifact, CARD); + expect(s.agentActions.find((c) => c.check === 'action:task.delete_task->confirm-gated')?.pass).toBe(false); + }); + + it('finds an action written on its object, and lets it win over a standalone one of the same name', () => { + const artifact = richArtifact(); + const task = (artifact.objects as Bag[]).find((o) => o.name === 'task')!; + task.actions = [structuredClone(artifact.actions[1])]; + artifact.actions[1].ai.exposed = false; + const s = scoreScenarioArtifact(artifact, CARD); + expect(s.agentActions.find((c) => c.check === 'action:task.delete_task->ai-exposed')?.pass).toBe(true); + }); +}); + +describe('a multi-package artifact (collections only under packages[])', () => { + it('scores exactly as the single-package one', () => { + const artifact = richArtifact(); + const { manifest, ...collections } = artifact; + const option = { manifest, packages: [{ manifest: { ...manifest, ...collections } }] }; + const single = scoreScenarioArtifact(artifact, CARD); + const multi = scoreScenarioArtifact(option, CARD); + expect(multi.tags).toEqual(single.tags); + expect(multi.agentActions).toEqual(single.agentActions); + expect(multi.score).toBe(single.score); + }); +}); diff --git a/packages/qa/scenarios/vitest.config.ts b/packages/qa/scenarios/vitest.config.ts index 355a7ef2ecd..8c94db65623 100644 --- a/packages/qa/scenarios/vitest.config.ts +++ b/packages/qa/scenarios/vitest.config.ts @@ -27,6 +27,12 @@ export default defineConfig({ replacement: path.join(path.resolve(__dirname, '..', '..'), 'spec/src/$1/index.ts'), }, { find: /^@objectstack\/spec$/, replacement: path.resolve(__dirname, '../../spec/src/index.ts') }, + // The scorer's three readers (`src/score/`): the stack-collection rule, + // the standalone-action owner and the effective sharing model are judged + // as the checkout states them, never as the last build did. + { find: /^@objectstack\/core$/, replacement: path.resolve(__dirname, '../../core/src/index.ts') }, + { find: /^@objectstack\/objectql$/, replacement: path.resolve(__dirname, '../../objectql/src/index.ts') }, + { find: /^@objectstack\/plugin-sharing$/, replacement: path.resolve(__dirname, '../../plugins/plugin-sharing/src/index.ts') }, ], }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dcf0637e4a4..3c6519037e4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2481,6 +2481,15 @@ importers: packages/qa/scenarios: dependencies: + '@objectstack/core': + specifier: workspace:* + version: link:../../core + '@objectstack/objectql': + specifier: workspace:* + version: link:../../objectql + '@objectstack/plugin-sharing': + specifier: workspace:* + version: link:../../plugins/plugin-sharing '@objectstack/spec': specifier: workspace:* version: link:../../spec diff --git a/scripts/cross-package-test-inputs.mjs b/scripts/cross-package-test-inputs.mjs index 297f32ae141..fd12726cf4e 100644 --- a/scripts/cross-package-test-inputs.mjs +++ b/scripts/cross-package-test-inputs.mjs @@ -1365,7 +1365,13 @@ export const CROSS_PACKAGE_TEST_INPUTS = { // suite's verdict while touching nothing in the package. The capability // area closed set is the basenames of the checklist's area files, read from // the tree by every suite here, so adding or renaming an area does too. - globs: ['docs/qa/scenarios/*.json', 'docs/qa/platform-checklist/areas/*.json'], + // + // `corpus.ts` is the CLI's golden `os lint --eval` corpus, imported by the + // scorer's suites under test/score/ (#30291): the golden builds the + // discrimination tests degrade are its fixtures, and scenario-cards.test.ts + // holds each control card's `expect` equal to what that corpus derives, so + // a corpus edit moves this suite's verdict. + globs: ['docs/qa/scenarios/*.json', 'docs/qa/platform-checklist/areas/*.json', 'packages/cli/src/lint/corpus.ts'], heldBy: { // The areas directory is listed (`readdirSync`) and its basenames taken, // so the roster holds the directory and no `*.json` name under it. diff --git a/turbo.json b/turbo.json index 6151ccd8056..3ee6edd0eef 100644 --- a/turbo.json +++ b/turbo.json @@ -711,7 +711,8 @@ "!.turbo/**", "!**/node_modules/**", "$TURBO_ROOT$/docs/qa/scenarios/*.json", - "$TURBO_ROOT$/docs/qa/platform-checklist/areas/*.json" + "$TURBO_ROOT$/docs/qa/platform-checklist/areas/*.json", + "$TURBO_ROOT$/packages/cli/src/lint/corpus.ts" ] }, "create-objectstack#test": {