From 2ae2104b1147d486bd31143e54cfc796f9d596d2 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Sat, 26 Sep 2026 18:35:46 +0000 Subject: [PATCH 1/7] feat(docs): let a tutorial page write files and leave a service running test-file writes a block the page shows as a whole file, test-background keeps a fence running beside the journey until its ready URL answers, and test-cwd names the directory the page tells the reader to return to. Every process a journey starts is stopped when it ends, passing or failing. Signed-off-by: Jeremi Joslin --- docs/site/scripts/run-tutorial.mjs | 72 +++++++++-- .../scripts/tutorial-runner/background.mjs | 99 +++++++++++++++ docs/site/scripts/tutorial-runner/page.mjs | 59 ++++++++- .../scripts/tutorial-runner/page.test.mjs | 57 ++++++++- .../tutorial-runner/run-tutorial.test.mjs | 118 ++++++++++++++++++ 5 files changed, 390 insertions(+), 15 deletions(-) create mode 100644 docs/site/scripts/tutorial-runner/background.mjs diff --git a/docs/site/scripts/run-tutorial.mjs b/docs/site/scripts/run-tutorial.mjs index f068f753b..a11326d1f 100644 --- a/docs/site/scripts/run-tutorial.mjs +++ b/docs/site/scripts/run-tutorial.mjs @@ -21,8 +21,15 @@ // tutorial_test.checkout, the reader directory starts as a copy of this // checkout instead (tutorial-runner/checkout.mjs). // +// A test-file block writes its file from the shell's current directory. A +// test-background fence runs beside the journey until its ready URL answers, +// and stays running until the next background fence starts or its page ends +// (tutorial-runner/background.mjs). test-cwd moves the shell to a directory +// under the reader directory before its fence runs. +// // The toolset puts the product binaries under test on PATH and stops any -// service the journey left running, whether it passed or failed. +// service the journey left running, whether it passed or failed. Any process +// the journey started and left behind is stopped when it ends. // // With --gate, the pages come from their own frontmatter instead of the // command line (tutorial-runner/gate.mjs): every page under start/ or @@ -40,6 +47,7 @@ import { tmpdir } from 'node:os'; import { basename, dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { stopGroup } from './tutorial-runner/background.mjs'; import { checkExcerpt } from './tutorial-runner/excerpt.mjs'; import { checkExpectation } from './tutorial-runner/expect.mjs'; import { copyCheckout } from './tutorial-runner/checkout.mjs'; @@ -51,6 +59,7 @@ const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const USAGE = 'usage: run-tutorial.mjs [--dry-run] [--toolset breg|casework|none] ...\n run-tutorial.mjs [--dry-run] --gate breg|casework'; const DOCS_ROOT = process.env.TUTORIAL_DOCS_ROOT ?? resolve(dirname(fileURLToPath(import.meta.url)), '../src/content/docs'); const APPLY_EDIT = join(dirname(fileURLToPath(import.meta.url)), 'tutorial-runner/apply-edit.mjs'); +const BACKGROUND = join(dirname(fileURLToPath(import.meta.url)), 'tutorial-runner/background.mjs'); function usageError(message) { console.error(`${message}\n${USAGE}`); @@ -78,7 +87,7 @@ function parseArgs(argv) { const at = (step) => `${step.page ? `${step.page} ` : ''}line ${step.line}`; const where = (step) => `${at(step)}${step.heading ? ` (${step.heading})` : ''}`; -const BLOCK_NAMES = { edit: 'the edit', excerpt: 'the excerpt' }; +const BLOCK_NAMES = { edit: 'the edit', excerpt: 'the excerpt', file: 'the file' }; const blockName = (step) => BLOCK_NAMES[step.kind] ?? 'the sh fence'; const quote = (text) => `'${text.replaceAll("'", "'\\''")}'`; const outName = (index) => `${String(index).padStart(3, '0')}.out`; @@ -86,10 +95,15 @@ const outName = (index) => `${String(index).padStart(3, '0')}.out`; function printPlan(steps, checkout) { if (checkout) console.log('start in a copy of the checkout'); for (const step of steps) { - const exit = step.exit === undefined ? '' : ` (expects exit ${step.exit})`; - if (step.kind === 'run') console.log(`run ${where(step)}: ${step.code.split('\n')[0]}${exit}`); + const notes = []; + if (step.cwd) notes.push(`in ${step.cwd}`); + if (step.exit !== undefined) notes.push(`expects exit ${step.exit}`); + if (step.background) notes.push(`in the background until ${step.background} answers`); + const note = notes.length > 0 ? ` (${notes.join(', ')})` : ''; + if (step.kind === 'run') console.log(`run ${where(step)}: ${step.code.split('\n')[0]}${note}`); else if (step.kind === 'skip') console.log(`skip ${where(step)}: ${step.reason}`); else if (step.kind === 'edit') console.log(`edit ${where(step)}: ${step.path}`); + else if (step.kind === 'file') console.log(`file ${where(step)}: ${step.path}`); else if (step.kind === 'excerpt' && step.path) console.log(`excerpt ${at(step)}: ${step.path}`); else if (step.kind === 'excerpt') console.log(`excerpt ${at(step)}: checks line ${steps[step.runIndex].line}`); else console.log(`expect ${at(step)}: checks line ${steps[step.runIndex].line}`); @@ -110,10 +124,17 @@ function printPlan(steps, checkout) { // reaches its end leaves `page-N.done`; one whose fence ran `exit` does not, // and the journey stops there. // +// A background fence starts with job control on, which gives it a process +// group of its own that can be stopped whole. The file `background` holds the +// group and output file of the one running now; `backgrounds` lists every +// group started, for the cleanup after a journey that stopped early. +// // The script names every harness file by its literal path and sets no shell // variable, so a page's own variables neither see nor clobber the harness. -async function journeyScript(pages, outDir) { +async function journeyScript(pages, outDir, readerDir) { const file = (name) => quote(join(outDir, name)); + const node = quote(process.execPath); + const stopBackground = `${node} ${quote(BACKGROUND)} stop ${file('background')}`; const lines = ['set -euo pipefail', "trap 'exit 130' HUP INT TERM"]; let index = 0; for (const [pageIndex, steps] of pages.entries()) { @@ -121,6 +142,8 @@ async function journeyScript(pages, outDir) { for (const step of steps) { const out = file(outName(index)); lines.push(`printf '%s\n' ${index} >${file('current')}`); + const cd = step.cwd ? `cd -- ${quote(join(readerDir, step.cwd))}` : undefined; + if (cd && !step.background) lines.push(cd); if (step.kind === 'skip') { lines.push(`printf '%s\\n' ${quote(`skip ${where(step)}: ${step.reason}`)}`); } else if (step.kind === 'edit') { @@ -133,6 +156,23 @@ async function journeyScript(pages, outDir) { lines.push(`printf '\\n%s\\n' ${quote(`==> ${where(step)}`)}`); lines.push(`cat -- ${quote(step.path)} >${out} 2>&1 ${where(step)}`)}`); + lines.push(`cp -- ${quote(staged)} ${quote(step.path)} >${out} 2>&1 ${where(step)} (background)`)}`); + lines.push(`: >${out}`, 'set -m'); + // The other terminal's directory is its own, so a test-cwd here leaves + // the reader's shell where it stands. + lines.push(`{\n${cd ? `${cd}\n` : ''}${step.code}\n} >>${out} 2>&1 ${file('background')}`); + lines.push(`printf '%s\\n' "$!" >>${file('backgrounds')}`, 'set +m'); + lines.push(`${node} ${quote(BACKGROUND)} ready ${quote(step.background)} ${file('background')} >>${out} 2>&1`); + lines.push(`printf 'ready: %s\\n' ${quote(step.background)}`); } else if (step.kind === 'run' && step.exit === undefined) { lines.push(`printf '\\n%s\\n' ${quote(`==> ${where(step)}`)}`); lines.push(`{\n${step.code}\n} >${out} 2>&1 ${done}`, ')', `[[ -e ${done} ]] || exit 0`); + lines.push(stopBackground, `: >${done}`, ')', `[[ -e ${done} ]] || exit 0`); } lines.push(`printf "\\n" >${file('complete')}`); return `${lines.join('\n')}\n`; @@ -173,7 +213,10 @@ function runScript(scriptPath, readerDir, binDir, onSpawn) { if (child.exitCode === null && child.signalCode === null) process.kill(-child.pid, signal); }); child.on('error', reject); - child.on('close', (code, signal) => resolvePromise(signal ? 130 : code)); + child.on('close', (code, signal) => { + // Whatever the journey started and left behind is stopped with it. + stopGroup(child.pid).then(() => resolvePromise(signal ? 130 : code), reject); + }); }); } @@ -233,6 +276,18 @@ async function checkBlocks(steps, outDir) { return failures; } +// Stop every background fence a journey that ended early left running. +async function stopBackgrounds(outDir) { + let text; + try { + text = await readFile(join(outDir, 'backgrounds'), 'utf8'); + } catch (error) { + if (error.code === 'ENOENT') return; + throw error; + } + for (const group of text.trim().split('\n').filter(Boolean)) await stopGroup(Number(group)); +} + async function replay(pages, toolset, checkout) { const steps = pages.flat(); const workRoot = await realpath(await mkdtemp(join(tmpdir(), 'tutorial-run.'))); @@ -257,7 +312,7 @@ async function replay(pages, toolset, checkout) { prepared = true; if (checkout) await copyCheckout(REPO_ROOT, readerDir); const scriptPath = join(workRoot, 'journey.sh'); - await writeFile(scriptPath, await journeyScript(pages, outDir)); + await writeFile(scriptPath, await journeyScript(pages, outDir, readerDir)); const code = await runScript(scriptPath, readerDir, binDir, (send) => { signalJourney = send; }); @@ -288,6 +343,7 @@ async function replay(pages, toolset, checkout) { } finally { for (const signal of signals) process.off(signal, onSignal); try { + await stopBackgrounds(outDir); unlock(workRoot); const stoppedAll = prepared ? await toolset.teardown({ readerDir, binDir }) : true; if (stoppedAll) { diff --git a/docs/site/scripts/tutorial-runner/background.mjs b/docs/site/scripts/tutorial-runner/background.mjs new file mode 100644 index 000000000..672abd175 --- /dev/null +++ b/docs/site/scripts/tutorial-runner/background.mjs @@ -0,0 +1,99 @@ +#!/usr/bin/env node +// The command a page leaves running in another terminal (test-background). +// +// The journey script starts a background fence in its own process group and +// records ` ` in a state file. This helper then waits for +// the fence's ready URL, or stops the group the state file names: +// +// background.mjs ready wait until the URL answers +// background.mjs stop stop the group, show its output +// +// A fence whose group ends before its URL answers, or that does not answer +// within READY_TIMEOUT_MS, fails the wait. + +import { readFile, writeFile } from 'node:fs/promises'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { pathToFileURL } from 'node:url'; + +const READY_TIMEOUT_MS = 60_000; +const STOP_GRACE_MS = 10_000; + +export function groupAlive(group) { + try { + process.kill(-group, 0); + return true; + } catch (error) { + if (error.code === 'ESRCH') return false; + throw error; + } +} + +// Send TERM to the process group, and KILL once the grace period has passed. +// Returns once no process of the group is left, so a port it held is free. +export async function stopGroup(group) { + if (!groupAlive(group)) return; + process.kill(-group, 'SIGTERM'); + const deadline = Date.now() + STOP_GRACE_MS; + while (groupAlive(group)) { + if (Date.now() > deadline) { + process.kill(-group, 'SIGKILL'); + while (groupAlive(group)) await sleep(50); + return; + } + await sleep(50); + } +} + +async function readState(stateFile) { + let text; + try { + text = (await readFile(stateFile, 'utf8')).trim(); + } catch (error) { + if (error.code === 'ENOENT') return undefined; + throw error; + } + if (text === '') return undefined; + const [group, output] = text.split('\t'); + return { group: Number(group), output }; +} + +async function ready(url, stateFile) { + const { group } = await readState(stateFile); + const deadline = Date.now() + READY_TIMEOUT_MS; + for (;;) { + try { + await fetch(url, { signal: AbortSignal.timeout(2000) }); + return 0; + } catch { + // Not answering yet: the command may still be starting. + } + if (!groupAlive(group)) { + console.error(`the command ended before ${url} answered`); + return 1; + } + if (Date.now() > deadline) { + console.error(`${url} did not answer within ${READY_TIMEOUT_MS / 1000} seconds`); + return 1; + } + await sleep(100); + } +} + +async function stop(stateFile) { + const state = await readState(stateFile); + if (!state) return 0; + await stopGroup(state.group); + const output = await readFile(state.output, 'utf8'); + console.log(`\nstopped the background command${output === '' ? '' : ', which printed:'}`); + process.stdout.write(output); + await writeFile(stateFile, ''); + return 0; +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + const [command, ...args] = process.argv.slice(2); + if (command === 'ready' && args.length === 2) process.exit(await ready(...args)); + if (command === 'stop' && args.length === 1) process.exit(await stop(...args)); + console.error('usage: background.mjs ready | stop '); + process.exit(2); +} diff --git a/docs/site/scripts/tutorial-runner/page.mjs b/docs/site/scripts/tutorial-runner/page.mjs index cb778c9d0..427c943bb 100644 --- a/docs/site/scripts/tutorial-runner/page.mjs +++ b/docs/site/scripts/tutorial-runner/page.mjs @@ -17,6 +17,14 @@ // stands at that point of the journey; bare, part of the output of the // nearest sh fence above it. // +// A block titled with a file path and marked test-file is the whole file the +// page asks the reader to create or replace in their editor. +// +// test-background="" on an sh fence is a command the page leaves running +// while the reader carries on in another terminal: the journey goes on once +// the URL answers. test-cwd="" on an sh fence names the directory, inside +// the reader directory, that the page tells the reader to return to first. +// // The page is parsed as Markdown rather than MDX, as check-draft-links.mjs // does: fences, including fences indented inside list items, parse the same // way, and JSX or comment lines read as paragraphs the journey never runs. @@ -26,7 +34,16 @@ import remarkParse from 'remark-parse'; import { unified } from 'unified'; const parser = unified().use(remarkParse).use(remarkGfm); -const ANNOTATIONS = new Set(['test-skip', 'test-expect', 'test-exit', 'test-edit', 'test-excerpt']); +const ANNOTATIONS = new Set([ + 'test-skip', + 'test-expect', + 'test-exit', + 'test-edit', + 'test-excerpt', + 'test-file', + 'test-background', + 'test-cwd', +]); // Parse the test- tokens of a fence meta string: bare flags and key="value" // pairs. Other tokens belong to Expressive Code and are ignored here. @@ -82,10 +99,11 @@ function* walk(node) { } // Return { steps, errors }. A step is one of -// { kind: 'run', line, heading, code, exit? } +// { kind: 'run', line, heading, code, exit?, background?, cwd? } // { kind: 'skip', line, heading, code, reason } // { kind: 'expect', line, heading, format, text, runIndex } // { kind: 'edit', line, heading, path, before, after } +// { kind: 'file', line, heading, path, text } // { kind: 'excerpt', line, heading, format, text, runIndex } // { kind: 'excerpt', line, heading, format, text, path } // where runIndex is the index in steps of the fence whose output it checks. @@ -110,6 +128,9 @@ export function readJourney(text) { const exit = annotations['test-exit']; const edit = annotations['test-edit']; const excerpt = annotations['test-excerpt']; + const file = annotations['test-file']; + const background = annotations['test-background']; + const cwd = annotations['test-cwd']; if (node.lang === 'sh') { if (expect !== undefined) { @@ -122,18 +143,48 @@ export function readJourney(text) { if (excerpt !== undefined) { errors.push(`line ${line}: test-excerpt belongs on a block the page shows, not on an sh fence`); } + if (file !== undefined) errors.push(`line ${line}: test-file belongs on a block showing the file, not on an sh fence`); const step = { kind: skip === undefined ? 'run' : 'skip', line, heading, code: node.value }; if (typeof skip === 'string' && skip !== '') step.reason = skip; if (exit !== undefined) { if (typeof exit === 'string' && /^\d+$/u.test(exit)) step.exit = Number(exit); else errors.push(`line ${line}: test-exit takes an exit status, as test-exit="1"`); } + if (background !== undefined) { + if (typeof background !== 'string' || !/^https?:\/\/\S+$/u.test(background)) { + errors.push( + `line ${line}: test-background takes the URL that answers once the command is ready, as test-background="http://127.0.0.1:4010/"`, + ); + } else if (exit !== undefined) { + errors.push(`line ${line}: a test-background fence keeps running, so it cannot also be test-exit`); + } else if (skip !== undefined) { + errors.push(`line ${line}: a test-background fence is run, so it cannot also be test-skip`); + } else step.background = background; + } + if (cwd !== undefined) { + if (typeof cwd !== 'string' || cwd === '' || cwd.startsWith('/') || cwd.split('/').includes('..')) { + errors.push(`line ${line}: test-cwd takes a directory inside the reader directory, as test-cwd="first-project"`); + } else step.cwd = cwd; + } lastCommand = steps.push(step) - 1; continue; } if (skip !== undefined) errors.push(`line ${line}: test-skip applies only to sh fences`); if (exit !== undefined) errors.push(`line ${line}: test-exit applies only to sh fences`); + if (background !== undefined) errors.push(`line ${line}: test-background applies only to sh fences`); + if (cwd !== undefined) errors.push(`line ${line}: test-cwd applies only to sh fences`); + if ([file, edit, expect, excerpt].filter((value) => value !== undefined).length > 1) { + errors.push(`line ${line}: a block is one of test-file, test-edit, test-expect, or test-excerpt`); + continue; + } + if (file !== undefined) { + const path = titleOf(node.meta); + if (node.lang === 'diff') errors.push(`line ${line}: test-file takes the whole file; a diff block is test-edit`); + else if (!path) errors.push(`line ${line}: test-file needs the file path, as title=""`); + else steps.push({ kind: 'file', line, heading, path, text: `${node.value}\n` }); + continue; + } if (edit !== undefined) { const path = titleOf(node.meta); const sides = node.lang === 'diff' ? diffSides(node.value) : undefined; @@ -153,10 +204,6 @@ export function readJourney(text) { continue; } if (expect === undefined && excerpt === undefined) continue; - if (expect !== undefined && excerpt !== undefined) { - errors.push(`line ${line}: a block is either test-expect or test-excerpt, not both`); - continue; - } const format = node.lang === 'json' ? 'json' : 'text'; if (typeof excerpt === 'string' && excerpt !== '') { steps.push({ kind: 'excerpt', line, heading, format, text: node.value, path: excerpt }); diff --git a/docs/site/scripts/tutorial-runner/page.test.mjs b/docs/site/scripts/tutorial-runner/page.test.mjs index f11e625dd..885f8f1a3 100644 --- a/docs/site/scripts/tutorial-runner/page.test.mjs +++ b/docs/site/scripts/tutorial-runner/page.test.mjs @@ -250,6 +250,61 @@ test('test-excerpt mistakes are errors that name the line', () => { 'line 1: test-excerpt has no sh fence above it to check', 'line 5: test-excerpt belongs on a block the page shows, not on an sh fence', 'line 13: test-excerpt checks the output of the sh fence at line 9, which is skipped', - 'line 17: a block is either test-expect or test-excerpt, not both', + 'line 17: a block is one of test-file, test-edit, test-expect, or test-excerpt', + ]); +}); + +test('a titled block marked test-file is the whole file the page asks the reader to create', () => { + const { steps, errors } = readJourney( + '## Ask\n\nOpen `questions/q.yaml` and add:\n\n```yaml title="questions/q.yaml" test-file\nid: q\n```\n', + ); + assert.deepEqual(errors, []); + assert.deepEqual(steps, [{ kind: 'file', line: 5, heading: 'Ask', path: 'questions/q.yaml', text: 'id: q\n' }]); +}); + +test('test-file mistakes are errors that name the line', () => { + const { errors } = readJourney( + '```yaml test-file\na: 1\n```\n\n```sh title="x.sh" test-file\ntrue\n```\n\n' + + '```diff title="a.yaml" test-file\n+a\n```\n\n```yaml title="a.yaml" test-file test-excerpt\na\n```\n', + ); + assert.deepEqual(errors, [ + 'line 1: test-file needs the file path, as title=""', + 'line 5: test-file belongs on a block showing the file, not on an sh fence', + 'line 9: test-file takes the whole file; a diff block is test-edit', + 'line 13: a block is one of test-file, test-edit, test-expect, or test-excerpt', + ]); +}); + +test('test-background leaves an sh fence running until its ready URL answers, and test-cwd says where a fence runs', () => { + const { steps, errors } = readJourney( + '```sh test-background="http://127.0.0.1:4010/health" test-cwd="first"\nserve\n```\n\n' + + '```text test-expect\nready\n```\n\n```sh test-cwd="first/project"\nls\n```\n', + ); + assert.deepEqual(errors, []); + assert.deepEqual(steps[0], { + kind: 'run', + line: 1, + heading: '', + code: 'serve', + background: 'http://127.0.0.1:4010/health', + cwd: 'first', + }); + assert.equal(steps[1].runIndex, 0); + assert.equal(steps[2].cwd, 'first/project'); +}); + +test('test-background and test-cwd mistakes are errors that name the line', () => { + const { errors } = readJourney( + '```sh test-background\nserve\n```\n\n```sh test-background="http://x/" test-exit="1"\nserve\n```\n\n' + + '```sh test-background="http://x/" test-skip="offline"\nserve\n```\n\n' + + '```sh test-cwd="/abs"\nls\n```\n\n```sh test-cwd="../up"\nls\n```\n\n```text test-cwd="a"\nx\n```\n', + ); + assert.deepEqual(errors, [ + 'line 1: test-background takes the URL that answers once the command is ready, as test-background="http://127.0.0.1:4010/"', + 'line 5: a test-background fence keeps running, so it cannot also be test-exit', + 'line 9: a test-background fence is run, so it cannot also be test-skip', + 'line 13: test-cwd takes a directory inside the reader directory, as test-cwd="first-project"', + 'line 17: test-cwd takes a directory inside the reader directory, as test-cwd="first-project"', + 'line 21: test-cwd applies only to sh fences', ]); }); diff --git a/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs b/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs index 213681aff..9fa2d4437 100644 --- a/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs +++ b/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import { execFile, spawn } from 'node:child_process'; import { existsSync } from 'node:fs'; import { chmod, mkdir, mkdtemp, readdir, readFile, realpath, rm, writeFile } from 'node:fs/promises'; +import { createServer } from 'node:net'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import test from 'node:test'; @@ -472,3 +473,120 @@ test('a gate dry run builds nothing and starts nothing', async () => { assert.equal(existsSync(join(bin, 'calls')), false, output); }); }); + +// A loopback port nothing listens on right now. +async function freePort() { + const server = createServer(); + await new Promise((resolvePromise) => server.listen(0, '127.0.0.1', resolvePromise)); + const { port } = server.address(); + await new Promise((resolvePromise) => server.close(resolvePromise)); + return port; +} + +const alive = (pid) => { + try { + process.kill(pid, 0); + return true; + } catch (error) { + if (error.code === 'ESRCH') return false; + throw error; + } +}; + +test('a test-file block writes the whole file where the reader stands', async () => { + const body = + '## Write\n\n' + + fence('sh', 'mkdir -p work/questions\ncd work\necho stale >questions/q.yaml') + + 'Open `questions/q.yaml` and add:\n\n' + + fence('yaml title="questions/q.yaml" test-file', 'id: q\npurpose: check') + + fence('sh', 'cat questions/q.yaml') + + fence('text test-expect', 'id: q\npurpose: check'); + await withPage(body, async ({ page }) => { + const { code, output } = await run([page]); + assert.equal(code, 0, output); + assert.match(output, /wrote questions\/q\.yaml/u); + assert.match(output, /tutorial PASS/u); + const plan = await run(['--dry-run', page]); + assert.match(plan.output, /file line 15 \(Write\): questions\/q\.yaml/u); + }); +}); + +test('a test-file block whose directory does not exist stops the journey', async () => { + const body = '## Write\n\n' + fence('yaml title="missing/q.yaml" test-file', 'id: q') + fence('sh', 'echo never'); + await withPage(body, async ({ page }) => { + const { code, output } = await run([page]); + assert.equal(code, 1, output); + assert.match(output, /the file at line 7 \(Write\) failed/u); + assert.doesNotMatch(output, /^never$/mu); + }); +}); + +test('a background fence runs until the next one starts or the page ends, and test-cwd says where a fence runs', async () => { + const port = await freePort(); + const url = `http://127.0.0.1:${port}/`; + const serve = `python3 -m http.server ${port} --bind 127.0.0.1`; + const body = + '## Serve\n\n' + + fence('sh', 'mkdir -p site/one site/two other\necho first >site/one/index.html\necho second >site/two/index.html\ncd other') + + fence(`sh test-background="${url}" test-cwd="site/one"`, serve) + + fence('sh', `curl -fsS ${url}\npwd | sed 's|.*/||'`) + + fence('text test-expect', 'first\nother') + + fence(`sh test-background="${url}" test-cwd="site/two"`, serve) + + fence('sh test-cwd="site"', `curl -fsS ${url}\npwd | sed 's|.*/||'`) + + fence('text test-expect', 'second\nsite'); + await withPage(body, async ({ page }) => { + const { code, output } = await run([page]); + assert.equal(code, 0, output); + assert.match(output, /tutorial PASS/u); + await assert.rejects(fetch(url), 'the page end must stop the background fence'); + const plan = await run(['--dry-run', page]); + assert.match(plan.output, new RegExp(`run line 14 \\(Serve\\): ${serve} \\(in site/one, in the background until ${url} answers\\)`, 'u')); + }); +}); + +test('a background fence that ends before it is ready stops the journey and shows what it printed', async () => { + const port = await freePort(); + const body = '## Serve\n\n' + fence(`sh test-background="http://127.0.0.1:${port}/"`, 'echo "port taken" >&2\nfalse') + fence('sh', 'echo never'); + await withPage(body, async ({ page }) => { + const { code, output } = await run([page]); + assert.equal(code, 1, output); + assert.match(output, /port taken/u); + assert.match(output, /the sh fence at line 7 \(Serve\) failed/u); + assert.doesNotMatch(output, /^never$/mu); + }); +}); + +test('an expectation on a background fence checks what it printed while it ran', async () => { + const port = await freePort(); + const url = `http://127.0.0.1:${port}/`; + const body = + '## Serve\n\n' + + fence(`sh test-background="${url}"`, `echo "ready on ${port}"\nexec python3 -m http.server ${port} --bind 127.0.0.1 2>/dev/null`) + + fence('text test-expect', 'ready on ') + + fence('sh', `curl -fsS -o /dev/null ${url}`); + await withPage(body, async ({ page }) => { + const { code, output } = await run([page]); + assert.equal(code, 0, output); + assert.match(output, /expect line 12: ok/u); + }); +}); + +test('a journey that ends, passing or failing, leaves no process it started running', async () => { + const port = await freePort(); + const body = (pids, last) => + '## Start\n\n' + + fence('sh', `sleep 300 &\necho "$!" >>'${pids}'`) + + fence(`sh test-background="http://127.0.0.1:${port}/"`, `echo "$BASHPID" >>'${pids}'\nexec python3 -m http.server ${port} --bind 127.0.0.1`) + + fence('sh', last); + for (const [last, expected] of [['true', 0], ['false', 1]]) { + await withPage('', async ({ dir, page }) => { + const pids = join(dir, 'pids'); + await writeFile(page, `---\ntitle: t\n---\n\n${body(pids, last)}`); + const { code, output } = await run([page]); + assert.equal(code, expected, output); + const started = (await readFile(pids, 'utf8')).trim().split('\n').map(Number); + assert.equal(started.length, 2, output); + for (const pid of started) assert.equal(alive(pid), false, `process ${pid} outlived the journey\n${output}`); + }); + } +}); From 609a3d6b69664bd555d6928e0c9560554bbb06b3 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Sat, 26 Sep 2026 18:40:04 +0000 Subject: [PATCH 2/7] feat(docs): serve Evidence binaries, the Python client, and the FHIR mock to tutorial journeys Signed-off-by: Jeremi Joslin --- docs/site/scripts/run-tutorial.mjs | 10 +- .../scripts/tutorial-runner/background.mjs | 2 +- .../tutorial-runner/run-tutorial.test.mjs | 68 +++++++++ .../site/scripts/tutorial-runner/toolsets.mjs | 142 ++++++++++++++++-- 4 files changed, 205 insertions(+), 17 deletions(-) diff --git a/docs/site/scripts/run-tutorial.mjs b/docs/site/scripts/run-tutorial.mjs index a11326d1f..dc7ad06bd 100644 --- a/docs/site/scripts/run-tutorial.mjs +++ b/docs/site/scripts/run-tutorial.mjs @@ -56,7 +56,7 @@ import { readJourney } from './tutorial-runner/page.mjs'; import { TOOLSETS, ToolsetError } from './tutorial-runner/toolsets.mjs'; const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); -const USAGE = 'usage: run-tutorial.mjs [--dry-run] [--toolset breg|casework|none] ...\n run-tutorial.mjs [--dry-run] --gate breg|casework'; +const USAGE = 'usage: run-tutorial.mjs [--dry-run] [--toolset breg|casework|evidence|none] ...\n run-tutorial.mjs [--dry-run] --gate breg|casework|evidence'; const DOCS_ROOT = process.env.TUTORIAL_DOCS_ROOT ?? resolve(dirname(fileURLToPath(import.meta.url)), '../src/content/docs'); const APPLY_EDIT = join(dirname(fileURLToPath(import.meta.url)), 'tutorial-runner/apply-edit.mjs'); const BACKGROUND = join(dirname(fileURLToPath(import.meta.url)), 'tutorial-runner/background.mjs'); @@ -203,8 +203,8 @@ async function journeyScript(pages, outDir, readerDir) { // command a fence is running and not only the shell waiting for it, which // would run its trap only once that command ended. onSpawn receives a // function that sends a signal to the whole group. -function runScript(scriptPath, readerDir, binDir, onSpawn) { - const env = { ...process.env, PATH: `${binDir}:${process.env.PATH}` }; +function runScript(scriptPath, readerDir, binDir, toolsetEnv, onSpawn) { + const env = { ...process.env, ...toolsetEnv, PATH: `${binDir}:${process.env.PATH}` }; // A reader has no CARGO_TARGET_DIR pointing into this checkout. delete env.CARGO_TARGET_DIR; return new Promise((resolvePromise, reject) => { @@ -308,12 +308,12 @@ async function replay(pages, toolset, checkout) { let status = 1; let prepared = false; try { - await toolset.prepare({ repoRoot: REPO_ROOT, binDir }); + const toolsetEnv = await toolset.prepare({ repoRoot: REPO_ROOT, binDir, workRoot }); prepared = true; if (checkout) await copyCheckout(REPO_ROOT, readerDir); const scriptPath = join(workRoot, 'journey.sh'); await writeFile(scriptPath, await journeyScript(pages, outDir, readerDir)); - const code = await runScript(scriptPath, readerDir, binDir, (send) => { + const code = await runScript(scriptPath, readerDir, binDir, toolsetEnv, (send) => { signalJourney = send; }); if (interrupted || code === 130) { diff --git a/docs/site/scripts/tutorial-runner/background.mjs b/docs/site/scripts/tutorial-runner/background.mjs index 672abd175..a75a7b567 100644 --- a/docs/site/scripts/tutorial-runner/background.mjs +++ b/docs/site/scripts/tutorial-runner/background.mjs @@ -90,7 +90,7 @@ async function stop(stateFile) { return 0; } -if (import.meta.url === pathToFileURL(process.argv[1]).href) { +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { const [command, ...args] = process.argv.slice(2); if (command === 'ready' && args.length === 2) process.exit(await ready(...args)); if (command === 'stop' && args.length === 1) process.exit(await stop(...args)); diff --git a/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs b/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs index 9fa2d4437..3fa78cb38 100644 --- a/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs +++ b/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs @@ -590,3 +590,71 @@ test('a journey that ends, passing or failing, leaves no process it started runn }); } }); + +// A wheel holding one package with no dependencies, as the assembled client is. +async function fakeWheel(dir) { + const wheel = join(dir, 'client.whl'); + await execFileAsync('python3', [ + '-c', + 'import sys, zipfile\nwith zipfile.ZipFile(sys.argv[1], "w") as z: z.writestr("tutorial_fake_client/__init__.py", "NAME = \\"unpacked\\"\\n")', + wheel, + ]); + return wheel; +} + +async function fakeEvidenceBinaries(dir, calls) { + const env = {}; + for (const [name, variable] of [ + ['evidence', 'EVIDENCE_BIN'], + ['evidencectl', 'EVIDENCECTL_BIN'], + ['evidence-oid4vci', 'EVIDENCE_OID4VCI_BIN'], + ]) { + await writeFile(join(dir, name), `#!/bin/sh\nprintf '%s %s\\n' ${name} "$*" >>'${calls}'\n`); + await chmod(join(dir, name), 0o755); + env[variable] = join(dir, name); + } + return env; +} + +test('the evidence toolset serves its binaries, the client package, and the FHIR mock, and stops dev sessions', async () => { + const body = + '## Start\n\n' + + fence( + 'sh', + 'evidence --version\nevidencectl dev\n"$EVIDENCE_OID4VCI_BIN" --version\n' + + 'python3 -c "import tutorial_fake_client; print(tutorial_fake_client.NAME)"\n' + + 'python3 -c "import os, urllib.request; print(urllib.request.urlopen(os.environ[\'FHIR_TUTORIAL_TEST_BASE_URL\'] + \'/healthz\').status)"\n' + + 'mkdir -p work/project/.evidence/dev\ntouch work/project/.evidence/dev/control.sock', + ) + + fence('text test-expect', 'unpacked\n200') + + fence('sh', 'false'); + await withPage(body, async ({ dir, page }) => { + const calls = join(dir, 'calls.log'); + const env = await fakeEvidenceBinaries(dir, calls); + const { code, output } = await run(['--toolset', 'evidence', page], { + ...env, + REGISTRY_CLIENT_PY_WHEEL: await fakeWheel(dir), + }); + assert.equal(code, 1, output); + const log = (await readFile(calls, 'utf8')).trim().split('\n'); + assert.deepEqual(log.slice(0, 3), ['evidence --version', 'evidencectl dev', 'evidence-oid4vci --version']); + assert.match(log[3], /^evidencectl dev stop --project \/\S+\/work\/project$/u); + assert.equal(log.length, 4); + await assert.rejects(fetch('http://127.0.0.1:8003/healthz'), 'the FHIR mock must stop with the journey'); + }); +}); + +test('the evidence toolset needs the client wheel as an absolute path to a file', async () => { + await withPage('## A\n\n' + fence('sh', 'true'), async ({ dir, page }) => { + const env = await fakeEvidenceBinaries(dir, join(dir, 'calls.log')); + for (const [wheel, message] of [ + ['', /REGISTRY_CLIENT_PY_WHEEL is unset: name a client wheel assembled with release\/scripts\/assemble-registry-client-packages\.py/u], + ['client.whl', /REGISTRY_CLIENT_PY_WHEEL must be an absolute path: client\.whl/u], + [join(dir, 'missing.whl'), /client wheel not found: \/\S+\/missing\.whl/u], + ]) { + const { code, output } = await run(['--toolset', 'evidence', page], { ...env, REGISTRY_CLIENT_PY_WHEEL: wheel }); + assert.equal(code, 2, output); + assert.match(output, message); + } + }); +}); diff --git a/docs/site/scripts/tutorial-runner/toolsets.mjs b/docs/site/scripts/tutorial-runner/toolsets.mjs index 4a9f819a9..d2a4ab6a6 100644 --- a/docs/site/scripts/tutorial-runner/toolsets.mjs +++ b/docs/site/scripts/tutorial-runner/toolsets.mjs @@ -5,10 +5,14 @@ // the journey left behind, whether it passed or failed. This is the only // product-specific code in the runner. -import { spawnSync } from 'node:child_process'; -import { accessSync, constants } from 'node:fs'; +import { spawn, spawnSync } from 'node:child_process'; +import { accessSync, closeSync, constants, existsSync, openSync, readFileSync, statSync } from 'node:fs'; import { mkdir, readdir, symlink } from 'node:fs/promises'; -import { dirname, isAbsolute, join } from 'node:path'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { fileURLToPath } from 'node:url'; + +import { stopGroup } from './background.mjs'; export class ToolsetError extends Error {} @@ -24,12 +28,23 @@ function checkBinary(variable, path) { // A product toolset: its binaries, built from this checkout unless every one // of their variables names an exact candidate or released binary, served by // name from binDir; and its local development sessions, stopped in the order -// given when the journey ends. -function productToolset({ label, binaries, cargoArgs, profileVariable, targetName, sessions, ...rest }) { +// given when the journey ends, each with the arguments stopArgs gives for its +// project directory. +function productToolset({ + label, + binaries, + cargoArgs, + profileVariable, + targetName, + sessions, + stopArgs = (project) => ['dev', 'stop', project, '--remove'], + ...rest +}) { const variables = binaries.map(([, variable]) => variable); return { ...rest, + // Returns the variables the journey's environment adds, if any. async prepare({ repoRoot, binDir }) { const given = binaries.map(([, variable]) => process.env[variable] || undefined); if (given.some(Boolean) && !given.every(Boolean)) { @@ -55,12 +70,13 @@ function productToolset({ label, binaries, cargoArgs, profileVariable, targetNam binaries.forEach(([, variable], i) => checkBinary(variable, paths[i])); await mkdir(binDir, { recursive: true }); for (const [i, [name]] of binaries.entries()) await symlink(paths[i], join(binDir, name)); + return {}; }, // Stop every local development session the journey started and reclaim - // its container and volume. A journey that fails halfway leaves its - // sessions running, and deleting the reader directory alone would orphan - // their containers. Stopping with --remove is idempotent. Returns false + // its container and volume, if it has one. A journey that fails halfway + // leaves its sessions running, and deleting the reader directory alone + // would orphan them. Stopping is idempotent. Returns false // when a session could not be stopped, so its project is kept for a second // attempt. async teardown({ readerDir, binDir }) { @@ -75,7 +91,7 @@ function productToolset({ label, binaries, cargoArgs, profileVariable, targetNam for (const [tool, state] of sessions) { for (const entry of entries.filter((path) => path.endsWith(state)).sort()) { const project = dirname(dirname(dirname(join(readerDir, entry)))); - const stop = spawnSync(join(binDir, tool), ['dev', 'stop', project, '--remove'], { encoding: 'utf8' }); + const stop = spawnSync(join(binDir, tool), stopArgs(project), { encoding: 'utf8' }); if (stop.status === 0) { console.log(`stopped the local development session in ${project}`); } else { @@ -129,12 +145,116 @@ const casework = productToolset({ ], }); +// Evidence: evidence, evidencectl, and evidence-oid4vci. Two things a reader +// sets up themselves are set up here instead: +// +// - The Python client package a reader installs with pip is unpacked from +// REGISTRY_CLIENT_PY_WHEEL, a wheel assembled from this checkout with +// release/scripts/assemble-registry-client-packages.py, onto PYTHONPATH. +// The package declares no dependencies, so unpacking is enough, and its +// bindings are built for the stable ABI, so it imports under any CPython. +// - The FHIR server a page reads is replaced by the sanitized local mock in +// fixtures/fhir-tutorial-mock.py, at FHIR_TUTORIAL_TEST_BASE_URL, for every +// journey. +// +// EVIDENCE_OID4VCI_BIN names the served evidence-oid4vci for the +// interoperability checks, which also take EVIDENCE_OID4VCI_INTEROP_TEST_BIN +// from the environment when it names a prebuilt interoperability test. +const FHIR_MOCK = resolve(dirname(fileURLToPath(import.meta.url)), '../fixtures/fhir-tutorial-mock.py'); +const FHIR_BASE_URL = 'http://127.0.0.1:8003'; +const FHIR_READY_TIMEOUT_MS = 30_000; +const WHEEL_HINT = 'release/scripts/assemble-registry-client-packages.py'; + +function clientWheel() { + const wheel = process.env.REGISTRY_CLIENT_PY_WHEEL ?? ''; + if (wheel === '') throw new ToolsetError(`REGISTRY_CLIENT_PY_WHEEL is unset: name a client wheel assembled with ${WHEEL_HINT}`); + if (!isAbsolute(wheel)) throw new ToolsetError(`REGISTRY_CLIENT_PY_WHEEL must be an absolute path: ${wheel}`); + if (!existsSync(wheel) || !statSync(wheel).isFile()) { + throw new ToolsetError(`client wheel not found: ${wheel}; assemble one with ${WHEEL_HINT}`); + } + return wheel; +} + +// Start the FHIR mock in its own process group and wait until it answers. +// Returns its process group. +async function startFhirMock(workRoot) { + const log = join(workRoot, 'fhir-tutorial-mock.log'); + const fd = openSync(log, 'w'); + const child = spawn('python3', [FHIR_MOCK], { detached: true, stdio: ['ignore', fd, fd] }); + closeSync(fd); + const failed = async (why) => { + await stopGroup(child.pid); + return new ToolsetError(`the FHIR tutorial mock ${why}:\n${readFileSync(log, 'utf8')}`); + }; + const deadline = Date.now() + FHIR_READY_TIMEOUT_MS; + for (;;) { + if (child.exitCode !== null || child.signalCode !== null) throw await failed(`ended before ${FHIR_BASE_URL} answered`); + try { + const response = await fetch(`${FHIR_BASE_URL}/healthz`, { signal: AbortSignal.timeout(2000) }); + if (response.ok) break; + } catch { + // Not answering yet: the mock may still be starting. + } + if (Date.now() > deadline) throw await failed(`did not answer within ${FHIR_READY_TIMEOUT_MS / 1000} seconds`); + await sleep(100); + } + // Another process may hold the port and have answered in its place. + await sleep(100); + if (child.exitCode !== null || child.signalCode !== null) throw await failed(`could not serve ${FHIR_BASE_URL}`); + return child.pid; +} + +const evidenceProduct = productToolset({ + label: 'evidence, evidencectl, and evidence-oid4vci', + // Not a path such as .evidence/dev, which a page may name. + commands: /(^|[^\w./-])(evidencectl|evidence-oid4vci|evidence)([^\w-]|$)/mu, + binaries: [ + ['evidence', 'EVIDENCE_BIN'], + ['evidencectl', 'EVIDENCECTL_BIN'], + ['evidence-oid4vci', 'EVIDENCE_OID4VCI_BIN'], + ], + cargoArgs: ['-p', 'registry-evidence', '-p', 'registry-evidencectl', '-p', 'registry-evidence-oid4vci'], + profileVariable: 'EVIDENCE_TUTORIAL_CARGO_PROFILE', + targetName: 'evidence-tutorial-source', + sessions: [['evidencectl', '.evidence/dev/control.sock']], + stopArgs: (project) => ['dev', 'stop', '--project', project], +}); + +let fhirMock; +const evidence = { + ...evidenceProduct, + + async prepare({ repoRoot, binDir, workRoot }) { + const wheel = clientWheel(); + await evidenceProduct.prepare({ repoRoot, binDir }); + const clientPackage = join(workRoot, 'client-package'); + const unpack = spawnSync('python3', ['-m', 'zipfile', '-e', wheel, clientPackage], { encoding: 'utf8' }); + if (unpack.status !== 0) throw new ToolsetError(`could not unpack ${wheel}:\n${unpack.stderr}`); + fhirMock = await startFhirMock(workRoot); + const pythonPath = process.env.PYTHONPATH ? `${clientPackage}:${process.env.PYTHONPATH}` : clientPackage; + return { + PYTHONPATH: pythonPath, + EVIDENCE_OID4VCI_BIN: join(binDir, 'evidence-oid4vci'), + FHIR_TUTORIAL_TEST_BASE_URL: FHIR_BASE_URL, + }; + }, + + async teardown(context) { + const stoppedAll = await evidenceProduct.teardown(context); + if (fhirMock !== undefined) await stopGroup(fhirMock); + fhirMock = undefined; + return stoppedAll; + }, +}; + // No product binaries: the journey runs against what is already on PATH. const none = { - async prepare() {}, + async prepare() { + return {}; + }, async teardown() { return true; }, }; -export const TOOLSETS = { breg, casework, none }; +export const TOOLSETS = { breg, casework, evidence, none }; From cfdc899cefed9ce56bfe6effe8be209d963bd246 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Sat, 26 Sep 2026 18:41:35 +0000 Subject: [PATCH 3/7] feat(docs): gate Evidence tutorials by their commands, including checkout scripts Signed-off-by: Jeremi Joslin --- docs/site/scripts/tutorial-runner/gate.mjs | 9 ++++++++- .../scripts/tutorial-runner/gate.test.mjs | 19 +++++++++++++++++++ .../site/scripts/tutorial-runner/toolsets.mjs | 5 +++-- .../build-and-deploy-evidence-project.mdx | 3 +++ .../connect-an-institution-source.mdx | 3 +++ .../tutorials/deploy-evidence-from-breg.mdx | 3 +++ ...evidence-candidate-with-docker-compose.mdx | 3 +++ ...e-a-birth-certificate-vc-from-opencrvs.mdx | 3 +++ ...issue-immunization-evidence-from-dhis2.mdx | 3 +++ .../manage-evidence-verifier-trust.mdx | 3 +++ .../move-evidence-to-production-signing.mdx | 3 +++ .../tutorials/prove-an-evidence-project.mdx | 3 +++ .../request-a-holder-bound-credential.mdx | 3 +++ .../rotate-evidence-signing-keys.mdx | 3 +++ ...rify-a-registered-parent-with-opencrvs.mdx | 3 +++ 15 files changed, 66 insertions(+), 3 deletions(-) diff --git a/docs/site/scripts/tutorial-runner/gate.mjs b/docs/site/scripts/tutorial-runner/gate.mjs index 16946333c..36c4fadea 100644 --- a/docs/site/scripts/tutorial-runner/gate.mjs +++ b/docs/site/scripts/tutorial-runner/gate.mjs @@ -20,7 +20,9 @@ // run both, and those pages belong to its gate rather than to each. A page that // runs the commands under a toolset that does not serve them, or replays none // of them, is an error too, so neither the declaration nor test-skip can take a -// page out of the gate that owns its commands. +// page out of the gate that owns its commands. The one exception is a page no +// toolset can replay because it runs the commands of two that neither serves +// both: it declares either one, with a skip reason. // A new tutorial therefore fails the gate on the commit that adds it, until it // is replayed or says why not. @@ -115,6 +117,11 @@ export async function planGate(docsRoot, toolset, toolsets) { continue; } if (runs && !serves(declaration.toolset)) { + // A page running the commands of two toolsets that neither serves both + // can only be replayed by neither, so it is skipped in the gate of the + // one it declares. + const runsDeclared = commands.some((code) => toolsets[declaration.toolset].commands?.test(code)); + if (declaration.skip !== undefined && runsDeclared) continue; errors.push( `${slug}.mdx runs ${toolset} commands but declares toolset ${declaration.toolset}, which does not serve them; declare toolset ${toolset}`, ); diff --git a/docs/site/scripts/tutorial-runner/gate.test.mjs b/docs/site/scripts/tutorial-runner/gate.test.mjs index 4e40bf793..aad5dfb9d 100644 --- a/docs/site/scripts/tutorial-runner/gate.test.mjs +++ b/docs/site/scripts/tutorial-runner/gate.test.mjs @@ -149,3 +149,22 @@ test('a page running the toolset under another toolset, or skipping every one of ]); }); }); + +test('a skipped page running the commands of two toolsets, neither serving the other, may declare either', async () => { + const toolsets = { ...TOOLSETS, evidence: { commands: /(^|[^\w./-])(evidencectl|evidence)([^\w-]|$)/mu } }; + const both = '```sh\nbregctl check project\nevidencectl source add\n```\n'; + const pages = { + 'tutorials/composed': { frontmatter: 'tutorial_test:\n toolset: breg\n skip: needs a container\n', body: both }, + 'tutorials/replayed': { frontmatter: 'tutorial_test:\n toolset: breg\n', body: both }, + }; + await withDocs(pages, async (root) => { + const evidence = await planGate(root, 'evidence', toolsets); + assert.deepEqual(evidence.errors, [ + 'tutorials/replayed.mdx runs evidence commands but declares toolset breg, which does not serve them; declare toolset evidence', + ]); + assert.deepEqual(evidence.skipped, []); + const breg = await planGate(root, 'breg', toolsets); + assert.deepEqual(breg.errors, []); + assert.deepEqual(breg.skipped, [{ slug: 'tutorials/composed', reason: 'needs a container' }]); + }); +}); diff --git a/docs/site/scripts/tutorial-runner/toolsets.mjs b/docs/site/scripts/tutorial-runner/toolsets.mjs index d2a4ab6a6..76f929314 100644 --- a/docs/site/scripts/tutorial-runner/toolsets.mjs +++ b/docs/site/scripts/tutorial-runner/toolsets.mjs @@ -206,8 +206,9 @@ async function startFhirMock(workRoot) { const evidenceProduct = productToolset({ label: 'evidence, evidencectl, and evidence-oid4vci', - // Not a path such as .evidence/dev, which a page may name. - commands: /(^|[^\w./-])(evidencectl|evidence-oid4vci|evidence)([^\w-]|$)/mu, + // Not a path such as .evidence/dev, which a page may name; but a page that + // runs Evidence's own checks from a checkout runs Evidence. + commands: /(^|[^\w./-])(evidencectl|evidence-oid4vci|evidence)([^\w-]|$)|(^|\s)products\/evidence\/scripts\//mu, binaries: [ ['evidence', 'EVIDENCE_BIN'], ['evidencectl', 'EVIDENCECTL_BIN'], diff --git a/docs/site/src/content/docs/tutorials/build-and-deploy-evidence-project.mdx b/docs/site/src/content/docs/tutorials/build-and-deploy-evidence-project.mdx index 5f554c71c..95a765507 100644 --- a/docs/site/src/content/docs/tutorials/build-and-deploy-evidence-project.mdx +++ b/docs/site/src/content/docs/tutorials/build-and-deploy-evidence-project.mdx @@ -12,6 +12,9 @@ persona: - operator locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: drift-checked by evidence-production-build-docs.test.mjs; needs a production build environment --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; diff --git a/docs/site/src/content/docs/tutorials/connect-an-institution-source.mdx b/docs/site/src/content/docs/tutorials/connect-an-institution-source.mdx index 8669d9580..6ab5feacc 100644 --- a/docs/site/src/content/docs/tutorials/connect-an-institution-source.mdx +++ b/docs/site/src/content/docs/tutorials/connect-an-institution-source.mdx @@ -11,6 +11,9 @@ persona: - assertion provider locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: how-to against the reader's own OpenAPI source; no fixed scenario to replay --- If this is your first Evidence Gateway project, complete diff --git a/docs/site/src/content/docs/tutorials/deploy-evidence-from-breg.mdx b/docs/site/src/content/docs/tutorials/deploy-evidence-from-breg.mdx index 0cee282fd..3b1dd26e9 100644 --- a/docs/site/src/content/docs/tutorials/deploy-evidence-from-breg.mdx +++ b/docs/site/src/content/docs/tutorials/deploy-evidence-from-breg.mdx @@ -13,6 +13,9 @@ persona: - operator locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: operated target handoff; native composition and production build tests cover offline candidates, target-host checks need provisioned dependencies --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; diff --git a/docs/site/src/content/docs/tutorials/integrate-evidence-candidate-with-docker-compose.mdx b/docs/site/src/content/docs/tutorials/integrate-evidence-candidate-with-docker-compose.mdx index 57db98899..aaf11e2ad 100644 --- a/docs/site/src/content/docs/tutorials/integrate-evidence-candidate-with-docker-compose.mdx +++ b/docs/site/src/content/docs/tutorials/integrate-evidence-candidate-with-docker-compose.mdx @@ -11,6 +11,9 @@ persona: - operator locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: drift-checked by evidence-production-build-docs.test.mjs; needs Docker Compose --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; diff --git a/docs/site/src/content/docs/tutorials/issue-a-birth-certificate-vc-from-opencrvs.mdx b/docs/site/src/content/docs/tutorials/issue-a-birth-certificate-vc-from-opencrvs.mdx index 663d4de18..da312f720 100644 --- a/docs/site/src/content/docs/tutorials/issue-a-birth-certificate-vc-from-opencrvs.mdx +++ b/docs/site/src/content/docs/tutorials/issue-a-birth-certificate-vc-from-opencrvs.mdx @@ -13,6 +13,9 @@ persona: - assertion provider locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: needs the public OpenCRVS Farajaland demo; live and opt-in, not replayed in CI --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; diff --git a/docs/site/src/content/docs/tutorials/issue-immunization-evidence-from-dhis2.mdx b/docs/site/src/content/docs/tutorials/issue-immunization-evidence-from-dhis2.mdx index e63805502..147819b49 100644 --- a/docs/site/src/content/docs/tutorials/issue-immunization-evidence-from-dhis2.mdx +++ b/docs/site/src/content/docs/tutorials/issue-immunization-evidence-from-dhis2.mdx @@ -11,6 +11,9 @@ persona: - assertion provider locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: needs the public DHIS2 demo; live and opt-in, not replayed in CI --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; diff --git a/docs/site/src/content/docs/tutorials/manage-evidence-verifier-trust.mdx b/docs/site/src/content/docs/tutorials/manage-evidence-verifier-trust.mdx index 0688427f9..03b79551b 100644 --- a/docs/site/src/content/docs/tutorials/manage-evidence-verifier-trust.mdx +++ b/docs/site/src/content/docs/tutorials/manage-evidence-verifier-trust.mdx @@ -11,6 +11,9 @@ persona: - consumer or verifier locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: how-to against the reader's own deployment; no fixed scenario to replay --- A valid signature proves control of a private key. Your consumer still decides which provider, diff --git a/docs/site/src/content/docs/tutorials/move-evidence-to-production-signing.mdx b/docs/site/src/content/docs/tutorials/move-evidence-to-production-signing.mdx index 2d5988112..99035780d 100644 --- a/docs/site/src/content/docs/tutorials/move-evidence-to-production-signing.mdx +++ b/docs/site/src/content/docs/tutorials/move-evidence-to-production-signing.mdx @@ -11,6 +11,9 @@ persona: - operator locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: drift-checked by evidence-production-build-docs.test.mjs; needs a Transit signer --- Use this procedure to move Evidence Gateway from disposable local signing to a strict deployment. diff --git a/docs/site/src/content/docs/tutorials/prove-an-evidence-project.mdx b/docs/site/src/content/docs/tutorials/prove-an-evidence-project.mdx index 04b0ad9a6..b19276334 100644 --- a/docs/site/src/content/docs/tutorials/prove-an-evidence-project.mdx +++ b/docs/site/src/content/docs/tutorials/prove-an-evidence-project.mdx @@ -12,6 +12,9 @@ persona: - operator locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: how-to against the reader's own project; no fixed scenario to replay --- Start with an editable project: the directory `evidencectl init` created, holding `questions/`, diff --git a/docs/site/src/content/docs/tutorials/request-a-holder-bound-credential.mdx b/docs/site/src/content/docs/tutorials/request-a-holder-bound-credential.mdx index 08b6a2f77..f6e32c7c1 100644 --- a/docs/site/src/content/docs/tutorials/request-a-holder-bound-credential.mdx +++ b/docs/site/src/content/docs/tutorials/request-a-holder-bound-credential.mdx @@ -14,6 +14,9 @@ locale: en standards_referenced: - sd-jwt-vc draft: true +tutorial_test: + toolset: evidence + skip: draft, hidden from the sidebar; no verified wallet flow exists to replay --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; diff --git a/docs/site/src/content/docs/tutorials/rotate-evidence-signing-keys.mdx b/docs/site/src/content/docs/tutorials/rotate-evidence-signing-keys.mdx index fc9cc55ee..4a91ffc77 100644 --- a/docs/site/src/content/docs/tutorials/rotate-evidence-signing-keys.mdx +++ b/docs/site/src/content/docs/tutorials/rotate-evidence-signing-keys.mdx @@ -11,6 +11,9 @@ persona: - operator locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: drift-checked by evidence-production-build-docs.test.mjs; needs a deployed signing key --- Use this procedure after diff --git a/docs/site/src/content/docs/tutorials/verify-a-registered-parent-with-opencrvs.mdx b/docs/site/src/content/docs/tutorials/verify-a-registered-parent-with-opencrvs.mdx index 3c24c3746..c262286db 100644 --- a/docs/site/src/content/docs/tutorials/verify-a-registered-parent-with-opencrvs.mdx +++ b/docs/site/src/content/docs/tutorials/verify-a-registered-parent-with-opencrvs.mdx @@ -13,6 +13,9 @@ persona: - assertion provider locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: needs the public OpenCRVS Farajaland demo; live and opt-in, not replayed in CI --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; From 5081dc8e984a817f9958f36b77157b90aff3b78a Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Sat, 26 Sep 2026 19:18:37 +0000 Subject: [PATCH 4/7] feat(docs): replay the Evidence tutorials with the page runner and retire the bash gate Every replayed Evidence page now carries its own test-file, test-expect, and test-excerpt annotations, the output blocks show what the commands print today, and CI runs run-tutorial.mjs --gate evidence. Signed-off-by: Jeremi Joslin --- .github/scripts/ci_changes.py | 81 +- .github/scripts/test_ci_changes.py | 63 +- .github/workflows/ci.yml | 21 +- docs/site/package.json | 5 +- docs/site/scripts/check-evidence-tutorials.sh | 1102 ----------------- .../scripts/check-evidence-tutorials.test.mjs | 804 ------------ docs/site/scripts/run-tutorial.mjs | 4 +- .../assert-a-role-bound-relationship.mdx | 23 +- .../tutorials/connect-a-sqlite-extract.mdx | 4 +- .../control-who-can-request-evidence.mdx | 39 +- .../tutorials/first-evidence-assertion.mdx | 56 +- .../tutorials/issue-fhir-evidence-as-vcs.mdx | 41 +- .../refuse-unsafe-evidence-requests.mdx | 29 +- .../request-evidence-as-sd-jwt-vc.mdx | 31 +- .../request-evidence-from-an-application.mdx | 51 +- .../tutorials/return-a-governed-value.mdx | 21 +- .../run-oid4vci-interoperability-checks.mdx | 13 +- .../verify-an-assertion-as-a-consumer.mdx | 22 +- 18 files changed, 276 insertions(+), 2134 deletions(-) delete mode 100755 docs/site/scripts/check-evidence-tutorials.sh delete mode 100644 docs/site/scripts/check-evidence-tutorials.test.mjs diff --git a/.github/scripts/ci_changes.py b/.github/scripts/ci_changes.py index 302df7010..110faf856 100644 --- a/.github/scripts/ci_changes.py +++ b/.github/scripts/ci_changes.py @@ -178,46 +178,45 @@ "docs/site/src/content/docs/tutorials/publish-governed-sqlite-registry.mdx", ) -# Every input the Evidence tutorial gate replays or is built from. The tutorial -# pages and helper scripts here must stay in step with the gate's own registry -# and the helpers it invokes, which test_ci_changes.py enforces: a tutorial or -# helper CI does not watch is one that rots silently. -EVIDENCE_TUTORIAL_INPUTS = frozenset( - { - "Cargo.lock", - "Cargo.toml", - "docs/site/package-lock.json", - "docs/site/package.json", - "docs/site/scripts/check-evidence-tutorials.sh", - "docs/site/scripts/check-evidence-tutorials.test.mjs", - "docs/site/scripts/evidence-tutorial-fence.sh", - "docs/site/scripts/fixtures/fhir-tutorial-mock.py", - "docs/site/src/content/docs/tutorials/assert-a-role-bound-relationship.mdx", - "docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx", - "docs/site/src/content/docs/tutorials/control-who-can-request-evidence.mdx", - "docs/site/src/content/docs/tutorials/first-evidence-assertion.mdx", - "docs/site/src/content/docs/tutorials/issue-fhir-evidence-as-vcs.mdx", - "docs/site/src/content/docs/tutorials/refuse-unsafe-evidence-requests.mdx", - "docs/site/src/content/docs/tutorials/request-evidence-as-sd-jwt-vc.mdx", - "docs/site/src/content/docs/tutorials/request-evidence-from-an-application.mdx", - "docs/site/src/content/docs/tutorials/run-oid4vci-interoperability-checks.mdx", - "docs/site/src/content/docs/tutorials/return-a-governed-value.mdx", - "docs/site/src/content/docs/tutorials/verify-an-assertion-as-a-consumer.mdx", - "products/evidence/fixtures/interoperability/inji-oid4vci/profile.json", - "products/evidence/fixtures/interoperability/inji-oid4vci/receipt.json", - "products/evidence/scripts/compat/inji-oid4vci-upstream.sh", - "products/evidence/scripts/compat/inji-oid4vci.sh", - # The application tutorial imports the maintained client package, and - # the job assembles that package from this commit with these scripts - # and this pinned build tool. A change to any of them changes what the - # replay imports. - "release/requirements/maturin-1.9.6.txt", - "release/scripts/assemble-registry-client-packages.py", - "release/scripts/assemble-registry-client-wheel.py", - "release/scripts/build-linux-python-client", - "release/scripts/zig-glibc-compiler", - "release/scripts/smoke-registry-client-package.py", - } +# Every input the Evidence tutorial gate replays or is built from: the page +# runner, the pages whose frontmatter it replays, the source mock the toolset +# starts, and the build inputs of what the pages run. The replayed pages here +# must stay in step with their tutorial_test frontmatter, which +# test_ci_changes.py enforces: a tutorial CI does not watch is one that rots +# silently. +EVIDENCE_TUTORIAL_INPUTS = ( + "Cargo.lock", + "Cargo.toml", + "docs/site/package-lock.json", + "docs/site/package.json", + "docs/site/scripts/run-tutorial.mjs", + "docs/site/scripts/tutorial-runner/**", + "docs/site/scripts/fixtures/fhir-tutorial-mock.py", + "docs/site/src/content/docs/tutorials/assert-a-role-bound-relationship.mdx", + "docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx", + "docs/site/src/content/docs/tutorials/control-who-can-request-evidence.mdx", + "docs/site/src/content/docs/tutorials/first-evidence-assertion.mdx", + "docs/site/src/content/docs/tutorials/issue-fhir-evidence-as-vcs.mdx", + "docs/site/src/content/docs/tutorials/refuse-unsafe-evidence-requests.mdx", + "docs/site/src/content/docs/tutorials/request-evidence-as-sd-jwt-vc.mdx", + "docs/site/src/content/docs/tutorials/request-evidence-from-an-application.mdx", + "docs/site/src/content/docs/tutorials/run-oid4vci-interoperability-checks.mdx", + "docs/site/src/content/docs/tutorials/return-a-governed-value.mdx", + "docs/site/src/content/docs/tutorials/verify-an-assertion-as-a-consumer.mdx", + "products/evidence/fixtures/interoperability/inji-oid4vci/profile.json", + "products/evidence/fixtures/interoperability/inji-oid4vci/receipt.json", + "products/evidence/scripts/compat/inji-oid4vci-upstream.sh", + "products/evidence/scripts/compat/inji-oid4vci.sh", + # The application tutorial imports the maintained client package, and + # the job assembles that package from this commit with these scripts + # and this pinned build tool. A change to any of them changes what the + # replay imports. + "release/requirements/maturin-1.9.6.txt", + "release/scripts/assemble-registry-client-packages.py", + "release/scripts/assemble-registry-client-wheel.py", + "release/scripts/build-linux-python-client", + "release/scripts/zig-glibc-compiler", + "release/scripts/smoke-registry-client-package.py", ) # Every input the Base Registry Engine tutorial gate replays or is built from: @@ -1216,7 +1215,7 @@ def classify( evidence_tutorial = ( complete - or any(path in EVIDENCE_TUTORIAL_INPUTS for path in paths) + or any(matches(path, *EVIDENCE_TUTORIAL_INPUTS) for path in paths) or bool( affected & (EVIDENCE_TUTORIAL_PACKAGES | ASSEMBLED_PYTHON_CLIENT_PACKAGES) ) diff --git a/.github/scripts/test_ci_changes.py b/.github/scripts/test_ci_changes.py index ff6fb7cd4..4551744a6 100644 --- a/.github/scripts/test_ci_changes.py +++ b/.github/scripts/test_ci_changes.py @@ -829,52 +829,35 @@ def test_manifest_core_changes_select_breg_through_linked_code( self.assertIn("registry-breg", outputs["rust_packages"]) self.assertIn("registry-manifest-core", outputs["rust_packages"]) - def test_evidence_tutorial_inputs_cover_every_registered_tutorial(self) -> None: - # The gate's registry is the source of truth for which tutorials exist. - # A tutorial missing here would not trigger the job that replays it, so - # it could break without any pull request noticing. - gate = ( - Path(__file__).resolve().parents[2] - / "docs/site/scripts/check-evidence-tutorials.sh" - ) - registry = re.search( - r"^EVIDENCE_TUTORIALS=\((.*?)^\)", gate.read_text(), re.DOTALL | re.MULTILINE - ) - if registry is None: - self.fail("the gate must declare EVIDENCE_TUTORIALS") - slugs = registry.group(1).split() - self.assertTrue(slugs, "the gate must register at least one tutorial") + def test_evidence_tutorial_inputs_cover_every_replayed_tutorial(self) -> None: + # Each page's tutorial_test frontmatter is the source of truth for + # which tutorials the gate replays. A replayed page missing here would + # not trigger the job that replays it, so it could break without any + # pull request noticing. + docs = Path(__file__).resolve().parents[2] / "docs/site/src/content/docs" + slugs = [] + for section in ("start", "tutorials"): + for page in sorted((docs / section).glob("*.mdx")): + frontmatter = yaml.safe_load(page.read_text().split("---\n")[1]) + declaration = frontmatter.get("tutorial_test") or {} + if declaration.get("toolset") == "evidence" and "skip" not in declaration: + slugs.append(f"{section}/{page.stem}") + self.assertIn("tutorials/first-evidence-assertion", slugs) for slug in slugs: with self.subTest(slug=slug): - self.assertIn( - f"docs/site/src/content/docs/tutorials/{slug}.mdx", - EVIDENCE_TUTORIAL_INPUTS, + page = f"docs/site/src/content/docs/{slug}.mdx" + self.assertTrue( + any( + fnmatch.fnmatchcase(page, pattern) + for pattern in EVIDENCE_TUTORIAL_INPUTS + ) ) - def test_evidence_tutorial_inputs_cover_every_helper_the_gate_invokes(self) -> None: - # Same reasoning as the tutorial registry above, one layer down. The gate - # delegates to sibling scripts, and a change to one of those changes what - # every tutorial replay does. A helper missing here routes the change - # past the job that would have caught it. - gate = ( - Path(__file__).resolve().parents[2] - / "docs/site/scripts/check-evidence-tutorials.sh" - ) - helpers = set( - re.findall( - r"\$SITE_ROOT/scripts/([A-Za-z0-9._/-]+\.(?:mjs|py|sh))", - gate.read_text(), - ) - ) - self.assertTrue(helpers, "the gate must invoke at least one helper") - for helper in sorted(helpers): - with self.subTest(helper=helper): - self.assertIn(f"docs/site/scripts/{helper}", EVIDENCE_TUTORIAL_INPUTS) - def test_evidence_tutorial_routing(self) -> None: infrastructure = ( - "docs/site/scripts/check-evidence-tutorials.sh", - "docs/site/scripts/check-evidence-tutorials.test.mjs", + "docs/site/scripts/run-tutorial.mjs", + "docs/site/scripts/tutorial-runner/toolsets.mjs", + "docs/site/scripts/fixtures/fhir-tutorial-mock.py", "docs/site/src/content/docs/tutorials/first-evidence-assertion.mdx", "docs/site/package.json", ) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 960925c58..e27356e49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1356,9 +1356,13 @@ jobs: cache-targets: false save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Test the tutorial gate helpers + - name: Install docs dependencies + working-directory: docs/site + run: npm ci + + - name: Test the tutorial runner working-directory: docs/site - run: npm run test:tutorial:evidence + run: npm run test:tutorial:runner - name: Check tutorial command drift working-directory: docs/site @@ -1393,8 +1397,6 @@ jobs: "${RUNNER_TEMP}/maturin/bin/pip" install --quiet \ --require-hashes --only-binary=:all: \ --requirement "${GITHUB_WORKSPACE}/release/requirements/maturin-1.9.6.txt" - # The output stays inside the workspace, because the container step - # below mounts the workspace and nothing else. out_dir="${GITHUB_WORKSPACE}/target/evidence-tutorial-client" python3 release/scripts/assemble-registry-client-packages.py \ --artifacts python \ @@ -1411,15 +1413,13 @@ jobs: exit 1 fi # Prove installer metadata and native facade loading from this exact - # wheel. The clean-container tutorials separately exercise requests. + # wheel. The application tutorial separately exercises requests. python3 -m venv "${RUNNER_TEMP}/client-install-smoke" "${RUNNER_TEMP}/client-install-smoke/bin/pip" install \ --no-index --no-deps "${wheel}" "${RUNNER_TEMP}/client-install-smoke/bin/python" -I \ release/scripts/smoke-registry-client-package.py - # The gate reads it inside the container, at the mounted path. - printf 'TUTORIAL_CLIENT_WHEEL=/work/%s\n' \ - "${wheel#"${GITHUB_WORKSPACE}/"}" >>"${GITHUB_ENV}" + printf 'REGISTRY_CLIENT_PY_WHEEL=%s\n' "${wheel}" >>"${GITHUB_ENV}" - name: Test the exact local Evidence lifecycle shell: bash @@ -1438,10 +1438,7 @@ jobs: EVIDENCECTL_BIN: ${{ github.workspace }}/target/debug/evidencectl EVIDENCE_OID4VCI_BIN: ${{ github.workspace }}/target/debug/evidence-oid4vci EVIDENCE_OID4VCI_INTEROP_TEST_BIN: ${{ github.workspace }}/target/debug/inji-oid4vci-interop-test - run: | - set -euo pipefail - REGISTRY_CLIENT_PY_WHEEL="${GITHUB_WORKSPACE}/${TUTORIAL_CLIENT_WHEEL#/work/}" \ - bash docs/site/scripts/check-evidence-tutorials.sh + run: node docs/site/scripts/run-tutorial.mjs --gate evidence breg-tutorial: name: Base Registry Engine tutorial from source diff --git a/docs/site/package.json b/docs/site/package.json index 7e5c2c3a4..94afdc352 100644 --- a/docs/site/package.json +++ b/docs/site/package.json @@ -47,9 +47,8 @@ "check:tutorial:discovery:dry-run": "bash scripts/check-discovery-tutorial.sh --dry-run", "check:tutorial:relay": "bash scripts/check-relay-tutorial.sh", "check:tutorial:relay:dry-run": "bash scripts/check-relay-tutorial.sh --dry-run", - "test:tutorial:evidence": "node --test scripts/check-evidence-tutorials.test.mjs", - "check:tutorial:evidence": "bash scripts/check-evidence-tutorials.sh", - "check:tutorial:evidence:dry-run": "bash scripts/check-evidence-tutorials.sh --dry-run", + "check:tutorial:evidence": "node scripts/run-tutorial.mjs --gate evidence", + "check:tutorial:evidence:dry-run": "node scripts/run-tutorial.mjs --gate evidence --dry-run", "test:tutorial:runner": "node --test \"scripts/tutorial-runner/*.test.mjs\"", "check:tutorial:breg": "node scripts/run-tutorial.mjs --gate breg", "check:tutorial:breg:dry-run": "node scripts/run-tutorial.mjs --gate breg --dry-run", diff --git a/docs/site/scripts/check-evidence-tutorials.sh b/docs/site/scripts/check-evidence-tutorials.sh deleted file mode 100755 index f169fdd8b..000000000 --- a/docs/site/scripts/check-evidence-tutorials.sh +++ /dev/null @@ -1,1102 +0,0 @@ -#!/usr/bin/env bash -# -# Execute the current Evidence tutorials from a fresh reader directory. -# -# What this gate is for: proving that the commands the tutorials document still -# run, and that a short list of behaviours a successful exit does not already -# prove still holds. A refusal that still refuses, tampering that is still -# caught, an audit entry that still records the disclosure it should. -# -# What this gate is NOT for: policing what a page says. It pins no fence count, -# no command string and no documented output. Prose, the text around a -# heading, output blocks and command wording are free to change without touching -# this file, and a writer may add or remove a command block under a heading the -# journey already runs with no change here at all. If you find yourself adding -# an array of strings a page must contain, stop: that is the pinning this file -# deliberately does not do, and it is what made these tutorials unreadable for -# a human reader once already. -# -# This gate builds the Evidence toolset from the checked-out source unless -# EVIDENCE_BIN, EVIDENCECTL_BIN, and EVIDENCE_OID4VCI_BIN select exact -# candidate or released bytes, then replays each registered tutorial's own -# shell fences in its own reader directory. Every tutorial creates the files it -# needs from its documented commands, so what CI runs is what a reader copies. -# -# Usage: -# scripts/check-evidence-tutorials.sh replay every tutorial -# scripts/check-evidence-tutorials.sh --dry-run resolve the journeys only -# scripts/check-evidence-tutorials.sh --only one tutorial and its prerequisites -# -# Registering a tutorial means adding its slug to EVIDENCE_TUTORIALS and a -# branch to load_spec. Each spec holds two things: -# -# SPEC_STEPS the reader journey, in order. It need not follow document -# order: a tutorial that leaves one terminal in an earlier -# directory is replayed by running its later fence first. -# Fences are addressed by the heading they sit under, never by -# position, so inserting a command block cannot silently move a -# step onto the wrong command. -# run: execute every sh fence under -# that heading, in document order -# run:| execute the nth sh fence under -# that heading -# run-fails:| execute one sh fence the page -# documents as refused, and -# require a non-zero exit -# background:| run a one-line sh fence the page -# leaves running in a second -# terminal -# stop-background stop the most recently started -# background fence, where the page -# says to press Ctrl+C -# save:H|lang|occ|target write a documented non-shell -# fence to the file the reader is -# told to create -# edit:H|lang|occ|H2|lang2|occ2|target -# apply a documented before/after -# fence pair to an existing file -# wait-http:URL block until that URL answers -# python-client put the client package assembled -# from this checkout on the import -# path, standing in for the -# documented install -# fhir-mock start the sanitized local FHIR -# mock this gate carries -# track-pid:PATH adopt a PID a fence wrote, so -# cleanup reaches it -# The | suffix is optional wherever a heading holds a single -# sh fence. Skipping is implicit: a fence under no listed -# heading is simply not run, and the summary names it so a -# reviewer can see the unverified surface. -# -# SPEC_ASSERTS behaviours the replay transcript must still show. One test -# decides membership: would this regress silently, without any -# command exiting non-zero? Startup chatter, "created", "ready" -# and "prepared" lines fail that test, because the next command -# would have failed without them. Do not grow this back into a -# transcript pin. -# -# Renaming a heading breaks the steps that name it, by name, in --dry-run. -# That is the trade, and it is a good one: a renamed heading is a structural -# edit to the journey, it fails loudly rather than replaying the wrong command, -# and it is exactly when the journey is worth walking again. -# -# Configuration: -# EVIDENCE_BIN / EVIDENCECTL_BIN / run these exact binaries instead of -# EVIDENCE_OID4VCI_BIN building from source -# EVIDENCE_OID4VCI_INTEROP_TEST_BIN run this prebuilt sanitized flow test -# REGISTRY_CLIENT_PY_WHEEL import the client package out of this -# assembled wheel -# EVIDENCE_TUTORIAL_CARGO_PROFILE ci (default) or release -# EVIDENCE_TUTORIAL_DOCS_ROOT tutorial directory override (tests) - -set -euo pipefail - -SITE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -REPO_ROOT="$(cd "$SITE_ROOT/../.." && pwd)" -# A fence helper written against the replay userland's floor: it locates a -# fence by heading, language and occurrence and applies it to a file, using -# only the shell and coreutils the container carries. -FENCE="$SITE_ROOT/scripts/evidence-tutorial-fence.sh" -FHIR_TUTORIAL_MOCK="$SITE_ROOT/scripts/fixtures/fhir-tutorial-mock.py" -DOCS_ROOT="${EVIDENCE_TUTORIAL_DOCS_ROOT:-$SITE_ROOT/src/content/docs/tutorials}" -BUILD_PROFILE="${EVIDENCE_TUTORIAL_CARGO_PROFILE:-ci}" -TARGET_DIR="$REPO_ROOT/target/evidence-tutorial-source" - -# --------------------------------------------------------------------------- -# Registered tutorials -# --------------------------------------------------------------------------- - -EVIDENCE_TUTORIALS=( - first-evidence-assertion - request-evidence-as-sd-jwt-vc - run-oid4vci-interoperability-checks - request-evidence-from-an-application - return-a-governed-value - assert-a-role-bound-relationship - refuse-unsafe-evidence-requests - verify-an-assertion-as-a-consumer - control-who-can-request-evidence - issue-fhir-evidence-as-vcs - connect-a-sqlite-extract -) - -# Every other page under DOCS_ROOT, and the reason it is not replayed here. -# check_tutorial_coverage below fails by name on a page in neither list, which -# is the gap that let broken DHIS2 tutorial commands ship once already. -EXCLUDED_EVIDENCE_TUTORIALS=( - evidence-from-breg # native Docker PostgreSQL journey; products/breg/evidence/tests/verify-composition.py covers offline composition, independent reader checks live steps - deploy-evidence-from-breg # operated target handoff; native composition and production build tests cover offline candidates, target-host checks need provisioned dependencies - build-and-deploy-evidence-project # drift-checked by evidence-production-build-docs.test.mjs; needs a production build environment - connect-an-institution-source # how-to against the reader's own OpenAPI source; no fixed scenario this gate can replay - first-run-with-solmara-lab # historical; the Solmara Lab stack is replayed by check-tutorial.sh, not here - first-breg # Base Registry Engine journey; product CI runs quickstart/run.sh --smoke, reader execution checks the documented steps - first-casework # Registry Casework journey; replayed end to end by run-tutorial.mjs --gate casework in the casework-tutorial job - first-render-document # Registry Render journey; offline render CLI steps against the products/render example bundles, verified in reader mode outside the Evidence runner - review-breg-changes-in-casework # cross-product boundary guide with no Evidence CLI journey; real BReg-to-Casework composition runs in the owning product aggregate - extend-a-registry-with-a-module # Base Registry Engine journey; offline bregctl steps on the quickstart project, verified in reader mode outside the Evidence runner - derive-a-registry-from-publicschema # Base Registry Engine journey; offline bregctl steps deriving a project from the embedded PublicSchema snapshot, verified in reader mode outside the Evidence runner - send-registry-events-to-a-webhook # Base Registry Engine journey; needs the demo launcher's webhook receiver, verified in reader mode outside the Evidence runner - build-a-breg-production-candidate # Base Registry Engine journey; needs a PostgreSQL container and a local signing key, verified in reader mode outside the Evidence runner - query-breg-client # BReg client journey; depends on the released unified packages, like query-relay-client - integrate-evidence-candidate-with-docker-compose # drift-checked by evidence-production-build-docs.test.mjs; needs Docker Compose - issue-a-birth-certificate-vc-from-opencrvs # needs the public OpenCRVS Farajaland demo; live and opt-in, not replayed in CI - issue-immunization-evidence-from-dhis2 # needs the public DHIS2 demo; live and opt-in, not replayed in CI - manage-evidence-verifier-trust # how-to against the reader's own deployment; no fixed scenario this gate can replay - move-evidence-to-production-signing # drift-checked by evidence-production-build-docs.test.mjs; needs a Transit signer - prove-an-evidence-project # how-to against the reader's own project; no fixed scenario this gate can replay - publish-and-consume-discovery-index # Discovery journey; replayed by check-discovery-tutorial.sh with native Evidence and Relay handoffs - publish-governed-sqlite-registry # Relay V2 journey; replayed by check-relay-tutorial.sh in the relay-v2-contracts job - query-relay-client # Relay client journey; depends on a released wheel and the Relay publishing prerequisite - request-a-holder-bound-credential # draft: true, hidden from the sidebar; no verified wallet flow exists to replay - review-registry-changes # Base Registry Engine journey; verify page commands in reader mode, outside the Evidence runner - rotate-evidence-signing-keys # drift-checked by evidence-production-build-docs.test.mjs; needs a deployed signing key - verify-a-registered-parent-with-opencrvs # needs the public OpenCRVS Farajaland demo; live and opt-in, not replayed in CI -) - -in_list() { - local needle="$1" - shift - local item - for item in "$@"; do - [[ "$item" == "$needle" ]] && return 0 - done - return 1 -} - -# Assert that every page under DOCS_ROOT is either registered for replay or -# named in EXCLUDED_EVIDENCE_TUTORIALS with a reason. A page in neither list -# is a coverage gap: nothing would ever replay it or explain why not. -check_tutorial_coverage() { - local file slug - local -a unregistered=() - for slug in "${EXCLUDED_EVIDENCE_TUTORIALS[@]}"; do - if in_list "$slug" "${EVIDENCE_TUTORIALS[@]}"; then - printf 'coverage error in %s: %s is both registered in EVIDENCE_TUTORIALS and excluded in EXCLUDED_EVIDENCE_TUTORIALS\n' \ - "${BASH_SOURCE[0]}" "$slug" >&2 - exit 2 - fi - if [[ ! -f "$DOCS_ROOT/$slug.mdx" ]]; then - printf 'coverage error in %s: %s.mdx in EXCLUDED_EVIDENCE_TUTORIALS does not exist under %s\n' \ - "${BASH_SOURCE[0]}" "$slug" "$DOCS_ROOT" >&2 - exit 2 - fi - done - for file in "$DOCS_ROOT"/*.mdx; do - [[ -e "$file" ]] || continue - slug="$(basename "$file" .mdx)" - if ! in_list "$slug" "${EVIDENCE_TUTORIALS[@]}" && ! in_list "$slug" "${EXCLUDED_EVIDENCE_TUTORIALS[@]}"; then - unregistered+=("$slug") - fi - done - if ((${#unregistered[@]} > 0)); then - printf 'tutorial coverage gap: the following pages are neither registered in EVIDENCE_TUTORIALS nor excluded in EXCLUDED_EVIDENCE_TUTORIALS:\n' >&2 - for slug in "${unregistered[@]}"; do - printf ' %s.mdx\n' "$slug" >&2 - done - printf 'add each to EVIDENCE_TUTORIALS (with a load_spec branch) or to EXCLUDED_EVIDENCE_TUTORIALS with a reason, in %s\n' \ - "${BASH_SOURCE[0]}" >&2 - exit 1 - fi -} - -check_tutorial_coverage - -load_spec() { - SPEC_STEPS=() - SPEC_ASSERTS=() - - case "$1" in - first-evidence-assertion) - SPEC_STEPS=( - "run:Preview a synthetic source|1" - "save:Preview a synthetic source|yaml|1|tutorial-source.openapi.yaml" - "background:Preview a synthetic source|2" - "wait-http:http://127.0.0.1:4010/people/person-123" - "run:Preview a synthetic source|3" - "stop-background" - "run:Create the Evidence Gateway project" - "run:Keep exact cases for the tutorial|1" - "save:Create the Evidence Gateway project|yaml|1|questions/adult-status.yaml" - "save:Create the Evidence Gateway project|rhai|1|derivations/adult-status.rhai" - "save:Keep exact cases for the tutorial|yaml|1|mocks/source.yaml" - "save:Keep exact cases for the tutorial|json|1|mocks/cases/person-123.json" - "save:Keep exact cases for the tutorial|json|2|mocks/cases/person-456.json" - "save:Keep exact cases for the tutorial|json|3|mocks/cases/person-789.json" - "run:Keep exact cases for the tutorial|2" - "background:Keep exact cases for the tutorial|3" - "wait-http:http://127.0.0.1:4010/people/person-123" - "run:Keep exact cases for the tutorial|4" - "run:Request an assertion" - "run:Verify before reading" - "run:Try the SD-JWT VC serialization" - "run:Stop the local services" - "run:Inspect the audit entry" - "run:Clean up" - ) - # The assertion was verified, the audit recorded who asked and why, and - # exactly one field was released. Nothing else here regresses in - # silence: a mock that did not start or a project that was not created - # ends the journey at the next command. - SPEC_ASSERTS=( - "VERIFIED" - "ACCESS AUTHORIZED adult-status age-check requester=" - "DISCLOSURE RELEASED is_adult" - ) - ;; - request-evidence-as-sd-jwt-vc) - SPEC_STEPS=( - "background:Restart the source mock|1" - "wait-http:http://127.0.0.1:4010/people/person-123" - "run:Restart the source mock|2" - "run:Request a scalar credential" - "run:Inspect the compact structure after verification" - "run:Inspect issuer discovery" - "run:Prove tampering is refused|1" - "run-fails:Prove tampering is refused|2" - "save:Model independently disclosed fields|yaml|1|schemas/adult-assessment.yaml" - "save:Model independently disclosed fields|yaml|2|questions/adult-assessment.yaml" - "save:Model independently disclosed fields|rhai|1|derivations/adult-assessment.rhai" - "run:Model independently disclosed fields" - "run:Clean up" - ) - # The disclosure names are what a holder actually hands over, and the - # fences that print them exit zero whatever the credential carries, so a - # credential that started disclosing more would pass unnoticed. - SPEC_ASSERTS=( - "disclosure: urn:registrystack:evidence:local:concept:adult-status:is_adult" - "evidencectl: Evidence response verification failed" - "disclosure: criterion" - "disclosure: isAdult" - "ACCESS AUTHORIZED adult-assessment age-assessment-review requester=" - "DISCLOSURE RELEASED adult_assessment" - ) - ;; - run-oid4vci-interoperability-checks) - SPEC_STEPS=( - "run:Copy the complete configuration" - "save:Copy the complete configuration|yaml|1|.tutorial/oid4vci-adopter/oid4vci.yaml" - "run:Replay the sanitized profile" - "run:Clean up" - ) - # The sanitized runner prints one line per phase and exits non-zero on - # any of them, so the phases hold themselves up. What they cannot hold - # up is having run at all: a filter that selects no test leaves the - # runner exiting zero with nothing done. One end-to-end line proves the - # wallet flow ran; the rest would be a transcript pin. - SPEC_ASSERTS=( - "PRESENTATION VERIFIED: public wallet flow returned holder-bound Evidence" - ) - ;; - return-a-governed-value) - SPEC_STEPS=( - "background:Restart the source mock" - "wait-http:http://127.0.0.1:4010/people/person-123" - "run:Add the age-bracket question" - "save:Add the age-bracket question|yaml|1|questions/age-bracket.yaml" - "save:Add the age-bracket question|rhai|1|derivations/age-bracket.rhai" - "run:Start the updated project" - "run:Request and verify the bracket" - "run:Inspect the audit and clean up" - ) - SPEC_ASSERTS=( - "VERIFIED" - "ACCESS AUTHORIZED age-bracket service-path-selection requester=" - "DISCLOSURE RELEASED age_bracket" - ) - ;; - control-who-can-request-evidence) - SPEC_STEPS=( - "background:Restart the source mock|1" - "wait-http:http://127.0.0.1:4010/people/person-123" - "run:Restart the source mock|2" - "run:Define two access policies" - "run:Register the first local application" - "run:Start the protected service" - "run:Make an allowed request" - "run:Add an application for the next generation" - "run:Use the application assigned the policy" - "run:Try a question the application was not granted" - "run:Revoke an application|1" - "run:Revoke an application|2" - "run-fails:Revoke an application|3" - "run:Stop the final generation" - "run:Clean up" - ) - # This tutorial teaches refusal, so the refusals are what must hold. - # The unauthorized request's curl carries no --fail-with-body, so it - # exits zero on a 403 and a boundary that started answering 200 would - # leave the journey green. The post-revocation preparation requires a - # non-zero exit; the message proves the client was revoked rather than - # refused for some unrelated reason. - SPEC_ASSERTS=( - "VERIFIED" - "HTTP 403" - '"code": "evidence.denied"' - "evidencectl: unknown or revoked active client age-checker" - "ACCESS REFUSED requester=" - "reason=not_authorized" - ) - ;; - assert-a-role-bound-relationship) - SPEC_STEPS=( - "run:Start a relationship registry|1" - "save:Start a relationship registry|python|1|registry.py" - "background:Start a relationship registry|2" - "wait-http:http://127.0.0.1:8002/openapi.json" - "run:Create the Evidence Gateway project" - "save:Create the Evidence Gateway project|yaml|1|questions/parent-relationship.yaml" - "save:Create the Evidence Gateway project|rhai|1|derivations/parent-relationship.rhai" - "run:Start the project" - "run:Bind both subjects to the request" - "run:Inspect the audit and clean up" - ) - SPEC_ASSERTS=( - "VERIFIED" - "ACCESS AUTHORIZED parent-relationship relationship-check requester=" - "DISCLOSURE RELEASED relationship_confirmed" - ) - ;; - refuse-unsafe-evidence-requests) - SPEC_STEPS=( - "background:Restart the local boundary|1" - "wait-http:http://127.0.0.1:4010/people/person-123" - "run:Restart the local boundary|2" - "run:Prepare one authorized request" - "run:Change the purpose after preparation" - "run:Obtain and verify the authorized response" - "run:Change the signed response|1" - "run-fails:Change the signed response|2" - "run:Clean up" - ) - # The whole page is these three outcomes: the altered request was - # refused, the untouched one verified, and the altered response was - # caught. The refusal curl exits zero on a 403, so only the printed - # status separates a boundary that refused from one that answered. - SPEC_ASSERTS=( - "HTTP 403" - "VERIFIED" - "evidencectl: Evidence response verification failed" - ) - ;; - verify-an-assertion-as-a-consumer) - SPEC_STEPS=( - "run:Start with three separate inputs" - "run:Re-verify the recorded decision" - ) - # `evidence verify` exits non-zero on both `authentic: no` and - # `currently-valid: no`, so the verdict holds itself up. The disclosed - # value does not: verification succeeds whatever the assertion says, and - # a consumer reading the wrong answer is the failure that matters. - SPEC_ASSERTS=( - '"value": true' - ) - ;; - request-evidence-from-an-application) - SPEC_STEPS=( - # The registry runs in the terminal the reader never moved out of - # the first tutorial's directory, so it starts before the `cd` the - # page opens with rather than where the page prints it. - "background:Start the local services|1" - "wait-http:http://127.0.0.1:4010/people/person-123" - "run:Give the application its own identity" - "run:Pin the keys your application trusts" - # Stands in for the fence under "Install the Python client", the - # documented install of the released client package. - "python-client" - "run:Start the local services|2" - "run-fails:Start the local services|3" - "run:Read the definitions once" - "run:Pin the procedure" - "save:Write the relying procedure|python|1|age_check.py" - "run:Run it" - "run-fails:Refuse before reading" - "run:Stop the local services" - ) - # What the relying application actually did. Both refusals already - # exit non-zero, so what is held here is the reason: an unnamed caller - # refused for want of a registered client, and an unverifiable response - # refused before anything was read. The two answers prove the right - # subject was resolved rather than a constant returned, the pinning line - # proves the subject binding is still recorded, and the assurance - # profile is the trust level a relying party reads off the deployment. - SPEC_ASSERTS=( - "evidencectl: the active project requires a registered client selected with --client" - '"assuranceProfile": "local"' - "person-123 is_adult=True" - "person-456 is_adult=False" - "pinned binding recorded in subject-bindings.json" - "unverifiable response, nothing read (policy)" - ) - ;; - connect-a-sqlite-extract) - SPEC_STEPS=( - "run:Create and prove the starter" - ) - # The scaffold and the fixture run hold themselves up: a starter that - # failed to scaffold ends the journey at the next command, and the - # fixture run exits non-zero on any case it cannot prove. - SPEC_ASSERTS=() - ;; - issue-fhir-evidence-as-vcs) - SPEC_STEPS=( - "run:Select live synthetic records|1" - "save:Select live synthetic records|python|1|discover-fhir-records.py" - "fhir-mock" - "run:Select live synthetic records|2" - "save:Run a live FHIR read-through adapter|python|1|fhir-read-through.py" - "run:Run a live FHIR read-through adapter" - "track-pid:fhir-read-through.pid" - "save:Describe the exact FHIR reads|yaml|1|fhir-smart-r4.openapi.yaml" - "run:Describe the exact FHIR reads" - "save:Author the patient coverage question|yaml|1|questions/fhir-coverage-status.yaml" - "save:Author the patient coverage question|rhai|1|derivations/fhir-coverage-status.rhai" - "save:Author the healthcare-establishment question|yaml|1|questions/fhir-healthcare-establishment.yaml" - "save:Author the healthcare-establishment question|rhai|1|derivations/fhir-healthcare-establishment.rhai" - "run:Start the project" - "run:Request the patient coverage credential" - "run:Request the healthcare-establishment credential" - "run:Inspect the audit and clean up" - ) - SPEC_ASSERTS=( - "VERIFIED" - "ACCESS AUTHORIZED fhir-healthcare-establishment healthcare-establishment-verification requester=" - "DISCLOSURE RELEASED healthcare_provider_record_active" - ) - ;; - *) - printf '%s is not a registered Evidence tutorial\n' "$1" >&2 - exit 2 - ;; - esac -} - -# --------------------------------------------------------------------------- -# Arguments -# --------------------------------------------------------------------------- - -DRY_RUN=0 -ONLY="" -while (($# > 0)); do - case "$1" in - --dry-run) - DRY_RUN=1 - shift - ;; - --only) - if (($# < 2)); then - printf -- '--only needs a tutorial slug\n' >&2 - exit 2 - fi - ONLY="$2" - shift 2 - ;; - *) - printf 'unknown argument: %s (expected --dry-run or --only )\n' "$1" >&2 - exit 2 - ;; - esac -done - -if [[ -n "$ONLY" ]]; then - # load_spec exits on an unregistered slug, which is the check we want here. - load_spec "$ONLY" - # Every follow-up begins from the project first-evidence-assertion builds. - # A full run gets that from the list order; --only has to name it. - case "$ONLY" in - request-evidence-as-sd-jwt-vc | return-a-governed-value | \ - refuse-unsafe-evidence-requests | verify-an-assertion-as-a-consumer | \ - control-who-can-request-evidence | request-evidence-from-an-application) - EVIDENCE_TUTORIALS=(first-evidence-assertion "$ONLY") - ;; - *) EVIDENCE_TUTORIALS=("$ONLY") ;; - esac -fi - -WORK_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/evidence-tutorial.XXXXXX")" -cleanup() { - local exit_code=$? - set +e - chmod -R u+w "$WORK_ROOT" 2>/dev/null - rm -rf "$WORK_ROOT" - if ((exit_code == 0)); then - printf 'Evidence tutorial gate: PASS\n' - else - printf 'Evidence tutorial gate: FAIL (exit %d)\n' "$exit_code" >&2 - fi -} -trap cleanup EXIT -trap 'exit 130' HUP INT TERM - -# --------------------------------------------------------------------------- -# Toolset under test -# --------------------------------------------------------------------------- - -resolve_profile_dir() { - case "$BUILD_PROFILE" in - ci | release) printf '%s' "$BUILD_PROFILE" ;; - *) - printf 'unsupported tutorial Cargo profile: %s (expected ci or release)\n' \ - "$BUILD_PROFILE" >&2 - exit 1 - ;; - esac -} - -SHIM_DIR="$WORK_ROOT/bin" - -prepare_toolset() { - if [[ -z "${EVIDENCE_BIN:-}" || -z "${EVIDENCECTL_BIN:-}" || \ - -z "${EVIDENCE_OID4VCI_BIN:-}" ]]; then - local profile_dir - profile_dir="$(resolve_profile_dir)" - (cd "$REPO_ROOT" && CARGO_TARGET_DIR="$TARGET_DIR" \ - cargo build --locked --profile "$BUILD_PROFILE" \ - -p registry-evidence -p registry-evidencectl \ - -p registry-evidence-oid4vci) - EVIDENCE_BIN="$TARGET_DIR/$profile_dir/evidence" - EVIDENCECTL_BIN="$TARGET_DIR/$profile_dir/evidencectl" - EVIDENCE_OID4VCI_BIN="$TARGET_DIR/$profile_dir/evidence-oid4vci" - fi - export EVIDENCE_OID4VCI_BIN - local bin - for bin in "$EVIDENCE_BIN" "$EVIDENCECTL_BIN" "$EVIDENCE_OID4VCI_BIN"; do - # Absoluteness first: the reader journey runs from its own directory and - # reaches the binaries through symlinks, so a relative path resolves - # against the wrong directory and would otherwise surface much later, - # mid-journey, as "command not found". - if [[ "$bin" != /* ]]; then - printf 'toolset binary path must be absolute: %s\n' "$bin" >&2 - exit 1 - fi - if [[ ! -x "$bin" ]]; then - printf 'toolset binary not executable: %s\n' "$bin" >&2 - exit 1 - fi - done - - # The tutorials call the binaries by name, so serve them from a shim dir. - mkdir -p "$SHIM_DIR" - ln -s "$EVIDENCE_BIN" "$SHIM_DIR/evidence" - ln -s "$EVIDENCECTL_BIN" "$SHIM_DIR/evidencectl" - ln -s "$EVIDENCE_OID4VCI_BIN" "$SHIM_DIR/evidence-oid4vci" -} - -# The unified client package, unpacked once for whichever tutorials import it. -# -# The documented install resolves the package from an index at the running -# runtime's version, which is the right instruction for a reader and the wrong -# one for this gate: it needs the network, and it would prove a released client -# rather than the one in this checkout. Importing a package assembled from this -# checkout instead is what makes a client regression fail this gate on the -# commit that introduces it. The documented install fence it stands in for is -# reported as unexecuted, so its version selector stays a reviewer's call -# rather than this gate's. -# The bindings inside are built for the stable ABI, so a wheel assembled -# outside this script imports under any CPython the replay userland carries, -# exactly as EVIDENCE_BIN's siblings let CI mount prebuilt binaries. Assembling -# one needs a build toolchain the replay userland does not carry, so this gate -# never assembles: the caller names a wheel, or the gate stops here. -REGISTRY_CLIENT_PY_WHEEL="${REGISTRY_CLIENT_PY_WHEEL:-}" - -prepare_python_client() { - if [[ -z "$REGISTRY_CLIENT_PY_WHEEL" ]]; then - printf 'REGISTRY_CLIENT_PY_WHEEL is unset: name a client wheel assembled with %s\n' \ - 'release/scripts/assemble-registry-client-packages.py' >&2 - exit 1 - fi - if [[ "$REGISTRY_CLIENT_PY_WHEEL" != /* ]]; then - printf 'client wheel path must be absolute: %s\n' "$REGISTRY_CLIENT_PY_WHEEL" >&2 - exit 1 - fi - if [[ ! -f "$REGISTRY_CLIENT_PY_WHEEL" ]]; then - printf 'client wheel not found: %s; assemble one with %s\n' \ - "$REGISTRY_CLIENT_PY_WHEEL" \ - 'release/scripts/assemble-registry-client-packages.py' >&2 - exit 1 - fi -} - -# --------------------------------------------------------------------------- -# Journey assembly -# --------------------------------------------------------------------------- - -# Resolve a heading address to the sh fence numbers it names, in document -# order, space separated. -# -# An address is a heading, optionally followed by | to name one -# fence under it. Addressing by heading rather than by position is what lets a -# writer add or remove a command block without touching a spec, and it is what -# stops an inserted block from silently moving a later step onto the wrong -# command. -resolve_fences() { - local slug="$1" address="$2" fence_dir="$3" - local heading="$address" occurrence="" - if [[ "$address" == *'|'* ]]; then - heading="${address%%|*}" - occurrence="${address##*|}" - if [[ ! "$occurrence" =~ ^[1-9][0-9]*$ ]]; then - printf 'tutorial spec error in %s: fence occurrence must be a positive integer: %s\n' \ - "$slug" "$address" >&2 - exit 2 - fi - fi - local matched - matched="$(awk -F '\t' -v want="$heading" -v want_occurrence="$occurrence" ' - $3 != want { next } - want_occurrence != "" && $2 != want_occurrence + 0 { next } - { printf "%s ", $1 } - ' "$fence_dir/index.tsv")" - matched="${matched% }" - if [[ -z "$matched" ]]; then - printf 'tutorial drift in %s: no sh fence answers to "%s"\n' "$slug" "$address" >&2 - printf 'A step names a heading the page no longer carries, or an occurrence under it that no longer exists.\n' >&2 - printf 'Renaming a heading is a structural edit to the journey; walk it again, then name the new heading in %s.\n' \ - "${BASH_SOURCE[0]}" >&2 - printf 'The page currently holds these sh fences:\n' >&2 - awk -F '\t' '{ printf " fence %s, occurrence %s under \"%s\"\n", $1, $2, $3 }' \ - "$fence_dir/index.tsv" >&2 - exit 1 - fi - printf '%s\n' "$matched" -} - -# Resolve a heading address that must name exactly one sh fence. -resolve_one_fence() { - local slug="$1" address="$2" fence_dir="$3" step_kind="$4" - local matched - matched="$(resolve_fences "$slug" "$address" "$fence_dir")" || exit $? - local -a numbers - read -r -a numbers <<<"$matched" - if ((${#numbers[@]} != 1)); then - printf 'tutorial spec error in %s: a %s step runs one fence, but "%s" names %d; add |\n' \ - "$slug" "$step_kind" "$address" "${#numbers[@]}" >&2 - exit 2 - fi - printf '%s\n' "${numbers[0]}" -} - -# Emit the sh fences named by a run: step, in document order. -emit_run_step() { - local slug="$1" address="$2" fence_dir="$3" - local matched - matched="$(resolve_fences "$slug" "$address" "$fence_dir")" || exit $? - local -a numbers - read -r -a numbers <<<"$matched" - local number - for number in "${numbers[@]}"; do - printf '\nprintf "==> %s fence %s\\n"\n' "$slug" "$number" - cat "$fence_dir/fence-$number.sh" - done -} - -# Emit one sh fence the tutorial documents as refused, and require it to fail. -# -# A refusal the tutorial teaches is as much a documented outcome as a success, -# so replaying it means asserting the non-zero exit rather than tolerating it: -# a fence that starts succeeding has stopped teaching what the page says. -# -# The fence runs on its own line rather than as an `if` condition, because bash -# suppresses errexit throughout a condition, subshells included, even one that -# sets it itself. A fence that refuses on its first command and then prints -# would run that print and report success, which is neither what the reader -# sees nor what the page documents. `set +e` around the run keeps the failure -# from ending the journey, and reinstates errexit for the steps after it. -emit_run_fails_step() { - local slug="$1" address="$2" fence_dir="$3" - local number - number="$(resolve_one_fence "$slug" "$address" "$fence_dir" run-fails)" || exit $? - printf '\nprintf "==> %s fence %s (documented refusal)\\n"\n' "$slug" "$number" - printf 'set +e\n' - printf '( set -e\n' - cat "$fence_dir/fence-$number.sh" - printf ')\nrefusal_status=$?\nset -e\n' - printf 'if ((refusal_status == 0))\nthen\n' - printf ' printf "tutorial drift in %s: fence %s succeeded, but the page documents a refusal\\n" >&2\n' \ - "$slug" "$number" - printf ' exit 1\n' - printf 'fi\n' -} - -# Put the client package assembled from this checkout where the tutorial's -# install fence puts the released one: on the import path of the shell the -# reader's commands run in. The replay userland carries unzip and no installer, -# so the wheel is unpacked rather than installed, which is enough because the -# package declares no dependencies of its own. -emit_python_client_step() { - local slug="$1" - printf '\nprintf "==> %s unpack the client package assembled from this checkout\\n"\n' "$slug" - printf 'mkdir -p client-package\n' - printf 'unzip -q -o %q -d client-package\n' "$REGISTRY_CLIENT_PY_WHEEL" - # shellcheck disable=SC2016 # PYTHONPATH expands in the emitted script - printf 'PYTHONPATH="$PWD/client-package${PYTHONPATH:+:$PYTHONPATH}"\nexport PYTHONPATH\n' -} - -# Emit a documented before/after fence pair applied to a file the reader edits. -# -# Both fences are read out of the tutorial here, while the journey is being -# assembled, so a pair the tutorial no longer carries fails by name before the -# reader's first command runs. -emit_edit_step() { - local slug="$1" spec="$2" tutorial_file="$3" edit_dir="$4" - local IFS='|' - # shellcheck disable=SC2206 # deliberate split on the field separator - local parts=($spec) - if ((${#parts[@]} != 7)); then - printf 'tutorial spec error in %s: edit step needs 7 fields, got %d: %s\n' \ - "$slug" "${#parts[@]}" "$spec" >&2 - exit 2 - fi - EDIT_INDEX=$((EDIT_INDEX + 1)) - local before after - before="$(printf '%s/edit-%02d-before' "$edit_dir" "$EDIT_INDEX")" - after="$(printf '%s/edit-%02d-after' "$edit_dir" "$EDIT_INDEX")" - if ! bash "$FENCE" write-fence "$tutorial_file" \ - "${parts[0]}" "${parts[1]}" "${parts[2]}" "$before" || - ! bash "$FENCE" write-fence "$tutorial_file" \ - "${parts[3]}" "${parts[4]}" "${parts[5]}" "$after"; then - printf 'tutorial drift in %s: edit step names a fence the tutorial no longer carries: %s\n' \ - "$slug" "$spec" >&2 - exit 1 - fi - printf '\nprintf "==> %s edit %s\\n"\n' "$slug" "${parts[6]}" - # shellcheck disable=SC2016 # FENCE expands in the emitted script - printf 'bash "$FENCE" replace-block %q %q %q\n' "${parts[6]}" "$before" "$after" -} - -# Save a documented non-shell fence as the file the reader is instructed to -# create. The maintained Markdown remains the single source of those bytes. -emit_save_step() { - local slug="$1" spec="$2" - local IFS='|' - # shellcheck disable=SC2206 # deliberate split on the field separator - local parts=($spec) - if ((${#parts[@]} != 4)); then - printf 'tutorial spec error in %s: save step needs 4 fields, got %d: %s\n' \ - "$slug" "${#parts[@]}" "$spec" >&2 - exit 2 - fi - printf '\nprintf "==> %s save %s\\n"\n' "$slug" "${parts[3]}" - # shellcheck disable=SC2016 # FENCE and TUTORIAL expand in the emitted script - printf 'bash "$FENCE" write-fence "$TUTORIAL" %q %q %q %q\n' \ - "${parts[0]}" "${parts[1]}" "${parts[2]}" "${parts[3]}" -} - -# A tutorial may ask the reader to leave one foreground command running in a -# second terminal. CI runs that exact one-line command in the background and -# retains its PID for cleanup. -emit_background_step() { - local slug="$1" address="$2" fence_dir="$3" - local number - number="$(resolve_one_fence "$slug" "$address" "$fence_dir" background)" || exit $? - local fence="$fence_dir/fence-$number.sh" - if [[ "$(wc -l <"$fence")" -ne 1 ]]; then - printf 'tutorial spec error in %s: a background step needs one sh line, but fence %s under "%s" holds more\n' \ - "$slug" "$number" "$address" >&2 - exit 2 - fi - local command - IFS= read -r command <"$fence" - printf '\nprintf "==> %s fence %s (background)\\n"\n' "$slug" "$number" - printf '%s &\n' "$command" - printf 'BACKGROUND_PIDS+=("$!")\n' -} - -# Stop the foreground command the page told the reader to leave running in -# another terminal. This models Ctrl+C without adding a shell fence that a -# reader would never type. -emit_stop_background_step() { - local slug="$1" - printf '\nprintf "==> %s stop the previous background fence\\n"\n' "$slug" - printf 'if ((${#BACKGROUND_PIDS[@]} == 0)); then printf "tutorial spec error in %s: no background fence to stop\\n" >&2; exit 2; fi\n' "$slug" - printf 'background_index=$((${#BACKGROUND_PIDS[@]} - 1))\n' - printf 'background_pid="${BACKGROUND_PIDS[$background_index]}"\n' - printf 'kill "$background_pid" >/dev/null 2>&1 || true\n' - printf 'wait "$background_pid" >/dev/null 2>&1 || true\n' - printf 'unset "BACKGROUND_PIDS[$background_index]"\n' -} - -emit_wait_http_step() { - local url="$1" - printf '\nfor attempt in {1..50}; do\n' - printf ' if curl -fs %q >/dev/null 2>&1; then break; fi\n' "$url" - printf ' if [[ "$attempt" -eq 50 ]]; then printf "tutorial service did not become ready\\n" >&2; exit 1; fi\n' - printf ' sleep 0.1\n' - printf 'done\n' -} - -emit_fhir_mock_step() { - printf '\nprintf "==> start sanitized local FHIR mock\\n"\n' - printf '%q >%q 2>&1 &\n' "$FHIR_TUTORIAL_MOCK" "$WORK_ROOT/fhir-tutorial-mock.log" - printf 'BACKGROUND_PIDS+=("$!")\n' - printf 'for attempt in {1..50}; do\n' - printf ' if curl --noproxy "*" -fs http://127.0.0.1:8003/healthz >/dev/null 2>&1; then break; fi\n' - printf ' if [[ "$attempt" -eq 50 ]]; then printf "sanitized FHIR mock did not become ready\\n" >&2; exit 1; fi\n' - printf ' sleep 0.1\n' - printf 'done\n' -} - -emit_track_pid_step() { - local path="$1" - printf '\ntracked_pid="$(cat %q)"\n' "$path" - printf 'if [[ ! "$tracked_pid" =~ ^[1-9][0-9]*$ ]]; then printf %q >&2; exit 1; fi\n' \ - "invalid tracked PID in $path\n" - printf 'BACKGROUND_PIDS+=("$tracked_pid")\n' -} - -emit_journey() { - local slug="$1" fence_dir="$2" tutorial_file="$3" - local edit_dir="$WORK_ROOT/edits/$slug" - mkdir -p "$edit_dir" - EDIT_INDEX=0 - printf 'set -euo pipefail\n' - printf 'FENCE=%q\n' "$FENCE" - printf 'TUTORIAL=%q\n' "$tutorial_file" - printf 'BACKGROUND_PIDS=()\n' - printf 'cleanup_journey() {\n' - printf ' if [[ -S .evidence/dev/control.sock ]]; then evidencectl dev stop >/dev/null 2>&1 || true; fi\n' - printf ' local pid\n' - printf ' for pid in "${BACKGROUND_PIDS[@]}"; do kill "$pid" >/dev/null 2>&1 || true; wait "$pid" >/dev/null 2>&1 || true; done\n' - printf '}\n' - printf 'trap cleanup_journey EXIT\n' - printf 'trap "exit 130" HUP INT TERM\n' - local step - for step in ${SPEC_STEPS[@]+"${SPEC_STEPS[@]}"}; do - case "$step" in - run:*) emit_run_step "$slug" "${step#run:}" "$fence_dir" ;; - run-fails:*) emit_run_fails_step "$slug" "${step#run-fails:}" "$fence_dir" ;; - python-client) emit_python_client_step "$slug" ;; - fhir-mock) emit_fhir_mock_step ;; - track-pid:*) emit_track_pid_step "${step#track-pid:}" ;; - edit:*) emit_edit_step "$slug" "${step#edit:}" "$tutorial_file" "$edit_dir" ;; - save:*) emit_save_step "$slug" "${step#save:}" ;; - background:*) emit_background_step "$slug" "${step#background:}" "$fence_dir" ;; - stop-background) emit_stop_background_step "$slug" ;; - wait-http:*) emit_wait_http_step "${step#wait-http:}" ;; - *) - printf 'tutorial spec error in %s: unknown step: %s\n' "$slug" "$step" >&2 - exit 2 - ;; - esac - done -} - -# Resolve every fence-addressing step into EXECUTED_FENCES, in step order. -# -# This runs before the replay and in --dry-run, so a heading a spec names but -# the page no longer carries fails by name in seconds, without a toolchain. -resolve_journey_fences() { - local slug="$1" fence_dir="$2" - EXECUTED_FENCES=() - local step matched number - local -a numbers - for step in ${SPEC_STEPS[@]+"${SPEC_STEPS[@]}"}; do - case "$step" in - run:*) matched="$(resolve_fences "$slug" "${step#run:}" "$fence_dir")" || exit $? ;; - run-fails:*) - matched="$(resolve_one_fence "$slug" "${step#run-fails:}" "$fence_dir" run-fails)" || exit $? - ;; - background:*) - matched="$(resolve_one_fence "$slug" "${step#background:}" "$fence_dir" background)" || exit $? - ;; - *) continue ;; - esac - read -r -a numbers <<<"$matched" - for number in "${numbers[@]}"; do - if ! in_list "$number" ${EXECUTED_FENCES[@]+"${EXECUTED_FENCES[@]}"}; then - EXECUTED_FENCES+=("$number") - fi - done - done -} - -# Name the sh fences the journey never runs. -# -# This is information for a reviewer, not a rule: an install one-liner or a -# recovery block a reader only reaches on a bad day is documented and -# unverified, and saying so is more use than pinning its text would be. -report_unexecuted_fences() { - local slug="$1" fence_dir="$2" - local number occurrence heading first_line - while IFS=$'\t' read -r number occurrence heading; do - if in_list "$number" ${EXECUTED_FENCES[@]+"${EXECUTED_FENCES[@]}"}; then - continue - fi - first_line="" - IFS= read -r first_line <"$fence_dir/fence-$number.sh" || true - printf ' not executed: fence %s under "%s": %s\n' "$number" "$heading" "$first_line" - done <"$fence_dir/index.tsv" -} - -# Hold the behaviours a successful exit does not already prove. -# -# Read the SPEC_ASSERTS note in the header before adding an entry here. This -# holds outcomes, never the transcript: a page is free to reword everything -# around the line, and the line itself is only here because losing it would -# leave the journey green. -assert_transcript() { - local slug="$1" run_log="$2" - local expected - for expected in ${SPEC_ASSERTS[@]+"${SPEC_ASSERTS[@]}"}; do - if ! grep -F -q -- "$expected" "$run_log"; then - printf 'tutorial behaviour drift in %s: the replay ran, but its transcript never showed "%s"\n' \ - "$slug" "$expected" >&2 - printf 'Every command exited zero, so this is the kind of regression only this assertion catches.\n' >&2 - exit 1 - fi - done -} - -# The sanitized OID4VCI runner may fall back to Cargo when CI has not supplied -# its prebuilt interoperability test. Keep that build in this gate's target -# directory without changing the documented Cargo behavior of other tutorials. -run_journey_script() { - local slug="$1" reader_dir="$2" run_script="$3" - if [[ "$slug" == "run-oid4vci-interoperability-checks" ]]; then - (cd "$reader_dir" && PATH="$SHIM_DIR:$PATH" CARGO_TARGET_DIR="$TARGET_DIR" bash "$run_script") - elif [[ "$slug" == "issue-fhir-evidence-as-vcs" ]]; then - ( - unset CARGO_TARGET_DIR - cd "$reader_dir" - PATH="$SHIM_DIR:$PATH" \ - FHIR_TUTORIAL_TEST_BASE_URL="http://127.0.0.1:8003" \ - bash "$run_script" - ) - else - ( - unset CARGO_TARGET_DIR - cd "$reader_dir" - PATH="$SHIM_DIR:$PATH" bash "$run_script" - ) - fi -} - -# --------------------------------------------------------------------------- -# Replay -# --------------------------------------------------------------------------- - -if ((DRY_RUN == 0)) && ((${#EVIDENCE_TUTORIALS[@]} > 0)); then - prepare_toolset -fi - -for slug in "${EVIDENCE_TUTORIALS[@]}"; do - load_spec "$slug" - tutorial_file="$DOCS_ROOT/$slug.mdx" - if [[ ! -f "$tutorial_file" ]]; then - printf 'Evidence tutorial not found: %s\n' "$tutorial_file" >&2 - exit 1 - fi - - # Extract every sh fence, in order, into numbered files, and index each one - # by the heading it sits under and its occurrence there. Heading - # attribution matches the fence helper the save and edit steps use, so one - # address means the same thing everywhere in a spec: a level-2 heading opens - # a section, and occurrences are counted per heading. - fence_dir="$WORK_ROOT/fences/$slug" - mkdir -p "$fence_dir" - : >"$fence_dir/index.tsv" - fence_count="$(awk -v outdir="$fence_dir" -v index_file="$fence_dir/index.tsv" ' - in_fence == 0 && /^##[ \t]+/ { - heading = $0 - sub(/^##[ \t]+/, "", heading) - sub(/[ \t]+$/, "", heading) - next - } - in_fence == 0 && /^```[A-Za-z0-9_-]+$/ { - in_fence = 1 - capture = ($0 == "```sh") - if (capture) { - count += 1 - occurrence[heading] += 1 - printf "%02d\t%d\t%s\n", count, occurrence[heading], heading > index_file - } - next - } - in_fence && /^```$/ { in_fence = 0; capture = 0; next } - in_fence && capture { print > (outdir "/fence-" sprintf("%02d", count) ".sh") } - END { print count + 0 } - ' "$tutorial_file")" - - resolve_journey_fences "$slug" "$fence_dir" - - printf '%s: %s sh fences, %s executed\n' \ - "$slug" "$fence_count" "${#EXECUTED_FENCES[@]}" - report_unexecuted_fences "$slug" "$fence_dir" - - if ((DRY_RUN)); then - continue - fi - - # Replay the journey in one shell so `cd` persists exactly as a reader - # experiences it, from a reader directory of this tutorial's own. - case "$slug" in - first-evidence-assertion) - reader_dir="$WORK_ROOT/reader/evidence-start" - ;; - run-oid4vci-interoperability-checks) - # The runner remains sourced from the checkout, but the copied adopter - # configuration belongs to a fresh writable reader directory. This also - # proves the journey in CI, where the checkout is mounted read-only. - reader_dir="$WORK_ROOT/reader/run-oid4vci-interoperability-checks" - ;; - request-evidence-as-sd-jwt-vc) - # This follow-up deliberately rewrites the starter project to explore a - # structured VC. Give it a copy so the other follow-ups still begin from - # the exact project produced by first-evidence-assertion. - reader_dir="$WORK_ROOT/reader/request-evidence-as-sd-jwt-vc" - cp -R "$WORK_ROOT/reader/evidence-start/first-evidence-assertion" "$reader_dir" - ;; - request-evidence-from-an-application) - # This follow-up gives the project its first access policy, which - # retires the unnamed development caller the other follow-ups still - # use, and writes a trusted key file one of them writes too. Both are - # the reader's own project to change, so it gets a copy, and it takes - # it here, before any follow-up has touched the starter project. - reader_dir="$WORK_ROOT/reader/request-evidence-from-an-application" - cp -R "$WORK_ROOT/reader/evidence-start/first-evidence-assertion" "$reader_dir" - ;; - return-a-governed-value) - reader_dir="$WORK_ROOT/reader/evidence-start/first-evidence-assertion" - ;; - control-who-can-request-evidence) - reader_dir="$WORK_ROOT/reader/evidence-start/first-evidence-assertion" - ;; - refuse-unsafe-evidence-requests) - reader_dir="$WORK_ROOT/reader/evidence-start/first-evidence-assertion" - ;; - verify-an-assertion-as-a-consumer) - reader_dir="$WORK_ROOT/reader/evidence-start/first-evidence-assertion" - ;; - *) reader_dir="$WORK_ROOT/reader/$slug" ;; - esac - mkdir -p "$reader_dir" - if [[ "$slug" == "run-oid4vci-interoperability-checks" ]]; then - ln -s "$REPO_ROOT/products" "$reader_dir/products" - ln -s "$REPO_ROOT/crates" "$reader_dir/crates" - ln -s "$REPO_ROOT/Cargo.toml" "$reader_dir/Cargo.toml" - ln -s "$REPO_ROOT/Cargo.lock" "$reader_dir/Cargo.lock" - fi - for step in ${SPEC_STEPS[@]+"${SPEC_STEPS[@]}"}; do - if [[ "$step" == "python-client" ]]; then - prepare_python_client - fi - done - run_script="$WORK_ROOT/run-$slug.sh" - emit_journey "$slug" "$fence_dir" "$tutorial_file" >"$run_script" - - run_log="$WORK_ROOT/run-$slug.log" - if ! run_journey_script "$slug" "$reader_dir" "$run_script" 2>&1 | - tee "$run_log"; then - printf 'tutorial %s failed; the transcript ends just before this line\n' \ - "$slug" >&2 - exit 1 - fi - - assert_transcript "$slug" "$run_log" -done - -if ((${#EVIDENCE_TUTORIALS[@]} == 1)); then - printf 'Checked 1 tutorial.\n' -else - printf 'Checked %d tutorials.\n' "${#EVIDENCE_TUTORIALS[@]}" -fi diff --git a/docs/site/scripts/check-evidence-tutorials.test.mjs b/docs/site/scripts/check-evidence-tutorials.test.mjs deleted file mode 100644 index d3fa31c3a..000000000 --- a/docs/site/scripts/check-evidence-tutorials.test.mjs +++ /dev/null @@ -1,804 +0,0 @@ -import assert from 'node:assert/strict'; -import { execFile } from 'node:child_process'; -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { dirname, join, resolve } from 'node:path'; -import test from 'node:test'; -import { fileURLToPath } from 'node:url'; -import { promisify } from 'node:util'; - -const execFileAsync = promisify(execFile); -const scriptDir = dirname(fileURLToPath(import.meta.url)); -const gate = resolve(scriptDir, 'check-evidence-tutorials.sh'); -const fenceHelper = resolve(scriptDir, 'evidence-tutorial-fence.sh'); -const fhirTutorial = resolve( - scriptDir, - '../src/content/docs/tutorials/issue-fhir-evidence-as-vcs.mdx', -); - -async function runGate(env = {}, args = ['--dry-run']) { - try { - const { stdout, stderr } = await execFileAsync('bash', [gate, ...args], { - env: { ...process.env, ...env }, - }); - return { code: 0, output: `${stdout}${stderr}` }; - } catch (error) { - return { code: error.code ?? 1, output: `${error.stdout}${error.stderr}` }; - } -} - -async function runShell(script) { - try { - const { stdout, stderr } = await execFileAsync('bash', ['-c', script]); - return { code: 0, output: `${stdout}${stderr}` }; - } catch (error) { - return { code: error.code ?? 1, output: `${error.stdout}${error.stderr}` }; - } -} - -// Counts are reported, never required: a writer who adds or removes a command -// block under an existing heading changes these numbers and neither the gate -// nor this test may object. Only the registration is asserted. -test('the dry-run gate resolves every registered Evidence tutorial', async () => { - const { code, output } = await runGate(); - assert.equal(code, 0, output); - for (const slug of [ - 'first-evidence-assertion', - 'request-evidence-as-sd-jwt-vc', - 'run-oid4vci-interoperability-checks', - 'request-evidence-from-an-application', - 'return-a-governed-value', - 'assert-a-role-bound-relationship', - 'refuse-unsafe-evidence-requests', - 'verify-an-assertion-as-a-consumer', - 'control-who-can-request-evidence', - 'issue-fhir-evidence-as-vcs', - 'connect-a-sqlite-extract', - ]) { - assert.match(output, new RegExp(`${slug}: \\d+ sh fences, \\d+ executed`, 'u')); - } - assert.match(output, /Checked 11 tutorials\./u); -}); - -// The unexecuted surface is information a reviewer needs, not a rule: the -// install one-liner and the port-conflict recovery block are documented and -// never replayed, so the gate says so rather than pinning their text. -test('the gate names the sh fences it did not execute', async () => { - const { code, output } = await runGate({}, ['--dry-run', '--only', 'first-evidence-assertion']); - assert.equal(code, 0, output); - assert.match(output, /not executed: fence 01 under "Install Evidence Gateway"/u); - assert.match(output, /not executed: fence \d+ under "If local ports are already in use"/u); -}); - -test('--only accepts the current first Evidence tutorial', async () => { - const { code, output } = await runGate({}, [ - '--dry-run', - '--only', - 'first-evidence-assertion', - ]); - assert.equal(code, 0, output); - assert.match(output, /Checked 1 tutorial\./u); - const source = await readFile(gate, 'utf8'); - const branch = source.match(/\n\tfirst-evidence-assertion\)[\s\S]*?\n\t\t;;/u)?.[0]; - assert.ok(branch, 'the first Evidence replay spec must exist'); - assert.match(branch, /stop-background/u); - assert.match(branch, /run:Preview a synthetic source\|3/u); - assert.match(branch, /run:Try the SD-JWT VC serialization/u); -}); - -test('--only accepts the role-bound relationship follow-up', async () => { - const { code, output } = await runGate({}, [ - '--dry-run', - '--only', - 'assert-a-role-bound-relationship', - ]); - assert.equal(code, 0, output); - assert.match(output, /Checked 1 tutorial\./u); -}); - -test('--only accepts the deterministic FHIR tutorial replay', async () => { - const { code, output } = await runGate({}, [ - '--dry-run', - '--only', - 'issue-fhir-evidence-as-vcs', - ]); - assert.equal(code, 0, output); - assert.match(output, /issue-fhir-evidence-as-vcs: 10 sh fences, 10 executed/u); - assert.match(output, /Checked 1 tutorial\./u); -}); - -test('both FHIR tutorial clients bypass ambient proxies', async () => { - const source = await readFile(fhirTutorial, 'utf8'); - const proxyFreeOpeners = source.match( - /build_opener\(ProxyHandler\(\{\}\), NoRedirect\)/gu, - ); - assert.equal(proxyFreeOpeners?.length, 2); -}); - -test('the FHIR tutorial test origin refuses a remote endpoint', async () => { - const root = await mkdtemp(join(tmpdir(), 'fhir-tutorial-origin-test-')); - const discovery = join(root, 'discover-fhir-records.py'); - try { - await execFileAsync('bash', [ - fenceHelper, - 'write-fence', - fhirTutorial, - 'Select live synthetic records', - 'python', - '1', - discovery, - ]); - await assert.rejects( - execFileAsync('python3', [discovery], { - env: { - ...process.env, - FHIR_TUTORIAL_TEST_BASE_URL: 'https://example.com', - }, - }), - (error) => { - assert.match(error.stderr, /test origin must be numeric loopback HTTP/u); - return true; - }, - ); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('the FHIR replay tracks the read-through adapter for cleanup', async () => { - const source = await readFile(gate, 'utf8'); - const branch = source.match( - /\n\tissue-fhir-evidence-as-vcs\)[\s\S]*?\n\t\t;;/u, - )?.[0]; - assert.ok(branch, 'the FHIR replay spec must exist'); - assert.match( - branch, - /"run:Run a live FHIR read-through adapter"\s+"track-pid:fhir-read-through\.pid"/u, - ); - assert.match(source, /track-pid:\*\) emit_track_pid_step/u); - assert.match(source, /BACKGROUND_PIDS\+=\("\$tracked_pid"\)/u); -}); - -// Every follow-up below begins from the project first-evidence-assertion -// builds. A full run gets that from the registration order, so a --only that -// skipped it would fail on the reader directory rather than on the tutorial, -// and only for the person running one slug by hand. -for (const slug of [ - 'request-evidence-as-sd-jwt-vc', - 'request-evidence-from-an-application', - 'return-a-governed-value', - 'refuse-unsafe-evidence-requests', - 'verify-an-assertion-as-a-consumer', - 'control-who-can-request-evidence', -]) { - test(`--only runs the starter project before ${slug}`, async () => { - const { code, output } = await runGate({}, ['--dry-run', '--only', slug]); - assert.equal(code, 0, output); - const prerequisite = output.indexOf('first-evidence-assertion:'); - const followUp = output.indexOf(`${slug}:`); - assert.notEqual(prerequisite, -1, output); - assert.ok(followUp > prerequisite, output); - assert.match(output, /Checked 2 tutorials\./u); - }); -} - -// The application tutorial is the only registered replay that reaches the -// Evidence client SDK, and it reaches it through the Python binding. Losing -// either the registration or the substituted build would leave that path -// unproven while the gate still reported PASS. -test('the application tutorial replays the Python client from this checkout', async () => { - const source = await readFile(gate, 'utf8'); - assert.match(source, /^\trequest-evidence-from-an-application$/mu); - const branch = source.match( - /\n\trequest-evidence-from-an-application\)[\s\S]*?\n\t\t;;/u, - )?.[0]; - assert.ok(branch, 'the application replay spec must exist'); - assert.match(branch, /"python-client"/u); - assert.match(branch, /person-123 is_adult=True/u); - assert.match(branch, /person-456 is_adult=False/u); -}); - -// The tutorial installs the one maintained client distribution, so the gate -// has to import that same distribution: a per-product extension module would -// prove a package no reader can install, and would leave the namespaces the -// tutorial imports unexercised. -test('the application replay imports the assembled client package', async () => { - const source = await readFile(gate, 'utf8'); - const step = source.match(/\nemit_python_client_step\(\) \{\n[\s\S]*?\n\}\n/u)?.[0]; - assert.ok(step, 'the client step must exist'); - // The replay userland carries unzip and no installer, so the package is - // unpacked onto the import path rather than installed. - assert.match(step, /unzip/u); - assert.match(step, /PYTHONPATH/u); - assert.match(source, /REGISTRY_CLIENT_PY_WHEEL/u); - assert.doesNotMatch(source, /EVIDENCE_CLIENT_PY_LIB/u); - assert.doesNotMatch(source, /registry_evidence_client\.so/u); -}); - -// The gate cannot assemble the package itself: assembling needs a build -// toolchain the replay userland does not carry. Refusing early, by name, is -// what keeps that from surfacing as an import error twenty steps in. -test('the gate refuses a client package it cannot use', async () => { - const source = await readFile(gate, 'utf8'); - const prepare = source.match(/\nprepare_python_client\(\) \{\n[\s\S]*?\n\}\n/u)?.[0]; - assert.ok(prepare, 'the client preparation must exist'); - assert.match(prepare, /!= \/\*/u, 'the path must be required to be absolute'); - assert.match(prepare, /must be absolute/u); - assert.match(prepare, /-f "\$REGISTRY_CLIENT_PY_WHEEL"/u); - assert.match(prepare, /assemble-registry-client-packages/u); -}); - -test('the caller-access replay expects the privacy-safe refusal audit line', async () => { - const source = await readFile(gate, 'utf8'); - const branch = source.match( - /\n\tcontrol-who-can-request-evidence\)[\s\S]*?\n\t\t;;/u, - )?.[0]; - assert.ok(branch, 'the caller-access replay spec must exist'); - assert.match( - branch, - /"run:Revoke an application\|1"\s+"run:Revoke an application\|2"\s+"run-fails:Revoke an application\|3"\s+"run:Stop the final generation"\s+"run:Clean up"/u, - ); - assert.match(branch, /"ACCESS REFUSED requester="/u); - assert.match(branch, /"reason=not_authorized"/u); - assert.doesNotMatch(branch, /ACCESS AUTHORIZED age-bracket/u); -}); - -// EVIDENCE_TUTORIALS and EXCLUDED_EVIDENCE_TUTORIALS between them must -// account for every page under the tutorials directory, so a new page can -// never ship unreplayed and unexplained. Read the lists from the gate -// itself rather than restating them, so this test tracks the gate instead -// of drifting from it. -function extractBashArray(source, name) { - const match = source.match(new RegExp(`\\n${name}=\\(([\\s\\S]*?)\\n\\)`, 'u')); - assert.ok(match, `${name} array must exist in the gate`); - return match[1] - .split('\n') - .map((line) => line.split('#')[0].trim()) - .filter(Boolean); -} - -test('the tutorial coverage check fails on an unregistered page', async () => { - const source = await readFile(gate, 'utf8'); - const excluded = extractBashArray(source, 'EXCLUDED_EVIDENCE_TUTORIALS'); - const root = await mkdtemp(join(tmpdir(), 'evidence-tutorial-coverage-test-')); - try { - // Stub every already-excluded page so only the deliberately unregistered - // page below can trip the check. - for (const slug of excluded) { - await writeFile(join(root, `${slug}.mdx`), '---\ntitle: stub\n---\n'); - } - await writeFile(join(root, 'orphan-tutorial.mdx'), '---\ntitle: stub\n---\n'); - const { code, output } = await runGate({ EVIDENCE_TUTORIAL_DOCS_ROOT: root }); - assert.notEqual(code, 0, 'an unregistered tutorial page must fail the gate'); - assert.match(output, /tutorial coverage gap/u); - assert.match(output, /orphan-tutorial\.mdx/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -// --------------------------------------------------------------------------- -// Heading addressing -// --------------------------------------------------------------------------- - -// Build a tutorials directory the gate will accept: every excluded page must -// exist, and the one registered page under test is the real one, edited. -async function tutorialFixtureRoot(edit) { - const source = await readFile(gate, 'utf8'); - const excluded = extractBashArray(source, 'EXCLUDED_EVIDENCE_TUTORIALS'); - const root = await mkdtemp(join(tmpdir(), 'evidence-tutorial-heading-test-')); - for (const slug of excluded) { - await writeFile(join(root, `${slug}.mdx`), '---\ntitle: stub\n---\n'); - } - const page = await readFile( - resolve(scriptDir, '../src/content/docs/tutorials/first-evidence-assertion.mdx'), - 'utf8', - ); - await writeFile(join(root, 'first-evidence-assertion.mdx'), edit(page)); - return root; -} - -// The point of heading addressing. A writer who adds a command block under a -// heading the journey already runs must not have to touch the gate, and the -// added block must be replayed rather than silently skipped. -test('a command block added under a replayed heading needs no gate change', async () => { - const root = await tutorialFixtureRoot((page) => - page.replace( - '\n## Verify before reading\n', - '\n```sh\nevidencectl request list\n```\n\n## Verify before reading\n', - ), - ); - try { - const before = await runGate({}, ['--dry-run', '--only', 'first-evidence-assertion']); - assert.equal(before.code, 0, before.output); - const baseline = before.output.match( - /first-evidence-assertion: (\d+) sh fences, (\d+) executed/u, - ); - assert.ok(baseline, before.output); - - const { code, output } = await runGate( - { EVIDENCE_TUTORIAL_DOCS_ROOT: root }, - ['--dry-run', '--only', 'first-evidence-assertion'], - ); - assert.equal(code, 0, output); - const added = output.match(/first-evidence-assertion: (\d+) sh fences, (\d+) executed/u); - assert.ok(added, output); - assert.equal(Number(added[1]), Number(baseline[1]) + 1); - assert.equal(Number(added[2]), Number(baseline[2]) + 1); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -// The trade heading addressing makes: a renamed heading is a structural edit -// to the journey, so it fails, by name, before any command runs. -test('a renamed heading fails the gate by name', async () => { - const root = await tutorialFixtureRoot((page) => - page.replace('\n## Request an assertion\n', '\n## Ask for an assertion\n'), - ); - try { - const { code, output } = await runGate( - { EVIDENCE_TUTORIAL_DOCS_ROOT: root }, - ['--dry-run', '--only', 'first-evidence-assertion'], - ); - assert.notEqual(code, 0, 'a renamed heading must fail the gate'); - assert.match(output, /no sh fence answers to "Request an assertion"/u); - // The message has to be actionable: it names the headings the page does - // carry, so the fix is reading the list rather than the script. - assert.match(output, /Ask for an assertion/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -// A heading holding more than one sh fence cannot answer a step that runs -// exactly one command, so the gate says which suffix is missing. -test('a one-fence step under a multi-fence heading names the missing occurrence', async () => { - const source = await readFile(gate, 'utf8'); - const root = await mkdtemp(join(tmpdir(), 'evidence-occurrence-test-')); - try { - await writeFile(join(root, 'index.tsv'), '01\t1\tRun it\n02\t2\tRun it\n'); - const harness = join(root, 'resolve.sh'); - await writeFile( - harness, - [ - '#!/usr/bin/env bash', - 'set -euo pipefail', - await liftFunction(source, 'resolve_fences'), - await liftFunction(source, 'resolve_one_fence'), - 'resolve_one_fence tutorial "Run it" "$1" background', - '', - ].join('\n'), - ); - const { code, output } = await runShell(`bash ${harness} ${root}`); - assert.notEqual(code, 0, 'an ambiguous one-fence step must fail'); - assert.match(output, /names 2/u); - assert.match(output, /\|/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -// --------------------------------------------------------------------------- -// Behaviour assertions -// --------------------------------------------------------------------------- - -async function runAssertTranscript(asserts, transcript) { - const source = await readFile(gate, 'utf8'); - const root = await mkdtemp(join(tmpdir(), 'evidence-asserts-test-')); - const log = join(root, 'run.log'); - await writeFile(log, transcript); - const harness = join(root, 'assert.sh'); - await writeFile( - harness, - [ - '#!/usr/bin/env bash', - 'set -euo pipefail', - await liftFunction(source, 'assert_transcript'), - `SPEC_ASSERTS=(${asserts.map((entry) => `'${entry}'`).join(' ')})`, - `assert_transcript tutorial '${log}'`, - '', - ].join('\n'), - ); - try { - return await runShell(`bash ${harness}`); - } finally { - await rm(root, { recursive: true, force: true }); - } -} - -test('a retained behaviour assertion missing from the transcript fails', async () => { - const { code, output } = await runAssertTranscript( - ['VERIFIED', 'DISCLOSURE RELEASED is_adult'], - '==> fence 12\nVERIFIED\n==> fence 19\nACCESS AUTHORIZED adult-status age-check requester=x\n', - ); - assert.notEqual(code, 0, 'a missing behaviour must fail the gate'); - assert.match(output, /DISCLOSURE RELEASED is_adult/u); -}); - -test('a transcript showing every retained behaviour passes', async () => { - const { code, output } = await runAssertTranscript( - ['VERIFIED', 'DISCLOSURE RELEASED is_adult'], - 'VERIFIED\nDISCLOSURE RELEASED is_adult\n', - ); - assert.equal(code, 0, output); -}); - -// Every retained assertion has to earn its place by regressing silently. The -// two the gate must never lose are the refusal that actually refused and the -// tamper that was actually caught, and both must be words a tool printed. A -// page that echoes its own verdict asserts nothing: the echo survives the -// regression it was supposed to catch and leaves the transcript quietly clean. -test('the refusal tutorial still asserts the refusal and the tamper', async () => { - const source = await readFile(gate, 'utf8'); - const branch = source.match( - /\n\trefuse-unsafe-evidence-requests\)[\s\S]*?\n\t\t;;/u, - )?.[0]; - assert.ok(branch, 'the refusal replay spec must exist'); - assert.match(branch, /"HTTP 403"/u); - assert.match(branch, /"evidencectl: Evidence response verification failed"/u); - // Startup chatter a successful exit already proves does not belong here. - assert.doesNotMatch(branch, /Evidence ready at/u); - assert.doesNotMatch(branch, /Prepared request:/u); - assert.doesNotMatch(branch, /Local Evidence stopped/u); -}); - -// This gate proves the documented commands still run. It does not police what -// a page says, and the two arrays below are how it used to: one pinned how -// many command blocks a page held, the other pinned strings the page had to -// keep. Both made ordinary prose edits fail CI, and neither verified anything -// replay does not already verify. Reintroducing either is the regression this -// test exists to catch. -test('the gate pins neither fence counts nor page strings', async () => { - const source = await readFile(gate, 'utf8'); - assert.doesNotMatch(source, /SPEC_FENCES/u); - assert.doesNotMatch(source, /SPEC_LITERALS/u); -}); - -test('--only refuses a slug that is not registered', async () => { - const { code, output } = await runGate({}, ['--dry-run', '--only', 'no-such-tutorial']); - assert.notEqual(code, 0, 'an unregistered slug must fail the gate'); - assert.match(output, /not a registered Evidence tutorial/u); -}); - -test('--only refuses an unpublished legacy tutorial', async () => { - const { code, output } = await runGate({}, [ - '--dry-run', - '--only', - 'serve-assertions-over-http', - ]); - assert.notEqual(code, 0, 'an unpublished tutorial must not be registered'); - assert.match(output, /not a registered Evidence tutorial/u); -}); - -// The replay runs inside a clean Debian userland holding a shell, coreutils -// and the toolset under test. An interpreter the container does not carry -// fails mid-journey, where the transcript makes it look like a tutorial -// defect, so the gate and everything it emits stay on that floor. -test('the gate depends on no interpreter beyond the replay userland', async () => { - const source = await readFile(gate, 'utf8'); - const offenders = source - .split('\n') - .map((line, index) => [index + 1, line]) - // `python-client` is a step name and `python-module` is the directory the - // application tutorial imports from. Both are data the gate writes or - // matches, never an interpreter it runs, so they are removed before the - // line is judged rather than exempting whole lines that carry them. - .map(([number, line]) => [number, line.replaceAll(/python-(?:client|module)/gu, '')]) - .filter(([, line]) => /\b(?:node|npm|npx|python3?|ruby|perl)\b/u.test(line)) - // A save step names the Markdown fence language as data. It extracts that - // fence with the shell helper and does not execute the named interpreter. - .filter(([, line]) => !/^\s*"save:[^"]+\|[^|]+\|\d+\|[^"]+",?$/u.test(line)); - assert.deepEqual(offenders, [], 'the gate must not reach for an interpreter'); -}); - -async function replayCargoTarget(slug) { - const source = await readFile(gate, 'utf8'); - const runner = source.match(/\nrun_journey_script\(\) \{\n[\s\S]*?\n\}\n/u)?.[0]; - assert.ok(runner, 'the journey runner must exist'); - const root = await mkdtemp(join(tmpdir(), 'evidence-cargo-target-test-')); - const journey = join(root, 'journey.sh'); - const harness = join(root, 'run.sh'); - await writeFile(journey, 'printf "%s\\n" "${CARGO_TARGET_DIR-unset}"\n'); - await writeFile( - harness, - [ - '#!/usr/bin/env bash', - 'set -euo pipefail', - runner, - 'SHIM_DIR="$1"', - 'TARGET_DIR="$2"', - 'run_journey_script "$3" "$1" "$4"', - '', - ].join('\n'), - ); - try { - const expectedTarget = join(root, 'oid4vci-target'); - const { stdout } = await execFileAsync( - 'bash', - [harness, root, expectedTarget, slug, journey], - { - env: { ...process.env, CARGO_TARGET_DIR: join(root, 'inherited-target') }, - }, - ); - return { output: stdout.trim(), expectedTarget }; - } finally { - await rm(root, { recursive: true, force: true }); - } -} - -test('the OID4VCI replay receives the gate Cargo target directory', async () => { - const { output, expectedTarget } = await replayCargoTarget( - 'run-oid4vci-interoperability-checks', - ); - assert.equal(output, expectedTarget); -}); - -test('other tutorial replays do not receive a Cargo target directory', async () => { - const { output } = await replayCargoTarget('first-evidence-assertion'); - assert.equal(output, 'unset'); -}); - -async function runFence(args) { - try { - const { stdout, stderr } = await execFileAsync('bash', [fenceHelper, ...args]); - return { code: 0, output: `${stdout}${stderr}` }; - } catch (error) { - return { code: error.code ?? 1, output: `${error.stdout}${error.stderr}` }; - } -} - -const fenceFixture = [ - '---', - 'title: A tutorial', - '---', - '', - '## Add a narrower selector profile', - '', - 'Before:', - '', - '```yaml', - '', - 'selectors:', - ' - kind: broad', - '', - '```', - '', - 'After:', - '', - '```yaml', - 'selectors:', - ' - kind: narrow', - '```', - '', - '## Run it', - '', - '```sh', - 'evidencectl check', - '```', - '', -].join('\n'); - -async function fenceScratch() { - const root = await mkdtemp(join(tmpdir(), 'evidence-fence-test-')); - await writeFile(join(root, 'tutorial.mdx'), fenceFixture); - return root; -} - -// Lift one named function out of the gate. Sourcing the gate would run it, so -// the tests below exercise the shipped text of the function instead of -// restating it. -async function liftFunction(source, name) { - const lifted = source.match( - new RegExp(`\\n${name}\\(\\) \\{\\n[\\s\\S]*?\\n\\}\\n`, 'u'), - )?.[0]; - assert.ok(lifted, `${name} must exist in the gate`); - return lifted; -} - -// Run the gate's own run-fails emitter over one fence addressed by heading, -// and return the journey lines it emits. -async function emitRunFailsStep(fenceBody) { - const source = await readFile(gate, 'utf8'); - const emitter = [ - await liftFunction(source, 'resolve_fences'), - await liftFunction(source, 'resolve_one_fence'), - await liftFunction(source, 'emit_run_fails_step'), - ].join('\n'); - const root = await mkdtemp(join(tmpdir(), 'evidence-refusal-test-')); - await writeFile(join(root, 'fence-09.sh'), fenceBody); - await writeFile(join(root, 'index.tsv'), '09\t1\tRefuse before reading\n'); - const harness = join(root, 'emit.sh'); - await writeFile( - harness, - [ - '#!/usr/bin/env bash', - 'set -euo pipefail', - emitter, - 'emit_run_fails_step tutorial "Refuse before reading" "$1"', - '', - ].join('\n'), - ); - const { stdout } = await execFileAsync('bash', [harness, root]); - return { root, journey: `set -euo pipefail\n${stdout}` }; -} - -// A refusal fence that prints after the command that refuses is the shape the -// pages actually carry: the reader sees the error, then the state it left -// behind. Bash suppresses errexit for everything inside an `if` condition, -// subshells included, so an emitter that tested the fence there would run the -// trailing line, read the whole fence as a success, and report drift on a -// tutorial that is doing exactly what it documents. -test('a documented refusal is accepted even when the fence prints after it', async () => { - const { root, journey } = await emitRunFailsStep( - 'false\nprintf "kept going\\n"\n', - ); - try { - const { code, output } = await runShell(journey); - assert.equal(code, 0, output); - assert.doesNotMatch(output, /kept going/u); - assert.doesNotMatch(output, /tutorial drift/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('a refusal fence that starts succeeding is reported as drift', async () => { - const { root, journey } = await emitRunFailsStep('true\n'); - try { - const { code, output } = await runShell(journey); - assert.notEqual(code, 0, 'a fence that no longer refuses must fail the gate'); - assert.match(output, /tutorial drift/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -// The steps after a documented refusal still run under the journey's errexit, -// so a later failure ends the journey where it happened instead of being -// carried past. -test('errexit is back in force after a documented refusal', async () => { - const { root, journey } = await emitRunFailsStep('false\n'); - try { - const { code, output } = await runShell(`${journey}\nfalse\nprintf "past it\\n"\n`); - assert.notEqual(code, 0, 'the journey must stop at the failure after the refusal'); - assert.doesNotMatch(output, /past it/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('write-fence extracts one fence by heading, language and occurrence', async () => { - const root = await fenceScratch(); - try { - const out = join(root, 'before.yaml'); - const { code, output } = await runFence([ - 'write-fence', - join(root, 'tutorial.mdx'), - 'Add a narrower selector profile', - 'yaml', - '1', - out, - ]); - assert.equal(code, 0, output); - // Blank lines at the edges of a fence are presentation, so they are - // trimmed exactly as the published fence renders. - assert.equal(await readFile(out, 'utf8'), 'selectors:\n - kind: broad\n'); - - const second = join(root, 'after.yaml'); - assert.equal((await runFence([ - 'write-fence', - join(root, 'tutorial.mdx'), - 'Add a narrower selector profile', - 'yaml', - '2', - second, - ])).code, 0); - assert.equal(await readFile(second, 'utf8'), 'selectors:\n - kind: narrow\n'); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('write-fence counts occurrences per heading and language', async () => { - const root = await fenceScratch(); - try { - const out = join(root, 'sh.txt'); - // The sh fence under a later heading is that heading's first, not the - // document's third. - const { code, output } = await runFence([ - 'write-fence', - join(root, 'tutorial.mdx'), - 'Run it', - 'sh', - '1', - out, - ]); - assert.equal(code, 0, output); - assert.equal(await readFile(out, 'utf8'), 'evidencectl check\n'); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('write-fence names the fence it could not find', async () => { - const root = await fenceScratch(); - try { - const { code, output } = await runFence([ - 'write-fence', - join(root, 'tutorial.mdx'), - 'Add a narrower selector profile', - 'yaml', - '3', - join(root, 'missing.yaml'), - ]); - assert.notEqual(code, 0, 'a missing fence must fail'); - assert.match(output, /missing yaml fence 3 under "Add a narrower selector profile"/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('replace-block applies a documented pair to the reader file', async () => { - const root = await fenceScratch(); - try { - const target = join(root, 'evidence.yaml'); - await writeFile(target, 'version: 1\nselectors:\n - kind: broad\ntrailer: keep\n'); - await writeFile(join(root, 'b'), 'selectors:\n - kind: broad\n'); - await writeFile(join(root, 'a'), 'selectors:\n - kind: narrow\n'); - const { code, output } = await runFence([ - 'replace-block', - target, - join(root, 'b'), - join(root, 'a'), - ]); - assert.equal(code, 0, output); - assert.equal( - await readFile(target, 'utf8'), - 'version: 1\nselectors:\n - kind: narrow\ntrailer: keep\n', - ); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('replace-block refuses a block that is not in the target exactly once', async () => { - const root = await fenceScratch(); - try { - const target = join(root, 'evidence.yaml'); - await writeFile(join(root, 'b'), 'kind: broad\n'); - await writeFile(join(root, 'a'), 'kind: narrow\n'); - - await writeFile(target, 'kind: broad\nkind: broad\n'); - const twice = await runFence(['replace-block', target, join(root, 'b'), join(root, 'a')]); - assert.notEqual(twice.code, 0, 'an ambiguous edit must fail'); - assert.match(twice.output, /found 2/u); - - await writeFile(target, 'kind: other\n'); - const never = await runFence(['replace-block', target, join(root, 'b'), join(root, 'a')]); - assert.notEqual(never.code, 0, 'an edit with nothing to change must fail'); - assert.match(never.output, /found 0/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('replace-block refuses a pair that changes nothing', async () => { - const root = await fenceScratch(); - try { - const target = join(root, 'evidence.yaml'); - await writeFile(target, 'kind: broad\n'); - await writeFile(join(root, 'b'), 'kind: broad\n'); - await writeFile(join(root, 'a'), 'kind: broad\n'); - const { code, output } = await runFence([ - 'replace-block', - target, - join(root, 'b'), - join(root, 'a'), - ]); - assert.notEqual(code, 0, 'a pair that changes nothing is a spec error'); - assert.match(output, /must change the target/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); diff --git a/docs/site/scripts/run-tutorial.mjs b/docs/site/scripts/run-tutorial.mjs index dc7ad06bd..81815c64b 100644 --- a/docs/site/scripts/run-tutorial.mjs +++ b/docs/site/scripts/run-tutorial.mjs @@ -1,8 +1,8 @@ #!/usr/bin/env node // Replay a tutorial page the way a reader follows it. // -// node scripts/run-tutorial.mjs [--dry-run] [--toolset breg|casework|none] ... -// node scripts/run-tutorial.mjs [--dry-run] --gate breg|casework +// node scripts/run-tutorial.mjs [--dry-run] [--toolset breg|casework|evidence|none] ... +// node scripts/run-tutorial.mjs [--dry-run] --gate breg|casework|evidence // // The page is the specification (see tutorial-runner/page.mjs): its sh fences // run in document order in one bash shell, from an empty reader directory diff --git a/docs/site/src/content/docs/tutorials/assert-a-role-bound-relationship.mdx b/docs/site/src/content/docs/tutorials/assert-a-role-bound-relationship.mdx index 1bdc6b4a1..2f0d242cd 100644 --- a/docs/site/src/content/docs/tutorials/assert-a-role-bound-relationship.mdx +++ b/docs/site/src/content/docs/tutorials/assert-a-role-bound-relationship.mdx @@ -11,6 +11,8 @@ persona: - assertion provider locale: en standards_referenced: [] +tutorial_test: + toolset: evidence --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -37,7 +39,7 @@ mkdir role-bound-relationship cd role-bound-relationship ``` -```python +```python title="registry.py" test-file import json from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import unquote, urlsplit @@ -98,7 +100,7 @@ ThreadingHTTPServer(("127.0.0.1", 8002), Registry).serve_forever() Start it in one terminal and leave it running: -```sh +```sh test-background="http://127.0.0.1:8002/openapi.json" python3 registry.py ``` @@ -121,7 +123,7 @@ cd parent-relationship Create the question definition: -```yaml +```yaml title="questions/parent-relationship.yaml" test-file id: parent-relationship question: Is the candidate registered as a parent of the child? purpose: relationship-check @@ -153,7 +155,7 @@ arguments. The source decision is the only fact that reaches the derivation. Create the answer logic: -```rhai +```rhai title="derivations/parent-relationship.rhai" test-file fn answer(facts, selectors, context) { #{ relationship_confirmed: @@ -172,7 +174,7 @@ and both subject bindings before signing. evidencectl dev start . ``` -```text +```text test-excerpt Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -213,7 +215,7 @@ evidencectl verify parent-relationship.jws.json \ --output parent-relationship.verified.json ``` -```text +```text test-expect VERIFIED ``` @@ -224,7 +226,7 @@ python3 -m json.tool parent-relationship.verified.json The verified assertion has two pseudonymous subject bindings, one for `child` and one for `candidate-parent`, plus this supported value: -```json +```json test-excerpt { "providesValueFor": "urn:registrystack:evidence:local:concept:parent-relationship:relationship_confirmed", "value": true @@ -242,6 +244,13 @@ evidencectl audit show --last-operation evidencectl dev clean ``` +```text test-expect +Local Evidence stopped +ACCESS AUTHORIZED parent-relationship relationship-check requester= +DISCLOSURE RELEASED relationship_confirmed +Removed stopped local Evidence state +``` + The audit identifies the authorized question, purpose, requester pseudonym, and disclosed concept. It does not record either source identifier or the boolean value. diff --git a/docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx b/docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx index 2bbd95f22..04495f22b 100644 --- a/docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx +++ b/docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx @@ -12,6 +12,8 @@ persona: - operator locale: en standards_referenced: [] +tutorial_test: + toolset: evidence --- Use a SQLite extract when the authority can publish an immutable snapshot but cannot offer a @@ -139,7 +141,7 @@ Create a complete governed target by following [Build and deploy an Evidence Gat project](../build-and-deploy-evidence-project/). Then build the editable project and run the checks against the exact candidate and mounted extract: -```sh +```sh test-skip="needs the complete governed deployment target from Build and deploy an Evidence Gateway project" evidencectl package registry-status \ --target registry-status/deployment-targets/staging \ --output candidate-staging diff --git a/docs/site/src/content/docs/tutorials/control-who-can-request-evidence.mdx b/docs/site/src/content/docs/tutorials/control-who-can-request-evidence.mdx index 70e697e36..841cb4642 100644 --- a/docs/site/src/content/docs/tutorials/control-who-can-request-evidence.mdx +++ b/docs/site/src/content/docs/tutorials/control-who-can-request-evidence.mdx @@ -11,6 +11,9 @@ persona: - assertion provider locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + after: tutorials/return-a-governed-value --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -65,13 +68,13 @@ A client change affects the next local generation. Stopping that generation also In one terminal, return to the working directory from the first tutorial and serve the checked synthetic cases: -```sh +```sh test-cwd="first-evidence-assertion" test-background="http://127.0.0.1:4010/people/person-123" evidencectl source mock serve --config adult-status/mocks/source.yaml ``` Leave the source mock running. In another terminal, enter the project containing both questions: -```sh +```sh test-cwd="first-evidence-assertion" cd adult-status ``` @@ -94,7 +97,7 @@ evidencectl access policy add age-checks --question adult-status evidencectl access policy add service-routing --question age-bracket ``` -```text +```text test-expect Added access policy age-checks for adult-status. Added access policy service-routing for age-bracket. ``` @@ -125,7 +128,7 @@ evidencectl access client add age-checker \ --generate-local-key ``` -```text +```text test-expect Added client age-checker with policy age-checks. ``` @@ -143,7 +146,7 @@ Compile the questions and both access policies, then start Evidence Gateway and evidencectl dev start . ``` -```text +```text test-excerpt Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -182,7 +185,7 @@ curl --silent --show-error --fail-with-body \ --write-out 'HTTP %{http_code}\n' ``` -```text +```text test-expect HTTP 200 ``` @@ -194,7 +197,7 @@ evidencectl verify age-checker-allowed.jws.json \ --output age-checker-allowed.verified.json ``` -```text +```text test-expect VERIFIED ``` @@ -213,7 +216,7 @@ evidencectl access client add service-router \ --generate-local-key ``` -```text +```text test-excerpt Added client service-router with policy service-routing. ``` @@ -264,7 +267,7 @@ evidencectl verify service-router-allowed.jws.json \ --output service-router-allowed.verified.json ``` -```text +```text test-expect HTTP 200 VERIFIED ``` @@ -306,7 +309,7 @@ curl --silent --show-error \ --write-out 'HTTP %{http_code}\n' ``` -```text +```text test-expect HTTP 403 ``` @@ -316,14 +319,14 @@ Inspect the safe problem response: python3 -m json.tool age-checker-refused.json ``` -```json +```json test-expect { "type": "https://id.registrystack.org/problems/registry-evidence/evidence/denied", "title": "Evidence request is not permitted", "status": 403, "detail": "the Evidence request is not permitted", "code": "evidence.denied", - "traceId": "<32-lowercase-hex-trace-id>" + "traceId": "" } ``` @@ -342,7 +345,7 @@ evidencectl dev stop evidencectl audit show --last-operation ``` -```text +```text test-expect Local Evidence stopped ACCESS REFUSED requester= reason=not_authorized ``` @@ -358,8 +361,8 @@ evidencectl access client revoke age-checker evidencectl dev start . ``` -```text -Revoked client age-checker. +```text test-expect +Revoked client age-checker (removed local private key .evidence/clients/age-checker). Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -369,7 +372,7 @@ revoked client. Try to prepare a fresh request as the revoked client: -```sh +```sh test-exit="1" evidencectl request prepare adult-status \ --purpose age-check \ --subject person_id=person-123 \ @@ -377,7 +380,7 @@ evidencectl request prepare adult-status \ --name age-checker-revoked ``` -```text +```text test-expect evidencectl: unknown or revoked active client age-checker ``` @@ -393,7 +396,7 @@ Stop the local services: evidencectl dev stop ``` -```text +```text test-expect Local Evidence stopped ``` diff --git a/docs/site/src/content/docs/tutorials/first-evidence-assertion.mdx b/docs/site/src/content/docs/tutorials/first-evidence-assertion.mdx index 5c674ba53..cc19859e7 100644 --- a/docs/site/src/content/docs/tutorials/first-evidence-assertion.mdx +++ b/docs/site/src/content/docs/tutorials/first-evidence-assertion.mdx @@ -11,6 +11,8 @@ persona: - assertion provider locale: en standards_referenced: [] +tutorial_test: + toolset: evidence --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -52,7 +54,7 @@ released assertion contains only the governed answer and an opaque subject bindi Install the latest Evidence Gateway toolset: -```sh +```sh test-skip="installs the released toolset; the replay serves the binaries under test" curl -fsSL https://github.com/registrystack/registry-stack/releases/latest/download/evidencectl-install.sh | bash evidencectl --version ``` @@ -77,7 +79,7 @@ cd first-evidence-assertion Open `tutorial-source.openapi.yaml` in your editor and add this OpenAPI description: -```yaml +```yaml title="tutorial-source.openapi.yaml" test-file openapi: 3.1.0 info: title: Tutorial registry @@ -136,11 +138,11 @@ Three details connect this contract to Evidence Gateway: Start a write-free preview directly from the OpenAPI description: -```sh +```sh test-background="http://127.0.0.1:4010/people/person-123" evidencectl source mock serve --openapi tutorial-source.openapi.yaml ``` -```text +```text test-excerpt Source mock ready: mode=ephemeral origin=http://127.0.0.1:4010 contract=evidencectl-source-mock-v1 seed=0 asOf=2025-01-01 digest=sha256: served=1 skipped=0 Next: evidencectl source mock generate --openapi ``` @@ -155,7 +157,7 @@ curl -s http://127.0.0.1:4010/people/person-123 | python3 -c \ 'import json,sys; print(sorted(json.load(sys.stdin)))' ``` -```text +```text test-expect ['date_of_birth', 'name', 'person_id'] ``` @@ -201,7 +203,7 @@ Later institution integrations use the reusable source, selector, adapter, and s Open `questions/adult-status.yaml` in your editor and add the question definition: -```yaml +```yaml title="questions/adult-status.yaml" test-file id: adult-status question: Is the person at least 18 years old? purpose: age-check @@ -244,7 +246,7 @@ matching the purpose declared here. Open `derivations/adult-status.rhai` in your editor and add the answer logic. Rhai is the bounded scripting language Evidence Gateway uses for requirement-specific derivations: -```rhai +```rhai title="derivations/adult-status.rhai" test-file fn answer(facts, selectors, context) { let born = parse_date(required(facts.date_of_birth, "date_of_birth_missing")); let adult_on = add_calendar_years(born, 18); @@ -275,7 +277,7 @@ mkdir -p mocks/cases Open `mocks/source.yaml` and add the three request cases: -```yaml +```yaml title="mocks/source.yaml" test-file version: 1 openapi: ../source.openapi.yaml operations: @@ -304,7 +306,7 @@ Add the exact synthetic bodies. ### `mocks/cases/person-123.json` -```json +```json title="mocks/cases/person-123.json" test-file { "person_id": "person-123", "name": "Amina Example", @@ -314,7 +316,7 @@ Add the exact synthetic bodies. ### `mocks/cases/person-456.json` -```json +```json title="mocks/cases/person-456.json" test-file { "person_id": "person-456", "name": "Mateo Example", @@ -324,7 +326,7 @@ Add the exact synthetic bodies. ### `mocks/cases/person-789.json` -```json +```json title="mocks/cases/person-789.json" test-file { "person_id": "person-789", "name": "Noor Example", @@ -338,7 +340,7 @@ Check the plan and every authored body offline: evidencectl source mock check --config mocks/source.yaml ``` -```text +```text test-expect Mock plan valid: operations=1 cases=3 ``` @@ -350,11 +352,11 @@ checked bytes. Start the materialized source and leave it running: -```sh +```sh test-background="http://127.0.0.1:4010/people/person-123" evidencectl source mock serve --config mocks/source.yaml ``` -```text +```text test-excerpt Source mock ready: mode=materialized origin=http://127.0.0.1:4010 served=3 skipped=0 ``` @@ -364,7 +366,7 @@ Start Evidence Gateway and the pinned local issuer: evidencectl dev start . ``` -```text +```text test-excerpt Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -391,7 +393,7 @@ before a response exists. `evidencectl` separately asks the local issuer for sho authorization. It sends no HTTP request to Evidence Gateway and does not contact the registry. The command creates exactly these owner-only artifacts: -```text +```text test-expect Prepared request: .evidence/requests/first-assertion/request.json Prepared verification context: .evidence/requests/first-assertion/verification.json Prepared authorization: .evidence/requests/first-assertion/authorization.curl @@ -417,7 +419,7 @@ curl --silent --show-error --fail-with-body \ --write-out 'HTTP %{http_code}\n' ``` -```text +```text test-expect HTTP 200 ``` @@ -438,7 +440,7 @@ evidencectl verify assertion.jws.json \ --output verified.json ``` -```text +```text test-expect VERIFIED ``` @@ -451,13 +453,13 @@ python3 -m json.tool verified.json The verified document contains generated identifiers, timestamps, and a pseudonymous subject binding. The fields relevant to this question look like this excerpt: -```json +```json test-excerpt { "assuranceProfile": "local", "purpose": "age-check", "subjects": [ { - "binding": "urn:evidence:subject:v1_…", + "binding": "urn:evidence:subject:v1_", "role": "person" } ], @@ -496,7 +498,7 @@ evidencectl request prepare adult-status \ --name first-vc ``` -```text +```text test-expect Prepared request: .evidence/requests/first-vc/request.json Prepared verification context: .evidence/requests/first-vc/verification.json Prepared authorization: .evidence/requests/first-vc/authorization.curl @@ -516,7 +518,7 @@ curl --silent --show-error --fail-with-body \ --write-out 'HTTP %{http_code}\n' ``` -```text +```text test-expect HTTP 200 ``` @@ -528,7 +530,7 @@ evidencectl verify assertion.sd-jwt \ --output verified-vc.json ``` -```text +```text test-expect VERIFIED ``` @@ -553,7 +555,7 @@ Stop Evidence Gateway and the local issuer before reading the completed audit ch evidencectl dev stop ``` -```text +```text test-expect Local Evidence stopped ``` @@ -565,7 +567,7 @@ Verify the completed audit chain and show its last operation: evidencectl audit show --last-operation ``` -```text +```text test-expect ACCESS AUTHORIZED adult-status age-check requester= DISCLOSURE RELEASED is_adult ``` @@ -578,7 +580,7 @@ record or access token. Choose two different unused ports when you start the local services: -```sh +```sh test-skip="the alternative for busy ports; the replay uses the default ports" evidencectl dev --evidence-port 8180 --issuer-port 8181 start . ``` @@ -603,7 +605,7 @@ Remove the stopped local generation, including the sealed bundle that Evidence G evidencectl dev clean ``` -```text +```text test-expect Removed stopped local Evidence state ``` diff --git a/docs/site/src/content/docs/tutorials/issue-fhir-evidence-as-vcs.mdx b/docs/site/src/content/docs/tutorials/issue-fhir-evidence-as-vcs.mdx index e27a31d7b..5d617652f 100644 --- a/docs/site/src/content/docs/tutorials/issue-fhir-evidence-as-vcs.mdx +++ b/docs/site/src/content/docs/tutorials/issue-fhir-evidence-as-vcs.mdx @@ -13,6 +13,8 @@ locale: en standards_referenced: - fhir-r4 - sd-jwt-vc +tutorial_test: + toolset: evidence --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -73,7 +75,7 @@ record whose beneficiary resolves to a Patient that exists, and one active Organ healthcare provider. It writes only their selectors to two owner-only local files, and prints neither identifiers nor FHIR resources: -```python +```python title="discover-fhir-records.py" test-file import json import os import re @@ -244,7 +246,7 @@ print("Organization selector file: ready") python3 discover-fhir-records.py ``` -```text +```text test-expect Coverage selector file: ready Organization selector file: ready ``` @@ -264,7 +266,7 @@ selectors never reach a log file. Create `fhir-read-through.py`: -```python +```python title="fhir-read-through.py" test-file import json import os import re @@ -383,7 +385,7 @@ done curl --silent --show-error --fail http://127.0.0.1:8000/healthz ``` -```text +```text test-expect ready ``` @@ -402,7 +404,7 @@ origin, paths, response media type, projected fields, and bounds are all stated Save this reviewed subset as `fhir-smart-r4.openapi.yaml`: -```yaml +```yaml title="fhir-smart-r4.openapi.yaml" test-file openapi: 3.1.0 info: title: SMART Health IT public FHIR R4 tutorial subset @@ -488,7 +490,7 @@ evidencectl init fhir-record-evidence \ cd fhir-record-evidence ``` -```text +```text test-excerpt Created an editable OpenAPI authoring project in fhir-record-evidence ``` @@ -500,7 +502,7 @@ authoring only; a deployment must use a reviewed authenticated HTTPS source. Create `questions/fhir-coverage-status.yaml`: -```yaml +```yaml title="questions/fhir-coverage-status.yaml" test-file id: fhir-coverage-status question: Does this coverage record report active coverage for the selected patient? purpose: coverage-record-verification @@ -534,7 +536,7 @@ Patient resource. Create `derivations/fhir-coverage-status.rhai`: -```rhai +```rhai title="derivations/fhir-coverage-status.rhai" test-file fn answer(facts, selectors, context) { let coverage_id = selectors["coverage-record"]["values"]["coverage_id"]; let patient_id = selectors["patient"]["values"]["patient_id"]; @@ -560,7 +562,7 @@ the payor, subscriber identifier, coverage class, period, and every other source Create `questions/fhir-healthcare-establishment.yaml`: -```yaml +```yaml title="questions/fhir-healthcare-establishment.yaml" test-file id: fhir-healthcare-establishment question: Does this organization record report an active healthcare provider? purpose: healthcare-establishment-verification @@ -589,7 +591,7 @@ disclosure: Create `derivations/fhir-healthcare-establishment.rhai`: -```rhai +```rhai title="derivations/fhir-healthcare-establishment.rhai" test-file fn answer(facts, selectors, context) { let organization_id = selectors["organization"]["values"]["organization_id"]; if required(facts.resource_id, "organization_id_missing") != organization_id { @@ -621,7 +623,7 @@ Compile the two questions and start Evidence Gateway with its stock identity pro evidencectl dev start . ``` -```text +```text test-excerpt Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -644,7 +646,7 @@ evidencectl request prepare fhir-coverage-status \ --name fhir-coverage-vc ``` -```text +```text test-expect Prepared request: .evidence/requests/fhir-coverage-vc/request.json Prepared verification context: .evidence/requests/fhir-coverage-vc/verification.json Prepared authorization: .evidence/requests/fhir-coverage-vc/authorization.curl @@ -672,7 +674,7 @@ evidencectl verify fhir-coverage.sd-jwt \ --output fhir-coverage.verified.json ``` -```text +```text test-expect HTTP 200 VERIFIED ``` @@ -685,7 +687,7 @@ python3 -m json.tool fhir-coverage.verified.json The verified payload carries one supported value: -```json +```json test-excerpt { "providesValueFor": "urn:registrystack:evidence:local:concept:fhir-coverage-status:coverage_record_reports_active", "value": true @@ -722,8 +724,10 @@ evidencectl verify fhir-healthcare-establishment.sd-jwt \ --output fhir-healthcare-establishment.verified.json ``` -```text +```text test-expect Prepared request: .evidence/requests/fhir-healthcare-establishment-vc/request.json +Prepared verification context: .evidence/requests/fhir-healthcare-establishment-vc/verification.json +Prepared authorization: .evidence/requests/fhir-healthcare-establishment-vc/authorization.curl HTTP 200 VERIFIED ``` @@ -751,6 +755,13 @@ rm -f \ fhir-organization-subjects.json ``` +```text test-expect +Local Evidence stopped +ACCESS AUTHORIZED fhir-healthcare-establishment healthcare-establishment-verification requester= +DISCLOSURE RELEASED healthcare_provider_record_active +Removed stopped local Evidence state +``` + The audit names the authorized question, the purpose it was authorized under, the requester pseudonym, and the concept that was released. Read it for what it leaves out: no selected FHIR identifier, no source resource, and not even the boolean value itself. An operator diff --git a/docs/site/src/content/docs/tutorials/refuse-unsafe-evidence-requests.mdx b/docs/site/src/content/docs/tutorials/refuse-unsafe-evidence-requests.mdx index 42d236621..f6054387d 100644 --- a/docs/site/src/content/docs/tutorials/refuse-unsafe-evidence-requests.mdx +++ b/docs/site/src/content/docs/tutorials/refuse-unsafe-evidence-requests.mdx @@ -12,6 +12,9 @@ persona: - consumer or verifier locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + after: tutorials/return-a-governed-value --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -42,18 +45,18 @@ you completed that tutorial. Without it, preparation stops with In one terminal, return to the `first-evidence-assertion` directory and serve the checked cases: -```sh +```sh test-cwd="first-evidence-assertion" test-background="http://127.0.0.1:4010/people/person-123" evidencectl source mock serve --config adult-status/mocks/source.yaml ``` In another terminal, enter the existing project and start a fresh local generation: -```sh +```sh test-cwd="first-evidence-assertion" cd adult-status evidencectl dev start . ``` -```text +```text test-excerpt Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -106,7 +109,7 @@ curl --silent --show-error \ --write-out 'HTTP %{http_code}\n' ``` -```text +```text test-expect HTTP 403 ``` @@ -116,7 +119,15 @@ Inspect the public problem: python3 -m json.tool unauthorized-response.json ``` -The response identifies only the closed `evidence.denied` problem and public trace ID. It +The response identifies only the closed `evidence.denied` problem and public trace ID: + +```json test-excerpt +{ + "code": "evidence.denied" +} +``` + +It does not reveal source facts, selector values, grants, or credentials. The registry terminal shows no new `GET /people/person-123` because authorization failed before source access. @@ -143,7 +154,7 @@ evidencectl verify authorized-response.jws.json \ --output authorized-response.verified.json ``` -```text +```text test-expect VERIFIED ``` @@ -169,13 +180,13 @@ PY Try to verify the changed response: -```sh +```sh test-exit="1" evidencectl verify tampered-response.jws.json \ --context .evidence/requests/refusal-check/verification.json \ --output tampered-response.verified.json ``` -```text +```text test-expect evidencectl: Evidence response verification failed ``` @@ -193,7 +204,7 @@ evidencectl dev stop evidencectl dev clean ``` -```text +```text test-expect Local Evidence stopped Removed stopped local Evidence state ``` diff --git a/docs/site/src/content/docs/tutorials/request-evidence-as-sd-jwt-vc.mdx b/docs/site/src/content/docs/tutorials/request-evidence-as-sd-jwt-vc.mdx index c78ba04fb..109ad6f60 100644 --- a/docs/site/src/content/docs/tutorials/request-evidence-as-sd-jwt-vc.mdx +++ b/docs/site/src/content/docs/tutorials/request-evidence-as-sd-jwt-vc.mdx @@ -13,6 +13,9 @@ persona: locale: en standards_referenced: - sd-jwt-vc +tutorial_test: + toolset: evidence + after: tutorials/first-evidence-assertion --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -53,13 +56,13 @@ concept URI. Omitting `responseFormats` keeps the project at signed JWS only. In one terminal, return to the `first-evidence-assertion` directory and serve the checked cases again: -```sh +```sh test-cwd="first-evidence-assertion" test-background="http://127.0.0.1:4010/people/person-123" evidencectl source mock serve --config adult-status/mocks/source.yaml ``` Leave it running. In another terminal, enter the existing Evidence Gateway project: -```sh +```sh test-cwd="first-evidence-assertion" cd adult-status ``` @@ -80,7 +83,7 @@ Start a fresh local generation: evidencectl dev start . ``` -```text +```text test-excerpt Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -110,7 +113,7 @@ curl --silent --show-error --fail-with-body \ --write-out 'HTTP %{http_code}\n' ``` -```text +```text test-expect HTTP 200 ``` @@ -123,7 +126,7 @@ evidencectl verify scalar.sd-jwt \ --output scalar.verified.json ``` -```text +```text test-expect VERIFIED ``` @@ -156,7 +159,7 @@ for disclosure in (part for part in parts[1:] if part): PY ``` -```text +```text test-expect typ: dc+sd-jwt vct: urn:registrystack:evidence:local:evidence-type:adult-status disclosure: urn:registrystack:evidence:local:concept:adult-status:is_adult @@ -200,13 +203,13 @@ PY Verification must fail and must not create trusted output: -```sh +```sh test-exit="1" evidencectl verify scalar-tampered.sd-jwt \ --context .evidence/requests/scalar-vc/verification.json \ --output scalar-tampered.verified.json ``` -```text +```text test-expect evidencectl: Evidence response verification failed ``` @@ -221,7 +224,7 @@ evaluated against. This is a second governed question, not a request-time option Create `schemas/adult-assessment.yaml`: -```yaml +```yaml title="schemas/adult-assessment.yaml" test-file $schema: https://json-schema.org/draft/2020-12/schema $id: urn:registrystack:evidence:local:schema:adult-assessment:v1 type: object @@ -237,7 +240,7 @@ properties: Create `questions/adult-assessment.yaml`: -```yaml +```yaml title="questions/adult-assessment.yaml" test-file id: adult-assessment question: What adult assessment applies to this person? purpose: age-assessment-review @@ -271,7 +274,7 @@ object would remain one atomic direct-field disclosure. Create `derivations/adult-assessment.rhai`: -```rhai +```rhai title="derivations/adult-assessment.rhai" test-file fn answer(facts, selectors, context) { let born = parse_date(required(facts.date_of_birth, "date_of_birth_missing")); let adult_on = add_calendar_years(born, 18); @@ -326,7 +329,7 @@ evidencectl verify structured.sd-jwt \ --output structured.verified.json ``` -```text +```text test-expect VERIFIED ``` @@ -349,7 +352,7 @@ for name in sorted(names): PY ``` -```text +```text test-expect disclosure: criterion disclosure: isAdult ``` @@ -369,7 +372,7 @@ evidencectl audit show --last-operation evidencectl dev clean ``` -```text +```text test-expect Local Evidence stopped ACCESS AUTHORIZED adult-assessment age-assessment-review requester= DISCLOSURE RELEASED adult_assessment diff --git a/docs/site/src/content/docs/tutorials/request-evidence-from-an-application.mdx b/docs/site/src/content/docs/tutorials/request-evidence-from-an-application.mdx index 5d884e876..5c7975d4b 100644 --- a/docs/site/src/content/docs/tutorials/request-evidence-from-an-application.mdx +++ b/docs/site/src/content/docs/tutorials/request-evidence-from-an-application.mdx @@ -11,6 +11,9 @@ persona: - consumer or verifier locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + after: tutorials/first-evidence-assertion --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -62,7 +65,7 @@ library refuses every response that does not match it. Enter the existing project: -```sh +```sh test-cwd="first-evidence-assertion" cd adult-status ``` @@ -76,7 +79,7 @@ evidencectl access client add age-check-app \ --generate-local-key ``` -```text +```text test-expect Added access policy app-age-checks for adult-status. Added client age-check-app with policy app-age-checks. ``` @@ -96,7 +99,7 @@ state for itself: grep evidenceAudience access/clients/age-check-app.yaml ``` -```text +```text test-expect evidenceAudience: urn:registrystack:evidence:local:client:age-check-app ``` @@ -114,7 +117,7 @@ from the project's own retained public signing key: evidencectl jwks --output trusted-issuer-keys.json secrets/signing-p256-public.jwk.json ``` -```text +```text test-expect wrote trusted-issuer-keys.json ``` @@ -130,7 +133,7 @@ The client is the `evidence` namespace of the maintained `registry-stack-client` version from the runtime you installed rather than typing one, and install the client at that exact version. From inside the project directory: -```sh +```sh test-skip="the gate provides the client package built from this checkout, which is not published yet" version="$(evidencectl --version | awk '{print $2}')" python3 -m venv .venv . .venv/bin/activate @@ -157,11 +160,11 @@ Evidence reads the source record through the materialized source mock from the f cleanup told you to stop it. In another terminal, return to the `first-evidence-assertion` directory and serve the same checked cases again. Leave it running: -```sh +```sh test-cwd="first-evidence-assertion" test-background="http://127.0.0.1:4010/people/person-123" evidencectl source mock serve --config adult-status/mocks/source.yaml ``` -```text +```text test-excerpt Source mock ready: mode=materialized origin=http://127.0.0.1:4010 served=3 skipped=0 ``` @@ -172,7 +175,7 @@ start Evidence and the pinned local issuer: evidencectl dev start . ``` -```text +```text test-excerpt Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -183,14 +186,14 @@ later, as a failed evidence request. The access policy is now part of the running generation, so a terminal request names a client too. Confirm that the project no longer accepts an unnamed one: -```sh +```sh test-exit="1" evidencectl request prepare adult-status \ --purpose age-check \ --subject person_id=person-123 \ --name unnamed-caller ``` -```text +```text test-expect evidencectl: the active project requires a registered client selected with --client ``` @@ -231,25 +234,33 @@ print(document) PY ``` -```json +```json test-expect { "assuranceProfile": "local", + "audience": "urn:registrystack:evidence:local:client:age-check-app", "definitions": [ { "concepts": [ { + "concept": "urn:registrystack:evidence:local:concept:adult-status:is_adult", "form": "boolean", - "id": "urn:registrystack:evidence:local:concept:adult-status:is_adult" + "handle": "is_adult", + "required": true } ], "configurationRevision": "sha256:", "evidenceType": "urn:registrystack:evidence:local:evidence-type:adult-status", + "handle": "adult-status", "kind": "criterion", "purpose": "age-check", "referenceFrameworks": [ "urn:registrystack:evidence:local:framework:adult-status" ], "requirement": "urn:registrystack:evidence:local:requirement:adult-status", + "responseFormats": [ + "signed-jws", + "sd-jwt-vc" + ], "subjects": [ { "cardinality": "one", @@ -377,7 +388,7 @@ print(document) PY ``` -```json +```json test-expect { "audience": "urn:registrystack:evidence:local:client:age-check-app", "clock_skew_seconds": 30, @@ -454,7 +465,7 @@ must be asked, moved. Open `age_check.py` in your editor and add the application. It loads the pinned procedure and never calls discovery again: -```python +```python title="age_check.py" test-file import fcntl import json import os @@ -591,7 +602,7 @@ umask 077 python3 age_check.py ``` -```text +```text test-expect person-123 is_adult=True pinned binding recorded in subject-bindings.json ``` @@ -603,7 +614,7 @@ again: python3 age_check.py ``` -```text +```text test-expect person-123 is_adult=True pinned binding recorded in subject-bindings.json ``` @@ -628,7 +639,7 @@ Ask about a different record: python3 age_check.py person-456 ``` -```text +```text test-expect person-456 is_adult=False pinned binding recorded in subject-bindings.json ``` @@ -640,7 +651,7 @@ The registry holds a name and a date of birth for both people. Neither answer co Change one stored binding to prove that the application, not the deployment, decides what it accepts: -```sh +```sh test-exit="1" python3 - <<'PY' import json from pathlib import Path @@ -652,7 +663,7 @@ PY python3 age_check.py person-123 ``` -```text +```text test-expect unverifiable response, nothing read (policy): the Evidence response failed verification: Evidence payload does not match the relying procedure ``` @@ -695,7 +706,7 @@ evidencectl dev stop evidencectl dev clean ``` -```text +```text test-expect Local Evidence stopped Removed stopped local Evidence state ``` diff --git a/docs/site/src/content/docs/tutorials/return-a-governed-value.mdx b/docs/site/src/content/docs/tutorials/return-a-governed-value.mdx index c4a55820e..6d0c1a90a 100644 --- a/docs/site/src/content/docs/tutorials/return-a-governed-value.mdx +++ b/docs/site/src/content/docs/tutorials/return-a-governed-value.mdx @@ -11,6 +11,9 @@ persona: - assertion provider locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + after: tutorials/first-evidence-assertion --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -52,7 +55,7 @@ age. In one terminal, return to the `first-evidence-assertion` directory and serve the same checked cases again: -```sh +```sh test-cwd="first-evidence-assertion" test-background="http://127.0.0.1:4010/people/person-123" evidencectl source mock serve --config adult-status/mocks/source.yaml ``` @@ -64,7 +67,7 @@ registry lookup from the first tutorial. In another terminal, enter the existing project: -```sh +```sh test-cwd="first-evidence-assertion" cd adult-status ``` @@ -72,7 +75,7 @@ cd adult-status Create a second question definition alongside `questions/adult-status.yaml`: -```yaml +```yaml title="questions/age-bracket.yaml" test-file id: age-bracket question: Which age bracket does this person belong to? purpose: service-path-selection @@ -103,7 +106,7 @@ codelist enforced by the Evidence Gateway runtime. Create this new derivation alongside `derivations/adult-status.rhai`: -```rhai +```rhai title="derivations/age-bracket.rhai" test-file fn answer(facts, selectors, context) { let born = parse_date(required(facts.date_of_birth, "date_of_birth_missing")); if compare_dates(context.legal_local_date, add_calendar_years(born, 18)) < 0 { @@ -129,7 +132,7 @@ Capture the edited project in a new immutable local generation: evidencectl dev start . ``` -```text +```text test-excerpt Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -170,7 +173,7 @@ evidencectl verify age-bracket.jws.json \ --output age-bracket.verified.json ``` -```text +```text test-expect VERIFIED ``` @@ -182,7 +185,7 @@ python3 -m json.tool age-bracket.verified.json The relevant supported value has this shape: -```json +```json test-excerpt { "providesValueFor": "urn:registrystack:evidence:local:concept:age-bracket:age_bracket", "value": "under-18" @@ -210,7 +213,7 @@ Inspect the last verified operation: evidencectl audit show --last-operation ``` -```text +```text test-expect ACCESS AUTHORIZED age-bracket service-path-selection requester= DISCLOSURE RELEASED age_bracket ``` @@ -232,7 +235,7 @@ Return to the registry terminal and press `Ctrl+C`. the first tutorial's services were left running. Stop and remove that session, then start the new generation: -```sh +```sh test-skip="recovers a session the first tutorial left running; the replay stopped it" evidencectl dev stop evidencectl dev clean ``` diff --git a/docs/site/src/content/docs/tutorials/run-oid4vci-interoperability-checks.mdx b/docs/site/src/content/docs/tutorials/run-oid4vci-interoperability-checks.mdx index 5c043d29e..a2d64cf50 100644 --- a/docs/site/src/content/docs/tutorials/run-oid4vci-interoperability-checks.mdx +++ b/docs/site/src/content/docs/tutorials/run-oid4vci-interoperability-checks.mdx @@ -13,6 +13,9 @@ locale: en standards_referenced: - oid4vci - sd-jwt-vc +tutorial_test: + toolset: evidence + checkout: true --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -55,7 +58,7 @@ mkdir -p .tutorial/oid4vci-adopter Copy this complete loopback configuration exactly: -```yaml +```yaml title=".tutorial/oid4vci-adopter/oid4vci.yaml" test-file version: 1 validationMode: supervised-local-development credentialIssuer: http://127.0.0.1:18440 @@ -100,7 +103,7 @@ EVIDENCE_OID4VCI_ADOPTER_ROOT="$PWD/.tutorial/oid4vci-adopter" \ The successful run ends with this exact receipt: -```text +```text test-excerpt PASS: sanitized Inji OID4VCI profile and Registry-side interoperability tests ``` @@ -121,12 +124,12 @@ Before those compatibility cases, the runner invokes the actual `evidence-oid4vc listener, completes an authorized wallet flow through the published metadata, and verifies the returned holder-bound presentation independently. Its safe milestones include: -```text -CONFIG COPIED: complete configuration has no untracked inputs +```text test-excerpt CONFIG CHECKED: complete delivery configuration is valid METADATA INSPECTED: derived holder-bound batch ceiling is 4 SERVICE READY: health and readiness are available on the delivery listener METRICS PRIVATE: metrics exist only on the separate loopback listener +TASK GRANT REFUSED: deferred wallet state was not created PRESENTATION VERIFIED: public wallet flow returned holder-bound Evidence CLEANUP COMPLETE: generated private material was removed ``` @@ -168,7 +171,7 @@ Simulator. Before running it, install Java 17, Android SDK platform 34 with Buil full Xcode, and an iPhone 15 simulator. The runner requires `git`, `npm`, `java`, `xcodebuild`, and `xcrun` on `PATH`, plus network access. -```sh +```sh test-skip="the upstream runner needs macOS with Xcode and the iOS Simulator, Java, the Android SDK, and network access" EVIDENCE_INJI_OID4VCI=1 products/evidence/scripts/compat/inji-oid4vci-upstream.sh ``` diff --git a/docs/site/src/content/docs/tutorials/verify-an-assertion-as-a-consumer.mdx b/docs/site/src/content/docs/tutorials/verify-an-assertion-as-a-consumer.mdx index 993eb2a88..f28e2b611 100644 --- a/docs/site/src/content/docs/tutorials/verify-an-assertion-as-a-consumer.mdx +++ b/docs/site/src/content/docs/tutorials/verify-an-assertion-as-a-consumer.mdx @@ -11,6 +11,9 @@ persona: - consumer or verifier locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + after: tutorials/first-evidence-assertion --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -35,7 +38,7 @@ trust and request expectations support later offline review. Enter the existing project. No service or registry needs to be running: -```sh +```sh test-cwd="first-evidence-assertion" cd adult-status ``` @@ -88,15 +91,24 @@ evidence verify \ The result begins: -```text +```text test-excerpt verified-at: authentic: yes currently-valid: yes ``` -The command then prints the verified Evidence Gateway payload. It opens no listener, contacts no issuer, -fetches no discovery document, and calls no source. The named JWKS file is the complete trust set -for this verification. +The command then prints the verified Evidence Gateway payload, which carries the same answer the first +tutorial verified: + +```json test-excerpt +{ + "providesValueFor": "urn:registrystack:evidence:local:concept:adult-status:is_adult", + "value": true +} +``` + +It opens no listener, contacts no issuer, fetches no discovery document, and calls no source. The +named JWKS file is the complete trust set for this verification. `authentic` and `currently-valid` answer different questions. An expired stored response can remain authentic evidence of what was signed and accepted at an earlier decision time. It must not be used From 99c7cfb701838a55c70906fe1fc823a2495a5832 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Sun, 27 Sep 2026 04:17:35 +0000 Subject: [PATCH 5/7] fix(docs): treat a background fence as ready only on a 2xx from its own live command Signed-off-by: Jeremi Joslin --- .../scripts/tutorial-runner/background.mjs | 20 ++++++----- .../tutorial-runner/run-tutorial.test.mjs | 34 +++++++++++++++++++ 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/docs/site/scripts/tutorial-runner/background.mjs b/docs/site/scripts/tutorial-runner/background.mjs index a75a7b567..57161da3d 100644 --- a/docs/site/scripts/tutorial-runner/background.mjs +++ b/docs/site/scripts/tutorial-runner/background.mjs @@ -3,9 +3,10 @@ // // The journey script starts a background fence in its own process group and // records ` ` in a state file. This helper then waits for -// the fence's ready URL, or stops the group the state file names: +// the fence's ready URL to answer with success, or stops the group the state +// file names: // -// background.mjs ready wait until the URL answers +// background.mjs ready wait until the URL answers 2xx // background.mjs stop stop the group, show its output // // A fence whose group ends before its URL answers, or that does not answer @@ -61,16 +62,19 @@ async function ready(url, stateFile) { const { group } = await readState(stateFile); const deadline = Date.now() + READY_TIMEOUT_MS; for (;;) { - try { - await fetch(url, { signal: AbortSignal.timeout(2000) }); - return 0; - } catch { - // Not answering yet: the command may still be starting. - } + // The group is checked first, so a service that already held the port + // cannot answer for a command that has ended. if (!groupAlive(group)) { console.error(`the command ended before ${url} answered`); return 1; } + try { + const response = await fetch(url, { signal: AbortSignal.timeout(2000) }); + await response.body?.cancel(); + if (response.ok && groupAlive(group)) return 0; + } catch { + // Not answering yet: the command may still be starting. + } if (Date.now() > deadline) { console.error(`${url} did not answer within ${READY_TIMEOUT_MS / 1000} seconds`); return 1; diff --git a/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs b/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs index 3fa78cb38..214558067 100644 --- a/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs +++ b/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import { execFile, spawn } from 'node:child_process'; import { existsSync } from 'node:fs'; import { chmod, mkdir, mkdtemp, readdir, readFile, realpath, rm, writeFile } from 'node:fs/promises'; +import { createServer as createHttpServer } from 'node:http'; import { createServer } from 'node:net'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; @@ -556,6 +557,39 @@ test('a background fence that ends before it is ready stops the journey and show }); }); +test('a background fence is ready only once its URL answers with success', async () => { + const port = await freePort(); + const url = `http://127.0.0.1:${port}/ready.txt`; + const body = + '## Serve\n\n' + + fence(`sh test-background="${url}"`, `(sleep 1; echo ok >ready.txt) &\nexec python3 -m http.server ${port} --bind 127.0.0.1 2>/dev/null`) + + fence('sh', 'cat ready.txt') + + fence('text test-expect', 'ok'); + await withPage(body, async ({ page }) => { + const { code, output } = await run([page]); + assert.equal(code, 0, output); + assert.match(output, /tutorial PASS/u); + }); +}); + +test('a background fence that ends is not ready although another service answers its URL', async () => { + const port = await freePort(); + const stale = createHttpServer((request, response) => response.end('stale')); + await new Promise((resolvePromise) => stale.listen(port, '127.0.0.1', resolvePromise)); + try { + const body = '## Serve\n\n' + fence(`sh test-background="http://127.0.0.1:${port}/"`, 'echo "port taken" >&2\nfalse') + fence('sh', 'echo never'); + await withPage(body, async ({ page }) => { + const { code, output } = await run([page]); + assert.equal(code, 1, output); + assert.match(output, /port taken/u); + assert.match(output, /the sh fence at line 7 \(Serve\) failed/u); + assert.doesNotMatch(output, /^never$/mu); + }); + } finally { + await new Promise((resolvePromise) => stale.close(resolvePromise)); + } +}); + test('an expectation on a background fence checks what it printed while it ran', async () => { const port = await freePort(); const url = `http://127.0.0.1:${port}/`; From 650f5d1a924d95e1b658bb03e2e5d2007d5c7726 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Sun, 27 Sep 2026 04:55:32 +0000 Subject: [PATCH 6/7] fix(docs): stop a path that shares a tool's name from routing a page into its gate Signed-off-by: Jeremi Joslin --- .../site/scripts/tutorial-runner/toolsets.mjs | 9 +++--- .../scripts/tutorial-runner/toolsets.test.mjs | 29 +++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 docs/site/scripts/tutorial-runner/toolsets.test.mjs diff --git a/docs/site/scripts/tutorial-runner/toolsets.mjs b/docs/site/scripts/tutorial-runner/toolsets.mjs index 76f929314..c77381fdd 100644 --- a/docs/site/scripts/tutorial-runner/toolsets.mjs +++ b/docs/site/scripts/tutorial-runner/toolsets.mjs @@ -110,8 +110,9 @@ function productToolset({ const breg = productToolset({ label: 'breg and bregctl', // A page whose sh fences match this runs the toolset, so its gate must - // cover it (tutorial-runner/gate.mjs). - commands: /(^|[^\w-])(bregctl|breg)([^\w-]|$)/mu, + // cover it (tutorial-runner/gate.mjs). A path such as .breg/dev or + // tutorial-work/breg names no command. + commands: /(^|[^\w./-])(bregctl|breg)([^\w./-]|$)/mu, binaries: [ ['breg', 'BREG_BIN'], ['bregctl', 'BREGCTL_BIN'], @@ -128,7 +129,7 @@ const breg = productToolset({ // so they no longer reconcile against a registry that is stopping. const casework = productToolset({ label: 'casework, caseworkctl, breg, and bregctl', - commands: /(^|[^\w-])(caseworkctl|casework)([^\w-]|$)/mu, + commands: /(^|[^\w./-])(caseworkctl|casework)([^\w./-]|$)/mu, includes: ['breg'], binaries: [ ['casework', 'CASEWORK_BIN'], @@ -208,7 +209,7 @@ const evidenceProduct = productToolset({ label: 'evidence, evidencectl, and evidence-oid4vci', // Not a path such as .evidence/dev, which a page may name; but a page that // runs Evidence's own checks from a checkout runs Evidence. - commands: /(^|[^\w./-])(evidencectl|evidence-oid4vci|evidence)([^\w-]|$)|(^|\s)products\/evidence\/scripts\//mu, + commands: /(^|[^\w./-])(evidencectl|evidence-oid4vci|evidence)([^\w./-]|$)|(^|\s)products\/evidence\/scripts\//mu, binaries: [ ['evidence', 'EVIDENCE_BIN'], ['evidencectl', 'EVIDENCECTL_BIN'], diff --git a/docs/site/scripts/tutorial-runner/toolsets.test.mjs b/docs/site/scripts/tutorial-runner/toolsets.test.mjs new file mode 100644 index 000000000..7a3eeb1c9 --- /dev/null +++ b/docs/site/scripts/tutorial-runner/toolsets.test.mjs @@ -0,0 +1,29 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { TOOLSETS } from './toolsets.mjs'; + +// A toolset's commands pattern decides which pages its gate must replay, so it +// matches a command a fence runs and not a path or file that shares its name. +const CASES = { + breg: { + runs: ['bregctl init .', 'bregctl dev start .', 'breg --version', 'version=$(bregctl --version)', 'bregctl dev stop .; echo done'], + names: ['cd tutorial-work/breg', 'cat .breg/dev/state.json', 'ls ./breg', 'cat breg.yaml', 'cd breg-demo', 'ls breg/'], + }, + casework: { + runs: ['caseworkctl dev start .', 'casework --version', 'caseworkctl check . | tail -1'], + names: ['cd tutorial-work/casework', 'cat .casework/dev/state.json', 'cat casework.yaml', 'ls casework/'], + }, + evidence: { + runs: ['evidencectl init .', 'evidence --version', 'evidence-oid4vci --help', 'products/evidence/scripts/check-contracts.sh'], + names: ['cd ~/work/evidence', 'ls .evidence/clients', 'cat evidence.yaml', 'ls evidence/'], + }, +}; + +for (const [name, { runs, names }] of Object.entries(CASES)) { + test(`the ${name} toolset matches the commands a fence runs, not paths that share their name`, () => { + const { commands } = TOOLSETS[name]; + for (const code of runs) assert.equal(commands.test(code), true, `${name} must match: ${code}`); + for (const code of names) assert.equal(commands.test(code), false, `${name} must not match: ${code}`); + }); +} From 1c378ddadc475b9e62a150ada101ccce40324b8d Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Sun, 27 Sep 2026 06:09:07 +0000 Subject: [PATCH 7/7] fix(docs): fail a tutorial when a background command exits before it is stopped A background fence promises to keep running until the next one starts or the page ends, but stop() only sent the group a signal and reported success even when the group had already ended on its own. A service that crashed after answering its readiness check was silently accepted. Check liveness before stopping and fail with what the command printed when it had already exited. Signed-off-by: Jeremi Joslin --- .../scripts/tutorial-runner/background.mjs | 15 ++++++++++++-- .../tutorial-runner/run-tutorial.test.mjs | 20 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/docs/site/scripts/tutorial-runner/background.mjs b/docs/site/scripts/tutorial-runner/background.mjs index 57161da3d..71bd5f9d7 100644 --- a/docs/site/scripts/tutorial-runner/background.mjs +++ b/docs/site/scripts/tutorial-runner/background.mjs @@ -10,7 +10,10 @@ // background.mjs stop stop the group, show its output // // A fence whose group ends before its URL answers, or that does not answer -// within READY_TIMEOUT_MS, fails the wait. +// within READY_TIMEOUT_MS, fails the wait. A group that has already ended by +// the time stop is called fails too: a background command promises to keep +// running until it is stopped, so one that exited on its own is a failure +// even though it answered while it was up. import { readFile, writeFile } from 'node:fs/promises'; import { setTimeout as sleep } from 'node:timers/promises'; @@ -86,11 +89,19 @@ async function ready(url, stateFile) { async function stop(stateFile) { const state = await readState(stateFile); if (!state) return 0; + // Checked before stopping it, so a group that already ended on its own is + // told apart from one this call is the one to stop. + const alreadyEnded = !groupAlive(state.group); await stopGroup(state.group); const output = await readFile(state.output, 'utf8'); + await writeFile(stateFile, ''); + if (alreadyEnded) { + console.error(`the background command had already exited, although it must keep running until it is stopped${output === '' ? '' : '; it printed:'}`); + process.stdout.write(output); + return 1; + } console.log(`\nstopped the background command${output === '' ? '' : ', which printed:'}`); process.stdout.write(output); - await writeFile(stateFile, ''); return 0; } diff --git a/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs b/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs index 214558067..ec5578737 100644 --- a/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs +++ b/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs @@ -605,6 +605,26 @@ test('an expectation on a background fence checks what it printed while it ran', }); }); +test('a background command that exits on its own before it is stopped fails the journey', async () => { + const port = await freePort(); + const url = `http://127.0.0.1:${port}/`; + const serve = `python3 -m http.server ${port} --bind 127.0.0.1`; + const body = + '## Serve\n\n' + + fence( + `sh test-background="${url}"`, + `${serve} &\nserver=$!\nsleep 1\nkill "$server"\nwait "$server" 2>/dev/null || true\necho "the flaky server has exited"`, + ) + + fence('sh', 'sleep 1.5\necho after'); + await withPage(body, async ({ page }) => { + const { code, output } = await run([page]); + assert.equal(code, 1, output); + assert.match(output, /the flaky server has exited/u, 'the background command must be named by what it printed'); + assert.match(output, /background command.*exited/isu); + assert.match(output, /tutorial FAIL/u); + }); +}); + test('a journey that ends, passing or failing, leaves no process it started running', async () => { const port = await freePort(); const body = (pids, last) =>