diff --git a/.github/scripts/ci_changes.py b/.github/scripts/ci_changes.py index 302df70101..110faf8562 100644 --- a/.github/scripts/ci_changes.py +++ b/.github/scripts/ci_changes.py @@ -178,46 +178,45 @@ "docs/site/src/content/docs/tutorials/publish-governed-sqlite-registry.mdx", ) -# Every input the Evidence tutorial gate replays or is built from. The tutorial -# pages and helper scripts here must stay in step with the gate's own registry -# and the helpers it invokes, which test_ci_changes.py enforces: a tutorial or -# helper CI does not watch is one that rots silently. -EVIDENCE_TUTORIAL_INPUTS = frozenset( - { - "Cargo.lock", - "Cargo.toml", - "docs/site/package-lock.json", - "docs/site/package.json", - "docs/site/scripts/check-evidence-tutorials.sh", - "docs/site/scripts/check-evidence-tutorials.test.mjs", - "docs/site/scripts/evidence-tutorial-fence.sh", - "docs/site/scripts/fixtures/fhir-tutorial-mock.py", - "docs/site/src/content/docs/tutorials/assert-a-role-bound-relationship.mdx", - "docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx", - "docs/site/src/content/docs/tutorials/control-who-can-request-evidence.mdx", - "docs/site/src/content/docs/tutorials/first-evidence-assertion.mdx", - "docs/site/src/content/docs/tutorials/issue-fhir-evidence-as-vcs.mdx", - "docs/site/src/content/docs/tutorials/refuse-unsafe-evidence-requests.mdx", - "docs/site/src/content/docs/tutorials/request-evidence-as-sd-jwt-vc.mdx", - "docs/site/src/content/docs/tutorials/request-evidence-from-an-application.mdx", - "docs/site/src/content/docs/tutorials/run-oid4vci-interoperability-checks.mdx", - "docs/site/src/content/docs/tutorials/return-a-governed-value.mdx", - "docs/site/src/content/docs/tutorials/verify-an-assertion-as-a-consumer.mdx", - "products/evidence/fixtures/interoperability/inji-oid4vci/profile.json", - "products/evidence/fixtures/interoperability/inji-oid4vci/receipt.json", - "products/evidence/scripts/compat/inji-oid4vci-upstream.sh", - "products/evidence/scripts/compat/inji-oid4vci.sh", - # The application tutorial imports the maintained client package, and - # the job assembles that package from this commit with these scripts - # and this pinned build tool. A change to any of them changes what the - # replay imports. - "release/requirements/maturin-1.9.6.txt", - "release/scripts/assemble-registry-client-packages.py", - "release/scripts/assemble-registry-client-wheel.py", - "release/scripts/build-linux-python-client", - "release/scripts/zig-glibc-compiler", - "release/scripts/smoke-registry-client-package.py", - } +# Every input the Evidence tutorial gate replays or is built from: the page +# runner, the pages whose frontmatter it replays, the source mock the toolset +# starts, and the build inputs of what the pages run. The replayed pages here +# must stay in step with their tutorial_test frontmatter, which +# test_ci_changes.py enforces: a tutorial CI does not watch is one that rots +# silently. +EVIDENCE_TUTORIAL_INPUTS = ( + "Cargo.lock", + "Cargo.toml", + "docs/site/package-lock.json", + "docs/site/package.json", + "docs/site/scripts/run-tutorial.mjs", + "docs/site/scripts/tutorial-runner/**", + "docs/site/scripts/fixtures/fhir-tutorial-mock.py", + "docs/site/src/content/docs/tutorials/assert-a-role-bound-relationship.mdx", + "docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx", + "docs/site/src/content/docs/tutorials/control-who-can-request-evidence.mdx", + "docs/site/src/content/docs/tutorials/first-evidence-assertion.mdx", + "docs/site/src/content/docs/tutorials/issue-fhir-evidence-as-vcs.mdx", + "docs/site/src/content/docs/tutorials/refuse-unsafe-evidence-requests.mdx", + "docs/site/src/content/docs/tutorials/request-evidence-as-sd-jwt-vc.mdx", + "docs/site/src/content/docs/tutorials/request-evidence-from-an-application.mdx", + "docs/site/src/content/docs/tutorials/run-oid4vci-interoperability-checks.mdx", + "docs/site/src/content/docs/tutorials/return-a-governed-value.mdx", + "docs/site/src/content/docs/tutorials/verify-an-assertion-as-a-consumer.mdx", + "products/evidence/fixtures/interoperability/inji-oid4vci/profile.json", + "products/evidence/fixtures/interoperability/inji-oid4vci/receipt.json", + "products/evidence/scripts/compat/inji-oid4vci-upstream.sh", + "products/evidence/scripts/compat/inji-oid4vci.sh", + # The application tutorial imports the maintained client package, and + # the job assembles that package from this commit with these scripts + # and this pinned build tool. A change to any of them changes what the + # replay imports. + "release/requirements/maturin-1.9.6.txt", + "release/scripts/assemble-registry-client-packages.py", + "release/scripts/assemble-registry-client-wheel.py", + "release/scripts/build-linux-python-client", + "release/scripts/zig-glibc-compiler", + "release/scripts/smoke-registry-client-package.py", ) # Every input the Base Registry Engine tutorial gate replays or is built from: @@ -1216,7 +1215,7 @@ def classify( evidence_tutorial = ( complete - or any(path in EVIDENCE_TUTORIAL_INPUTS for path in paths) + or any(matches(path, *EVIDENCE_TUTORIAL_INPUTS) for path in paths) or bool( affected & (EVIDENCE_TUTORIAL_PACKAGES | ASSEMBLED_PYTHON_CLIENT_PACKAGES) ) diff --git a/.github/scripts/test_ci_changes.py b/.github/scripts/test_ci_changes.py index ff6fb7cd4a..4551744a6c 100644 --- a/.github/scripts/test_ci_changes.py +++ b/.github/scripts/test_ci_changes.py @@ -829,52 +829,35 @@ def test_manifest_core_changes_select_breg_through_linked_code( self.assertIn("registry-breg", outputs["rust_packages"]) self.assertIn("registry-manifest-core", outputs["rust_packages"]) - def test_evidence_tutorial_inputs_cover_every_registered_tutorial(self) -> None: - # The gate's registry is the source of truth for which tutorials exist. - # A tutorial missing here would not trigger the job that replays it, so - # it could break without any pull request noticing. - gate = ( - Path(__file__).resolve().parents[2] - / "docs/site/scripts/check-evidence-tutorials.sh" - ) - registry = re.search( - r"^EVIDENCE_TUTORIALS=\((.*?)^\)", gate.read_text(), re.DOTALL | re.MULTILINE - ) - if registry is None: - self.fail("the gate must declare EVIDENCE_TUTORIALS") - slugs = registry.group(1).split() - self.assertTrue(slugs, "the gate must register at least one tutorial") + def test_evidence_tutorial_inputs_cover_every_replayed_tutorial(self) -> None: + # Each page's tutorial_test frontmatter is the source of truth for + # which tutorials the gate replays. A replayed page missing here would + # not trigger the job that replays it, so it could break without any + # pull request noticing. + docs = Path(__file__).resolve().parents[2] / "docs/site/src/content/docs" + slugs = [] + for section in ("start", "tutorials"): + for page in sorted((docs / section).glob("*.mdx")): + frontmatter = yaml.safe_load(page.read_text().split("---\n")[1]) + declaration = frontmatter.get("tutorial_test") or {} + if declaration.get("toolset") == "evidence" and "skip" not in declaration: + slugs.append(f"{section}/{page.stem}") + self.assertIn("tutorials/first-evidence-assertion", slugs) for slug in slugs: with self.subTest(slug=slug): - self.assertIn( - f"docs/site/src/content/docs/tutorials/{slug}.mdx", - EVIDENCE_TUTORIAL_INPUTS, + page = f"docs/site/src/content/docs/{slug}.mdx" + self.assertTrue( + any( + fnmatch.fnmatchcase(page, pattern) + for pattern in EVIDENCE_TUTORIAL_INPUTS + ) ) - def test_evidence_tutorial_inputs_cover_every_helper_the_gate_invokes(self) -> None: - # Same reasoning as the tutorial registry above, one layer down. The gate - # delegates to sibling scripts, and a change to one of those changes what - # every tutorial replay does. A helper missing here routes the change - # past the job that would have caught it. - gate = ( - Path(__file__).resolve().parents[2] - / "docs/site/scripts/check-evidence-tutorials.sh" - ) - helpers = set( - re.findall( - r"\$SITE_ROOT/scripts/([A-Za-z0-9._/-]+\.(?:mjs|py|sh))", - gate.read_text(), - ) - ) - self.assertTrue(helpers, "the gate must invoke at least one helper") - for helper in sorted(helpers): - with self.subTest(helper=helper): - self.assertIn(f"docs/site/scripts/{helper}", EVIDENCE_TUTORIAL_INPUTS) - def test_evidence_tutorial_routing(self) -> None: infrastructure = ( - "docs/site/scripts/check-evidence-tutorials.sh", - "docs/site/scripts/check-evidence-tutorials.test.mjs", + "docs/site/scripts/run-tutorial.mjs", + "docs/site/scripts/tutorial-runner/toolsets.mjs", + "docs/site/scripts/fixtures/fhir-tutorial-mock.py", "docs/site/src/content/docs/tutorials/first-evidence-assertion.mdx", "docs/site/package.json", ) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 960925c587..e27356e497 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1356,9 +1356,13 @@ jobs: cache-targets: false save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Test the tutorial gate helpers + - name: Install docs dependencies + working-directory: docs/site + run: npm ci + + - name: Test the tutorial runner working-directory: docs/site - run: npm run test:tutorial:evidence + run: npm run test:tutorial:runner - name: Check tutorial command drift working-directory: docs/site @@ -1393,8 +1397,6 @@ jobs: "${RUNNER_TEMP}/maturin/bin/pip" install --quiet \ --require-hashes --only-binary=:all: \ --requirement "${GITHUB_WORKSPACE}/release/requirements/maturin-1.9.6.txt" - # The output stays inside the workspace, because the container step - # below mounts the workspace and nothing else. out_dir="${GITHUB_WORKSPACE}/target/evidence-tutorial-client" python3 release/scripts/assemble-registry-client-packages.py \ --artifacts python \ @@ -1411,15 +1413,13 @@ jobs: exit 1 fi # Prove installer metadata and native facade loading from this exact - # wheel. The clean-container tutorials separately exercise requests. + # wheel. The application tutorial separately exercises requests. python3 -m venv "${RUNNER_TEMP}/client-install-smoke" "${RUNNER_TEMP}/client-install-smoke/bin/pip" install \ --no-index --no-deps "${wheel}" "${RUNNER_TEMP}/client-install-smoke/bin/python" -I \ release/scripts/smoke-registry-client-package.py - # The gate reads it inside the container, at the mounted path. - printf 'TUTORIAL_CLIENT_WHEEL=/work/%s\n' \ - "${wheel#"${GITHUB_WORKSPACE}/"}" >>"${GITHUB_ENV}" + printf 'REGISTRY_CLIENT_PY_WHEEL=%s\n' "${wheel}" >>"${GITHUB_ENV}" - name: Test the exact local Evidence lifecycle shell: bash @@ -1438,10 +1438,7 @@ jobs: EVIDENCECTL_BIN: ${{ github.workspace }}/target/debug/evidencectl EVIDENCE_OID4VCI_BIN: ${{ github.workspace }}/target/debug/evidence-oid4vci EVIDENCE_OID4VCI_INTEROP_TEST_BIN: ${{ github.workspace }}/target/debug/inji-oid4vci-interop-test - run: | - set -euo pipefail - REGISTRY_CLIENT_PY_WHEEL="${GITHUB_WORKSPACE}/${TUTORIAL_CLIENT_WHEEL#/work/}" \ - bash docs/site/scripts/check-evidence-tutorials.sh + run: node docs/site/scripts/run-tutorial.mjs --gate evidence breg-tutorial: name: Base Registry Engine tutorial from source diff --git a/docs/site/package.json b/docs/site/package.json index 7e5c2c3a47..94afdc3527 100644 --- a/docs/site/package.json +++ b/docs/site/package.json @@ -47,9 +47,8 @@ "check:tutorial:discovery:dry-run": "bash scripts/check-discovery-tutorial.sh --dry-run", "check:tutorial:relay": "bash scripts/check-relay-tutorial.sh", "check:tutorial:relay:dry-run": "bash scripts/check-relay-tutorial.sh --dry-run", - "test:tutorial:evidence": "node --test scripts/check-evidence-tutorials.test.mjs", - "check:tutorial:evidence": "bash scripts/check-evidence-tutorials.sh", - "check:tutorial:evidence:dry-run": "bash scripts/check-evidence-tutorials.sh --dry-run", + "check:tutorial:evidence": "node scripts/run-tutorial.mjs --gate evidence", + "check:tutorial:evidence:dry-run": "node scripts/run-tutorial.mjs --gate evidence --dry-run", "test:tutorial:runner": "node --test \"scripts/tutorial-runner/*.test.mjs\"", "check:tutorial:breg": "node scripts/run-tutorial.mjs --gate breg", "check:tutorial:breg:dry-run": "node scripts/run-tutorial.mjs --gate breg --dry-run", diff --git a/docs/site/scripts/check-evidence-tutorials.sh b/docs/site/scripts/check-evidence-tutorials.sh deleted file mode 100755 index f169fdd8b0..0000000000 --- a/docs/site/scripts/check-evidence-tutorials.sh +++ /dev/null @@ -1,1102 +0,0 @@ -#!/usr/bin/env bash -# -# Execute the current Evidence tutorials from a fresh reader directory. -# -# What this gate is for: proving that the commands the tutorials document still -# run, and that a short list of behaviours a successful exit does not already -# prove still holds. A refusal that still refuses, tampering that is still -# caught, an audit entry that still records the disclosure it should. -# -# What this gate is NOT for: policing what a page says. It pins no fence count, -# no command string and no documented output. Prose, the text around a -# heading, output blocks and command wording are free to change without touching -# this file, and a writer may add or remove a command block under a heading the -# journey already runs with no change here at all. If you find yourself adding -# an array of strings a page must contain, stop: that is the pinning this file -# deliberately does not do, and it is what made these tutorials unreadable for -# a human reader once already. -# -# This gate builds the Evidence toolset from the checked-out source unless -# EVIDENCE_BIN, EVIDENCECTL_BIN, and EVIDENCE_OID4VCI_BIN select exact -# candidate or released bytes, then replays each registered tutorial's own -# shell fences in its own reader directory. Every tutorial creates the files it -# needs from its documented commands, so what CI runs is what a reader copies. -# -# Usage: -# scripts/check-evidence-tutorials.sh replay every tutorial -# scripts/check-evidence-tutorials.sh --dry-run resolve the journeys only -# scripts/check-evidence-tutorials.sh --only one tutorial and its prerequisites -# -# Registering a tutorial means adding its slug to EVIDENCE_TUTORIALS and a -# branch to load_spec. Each spec holds two things: -# -# SPEC_STEPS the reader journey, in order. It need not follow document -# order: a tutorial that leaves one terminal in an earlier -# directory is replayed by running its later fence first. -# Fences are addressed by the heading they sit under, never by -# position, so inserting a command block cannot silently move a -# step onto the wrong command. -# run: execute every sh fence under -# that heading, in document order -# run:| execute the nth sh fence under -# that heading -# run-fails:| execute one sh fence the page -# documents as refused, and -# require a non-zero exit -# background:| run a one-line sh fence the page -# leaves running in a second -# terminal -# stop-background stop the most recently started -# background fence, where the page -# says to press Ctrl+C -# save:H|lang|occ|target write a documented non-shell -# fence to the file the reader is -# told to create -# edit:H|lang|occ|H2|lang2|occ2|target -# apply a documented before/after -# fence pair to an existing file -# wait-http:URL block until that URL answers -# python-client put the client package assembled -# from this checkout on the import -# path, standing in for the -# documented install -# fhir-mock start the sanitized local FHIR -# mock this gate carries -# track-pid:PATH adopt a PID a fence wrote, so -# cleanup reaches it -# The | suffix is optional wherever a heading holds a single -# sh fence. Skipping is implicit: a fence under no listed -# heading is simply not run, and the summary names it so a -# reviewer can see the unverified surface. -# -# SPEC_ASSERTS behaviours the replay transcript must still show. One test -# decides membership: would this regress silently, without any -# command exiting non-zero? Startup chatter, "created", "ready" -# and "prepared" lines fail that test, because the next command -# would have failed without them. Do not grow this back into a -# transcript pin. -# -# Renaming a heading breaks the steps that name it, by name, in --dry-run. -# That is the trade, and it is a good one: a renamed heading is a structural -# edit to the journey, it fails loudly rather than replaying the wrong command, -# and it is exactly when the journey is worth walking again. -# -# Configuration: -# EVIDENCE_BIN / EVIDENCECTL_BIN / run these exact binaries instead of -# EVIDENCE_OID4VCI_BIN building from source -# EVIDENCE_OID4VCI_INTEROP_TEST_BIN run this prebuilt sanitized flow test -# REGISTRY_CLIENT_PY_WHEEL import the client package out of this -# assembled wheel -# EVIDENCE_TUTORIAL_CARGO_PROFILE ci (default) or release -# EVIDENCE_TUTORIAL_DOCS_ROOT tutorial directory override (tests) - -set -euo pipefail - -SITE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -REPO_ROOT="$(cd "$SITE_ROOT/../.." && pwd)" -# A fence helper written against the replay userland's floor: it locates a -# fence by heading, language and occurrence and applies it to a file, using -# only the shell and coreutils the container carries. -FENCE="$SITE_ROOT/scripts/evidence-tutorial-fence.sh" -FHIR_TUTORIAL_MOCK="$SITE_ROOT/scripts/fixtures/fhir-tutorial-mock.py" -DOCS_ROOT="${EVIDENCE_TUTORIAL_DOCS_ROOT:-$SITE_ROOT/src/content/docs/tutorials}" -BUILD_PROFILE="${EVIDENCE_TUTORIAL_CARGO_PROFILE:-ci}" -TARGET_DIR="$REPO_ROOT/target/evidence-tutorial-source" - -# --------------------------------------------------------------------------- -# Registered tutorials -# --------------------------------------------------------------------------- - -EVIDENCE_TUTORIALS=( - first-evidence-assertion - request-evidence-as-sd-jwt-vc - run-oid4vci-interoperability-checks - request-evidence-from-an-application - return-a-governed-value - assert-a-role-bound-relationship - refuse-unsafe-evidence-requests - verify-an-assertion-as-a-consumer - control-who-can-request-evidence - issue-fhir-evidence-as-vcs - connect-a-sqlite-extract -) - -# Every other page under DOCS_ROOT, and the reason it is not replayed here. -# check_tutorial_coverage below fails by name on a page in neither list, which -# is the gap that let broken DHIS2 tutorial commands ship once already. -EXCLUDED_EVIDENCE_TUTORIALS=( - evidence-from-breg # native Docker PostgreSQL journey; products/breg/evidence/tests/verify-composition.py covers offline composition, independent reader checks live steps - deploy-evidence-from-breg # operated target handoff; native composition and production build tests cover offline candidates, target-host checks need provisioned dependencies - build-and-deploy-evidence-project # drift-checked by evidence-production-build-docs.test.mjs; needs a production build environment - connect-an-institution-source # how-to against the reader's own OpenAPI source; no fixed scenario this gate can replay - first-run-with-solmara-lab # historical; the Solmara Lab stack is replayed by check-tutorial.sh, not here - first-breg # Base Registry Engine journey; product CI runs quickstart/run.sh --smoke, reader execution checks the documented steps - first-casework # Registry Casework journey; replayed end to end by run-tutorial.mjs --gate casework in the casework-tutorial job - first-render-document # Registry Render journey; offline render CLI steps against the products/render example bundles, verified in reader mode outside the Evidence runner - review-breg-changes-in-casework # cross-product boundary guide with no Evidence CLI journey; real BReg-to-Casework composition runs in the owning product aggregate - extend-a-registry-with-a-module # Base Registry Engine journey; offline bregctl steps on the quickstart project, verified in reader mode outside the Evidence runner - derive-a-registry-from-publicschema # Base Registry Engine journey; offline bregctl steps deriving a project from the embedded PublicSchema snapshot, verified in reader mode outside the Evidence runner - send-registry-events-to-a-webhook # Base Registry Engine journey; needs the demo launcher's webhook receiver, verified in reader mode outside the Evidence runner - build-a-breg-production-candidate # Base Registry Engine journey; needs a PostgreSQL container and a local signing key, verified in reader mode outside the Evidence runner - query-breg-client # BReg client journey; depends on the released unified packages, like query-relay-client - integrate-evidence-candidate-with-docker-compose # drift-checked by evidence-production-build-docs.test.mjs; needs Docker Compose - issue-a-birth-certificate-vc-from-opencrvs # needs the public OpenCRVS Farajaland demo; live and opt-in, not replayed in CI - issue-immunization-evidence-from-dhis2 # needs the public DHIS2 demo; live and opt-in, not replayed in CI - manage-evidence-verifier-trust # how-to against the reader's own deployment; no fixed scenario this gate can replay - move-evidence-to-production-signing # drift-checked by evidence-production-build-docs.test.mjs; needs a Transit signer - prove-an-evidence-project # how-to against the reader's own project; no fixed scenario this gate can replay - publish-and-consume-discovery-index # Discovery journey; replayed by check-discovery-tutorial.sh with native Evidence and Relay handoffs - publish-governed-sqlite-registry # Relay V2 journey; replayed by check-relay-tutorial.sh in the relay-v2-contracts job - query-relay-client # Relay client journey; depends on a released wheel and the Relay publishing prerequisite - request-a-holder-bound-credential # draft: true, hidden from the sidebar; no verified wallet flow exists to replay - review-registry-changes # Base Registry Engine journey; verify page commands in reader mode, outside the Evidence runner - rotate-evidence-signing-keys # drift-checked by evidence-production-build-docs.test.mjs; needs a deployed signing key - verify-a-registered-parent-with-opencrvs # needs the public OpenCRVS Farajaland demo; live and opt-in, not replayed in CI -) - -in_list() { - local needle="$1" - shift - local item - for item in "$@"; do - [[ "$item" == "$needle" ]] && return 0 - done - return 1 -} - -# Assert that every page under DOCS_ROOT is either registered for replay or -# named in EXCLUDED_EVIDENCE_TUTORIALS with a reason. A page in neither list -# is a coverage gap: nothing would ever replay it or explain why not. -check_tutorial_coverage() { - local file slug - local -a unregistered=() - for slug in "${EXCLUDED_EVIDENCE_TUTORIALS[@]}"; do - if in_list "$slug" "${EVIDENCE_TUTORIALS[@]}"; then - printf 'coverage error in %s: %s is both registered in EVIDENCE_TUTORIALS and excluded in EXCLUDED_EVIDENCE_TUTORIALS\n' \ - "${BASH_SOURCE[0]}" "$slug" >&2 - exit 2 - fi - if [[ ! -f "$DOCS_ROOT/$slug.mdx" ]]; then - printf 'coverage error in %s: %s.mdx in EXCLUDED_EVIDENCE_TUTORIALS does not exist under %s\n' \ - "${BASH_SOURCE[0]}" "$slug" "$DOCS_ROOT" >&2 - exit 2 - fi - done - for file in "$DOCS_ROOT"/*.mdx; do - [[ -e "$file" ]] || continue - slug="$(basename "$file" .mdx)" - if ! in_list "$slug" "${EVIDENCE_TUTORIALS[@]}" && ! in_list "$slug" "${EXCLUDED_EVIDENCE_TUTORIALS[@]}"; then - unregistered+=("$slug") - fi - done - if ((${#unregistered[@]} > 0)); then - printf 'tutorial coverage gap: the following pages are neither registered in EVIDENCE_TUTORIALS nor excluded in EXCLUDED_EVIDENCE_TUTORIALS:\n' >&2 - for slug in "${unregistered[@]}"; do - printf ' %s.mdx\n' "$slug" >&2 - done - printf 'add each to EVIDENCE_TUTORIALS (with a load_spec branch) or to EXCLUDED_EVIDENCE_TUTORIALS with a reason, in %s\n' \ - "${BASH_SOURCE[0]}" >&2 - exit 1 - fi -} - -check_tutorial_coverage - -load_spec() { - SPEC_STEPS=() - SPEC_ASSERTS=() - - case "$1" in - first-evidence-assertion) - SPEC_STEPS=( - "run:Preview a synthetic source|1" - "save:Preview a synthetic source|yaml|1|tutorial-source.openapi.yaml" - "background:Preview a synthetic source|2" - "wait-http:http://127.0.0.1:4010/people/person-123" - "run:Preview a synthetic source|3" - "stop-background" - "run:Create the Evidence Gateway project" - "run:Keep exact cases for the tutorial|1" - "save:Create the Evidence Gateway project|yaml|1|questions/adult-status.yaml" - "save:Create the Evidence Gateway project|rhai|1|derivations/adult-status.rhai" - "save:Keep exact cases for the tutorial|yaml|1|mocks/source.yaml" - "save:Keep exact cases for the tutorial|json|1|mocks/cases/person-123.json" - "save:Keep exact cases for the tutorial|json|2|mocks/cases/person-456.json" - "save:Keep exact cases for the tutorial|json|3|mocks/cases/person-789.json" - "run:Keep exact cases for the tutorial|2" - "background:Keep exact cases for the tutorial|3" - "wait-http:http://127.0.0.1:4010/people/person-123" - "run:Keep exact cases for the tutorial|4" - "run:Request an assertion" - "run:Verify before reading" - "run:Try the SD-JWT VC serialization" - "run:Stop the local services" - "run:Inspect the audit entry" - "run:Clean up" - ) - # The assertion was verified, the audit recorded who asked and why, and - # exactly one field was released. Nothing else here regresses in - # silence: a mock that did not start or a project that was not created - # ends the journey at the next command. - SPEC_ASSERTS=( - "VERIFIED" - "ACCESS AUTHORIZED adult-status age-check requester=" - "DISCLOSURE RELEASED is_adult" - ) - ;; - request-evidence-as-sd-jwt-vc) - SPEC_STEPS=( - "background:Restart the source mock|1" - "wait-http:http://127.0.0.1:4010/people/person-123" - "run:Restart the source mock|2" - "run:Request a scalar credential" - "run:Inspect the compact structure after verification" - "run:Inspect issuer discovery" - "run:Prove tampering is refused|1" - "run-fails:Prove tampering is refused|2" - "save:Model independently disclosed fields|yaml|1|schemas/adult-assessment.yaml" - "save:Model independently disclosed fields|yaml|2|questions/adult-assessment.yaml" - "save:Model independently disclosed fields|rhai|1|derivations/adult-assessment.rhai" - "run:Model independently disclosed fields" - "run:Clean up" - ) - # The disclosure names are what a holder actually hands over, and the - # fences that print them exit zero whatever the credential carries, so a - # credential that started disclosing more would pass unnoticed. - SPEC_ASSERTS=( - "disclosure: urn:registrystack:evidence:local:concept:adult-status:is_adult" - "evidencectl: Evidence response verification failed" - "disclosure: criterion" - "disclosure: isAdult" - "ACCESS AUTHORIZED adult-assessment age-assessment-review requester=" - "DISCLOSURE RELEASED adult_assessment" - ) - ;; - run-oid4vci-interoperability-checks) - SPEC_STEPS=( - "run:Copy the complete configuration" - "save:Copy the complete configuration|yaml|1|.tutorial/oid4vci-adopter/oid4vci.yaml" - "run:Replay the sanitized profile" - "run:Clean up" - ) - # The sanitized runner prints one line per phase and exits non-zero on - # any of them, so the phases hold themselves up. What they cannot hold - # up is having run at all: a filter that selects no test leaves the - # runner exiting zero with nothing done. One end-to-end line proves the - # wallet flow ran; the rest would be a transcript pin. - SPEC_ASSERTS=( - "PRESENTATION VERIFIED: public wallet flow returned holder-bound Evidence" - ) - ;; - return-a-governed-value) - SPEC_STEPS=( - "background:Restart the source mock" - "wait-http:http://127.0.0.1:4010/people/person-123" - "run:Add the age-bracket question" - "save:Add the age-bracket question|yaml|1|questions/age-bracket.yaml" - "save:Add the age-bracket question|rhai|1|derivations/age-bracket.rhai" - "run:Start the updated project" - "run:Request and verify the bracket" - "run:Inspect the audit and clean up" - ) - SPEC_ASSERTS=( - "VERIFIED" - "ACCESS AUTHORIZED age-bracket service-path-selection requester=" - "DISCLOSURE RELEASED age_bracket" - ) - ;; - control-who-can-request-evidence) - SPEC_STEPS=( - "background:Restart the source mock|1" - "wait-http:http://127.0.0.1:4010/people/person-123" - "run:Restart the source mock|2" - "run:Define two access policies" - "run:Register the first local application" - "run:Start the protected service" - "run:Make an allowed request" - "run:Add an application for the next generation" - "run:Use the application assigned the policy" - "run:Try a question the application was not granted" - "run:Revoke an application|1" - "run:Revoke an application|2" - "run-fails:Revoke an application|3" - "run:Stop the final generation" - "run:Clean up" - ) - # This tutorial teaches refusal, so the refusals are what must hold. - # The unauthorized request's curl carries no --fail-with-body, so it - # exits zero on a 403 and a boundary that started answering 200 would - # leave the journey green. The post-revocation preparation requires a - # non-zero exit; the message proves the client was revoked rather than - # refused for some unrelated reason. - SPEC_ASSERTS=( - "VERIFIED" - "HTTP 403" - '"code": "evidence.denied"' - "evidencectl: unknown or revoked active client age-checker" - "ACCESS REFUSED requester=" - "reason=not_authorized" - ) - ;; - assert-a-role-bound-relationship) - SPEC_STEPS=( - "run:Start a relationship registry|1" - "save:Start a relationship registry|python|1|registry.py" - "background:Start a relationship registry|2" - "wait-http:http://127.0.0.1:8002/openapi.json" - "run:Create the Evidence Gateway project" - "save:Create the Evidence Gateway project|yaml|1|questions/parent-relationship.yaml" - "save:Create the Evidence Gateway project|rhai|1|derivations/parent-relationship.rhai" - "run:Start the project" - "run:Bind both subjects to the request" - "run:Inspect the audit and clean up" - ) - SPEC_ASSERTS=( - "VERIFIED" - "ACCESS AUTHORIZED parent-relationship relationship-check requester=" - "DISCLOSURE RELEASED relationship_confirmed" - ) - ;; - refuse-unsafe-evidence-requests) - SPEC_STEPS=( - "background:Restart the local boundary|1" - "wait-http:http://127.0.0.1:4010/people/person-123" - "run:Restart the local boundary|2" - "run:Prepare one authorized request" - "run:Change the purpose after preparation" - "run:Obtain and verify the authorized response" - "run:Change the signed response|1" - "run-fails:Change the signed response|2" - "run:Clean up" - ) - # The whole page is these three outcomes: the altered request was - # refused, the untouched one verified, and the altered response was - # caught. The refusal curl exits zero on a 403, so only the printed - # status separates a boundary that refused from one that answered. - SPEC_ASSERTS=( - "HTTP 403" - "VERIFIED" - "evidencectl: Evidence response verification failed" - ) - ;; - verify-an-assertion-as-a-consumer) - SPEC_STEPS=( - "run:Start with three separate inputs" - "run:Re-verify the recorded decision" - ) - # `evidence verify` exits non-zero on both `authentic: no` and - # `currently-valid: no`, so the verdict holds itself up. The disclosed - # value does not: verification succeeds whatever the assertion says, and - # a consumer reading the wrong answer is the failure that matters. - SPEC_ASSERTS=( - '"value": true' - ) - ;; - request-evidence-from-an-application) - SPEC_STEPS=( - # The registry runs in the terminal the reader never moved out of - # the first tutorial's directory, so it starts before the `cd` the - # page opens with rather than where the page prints it. - "background:Start the local services|1" - "wait-http:http://127.0.0.1:4010/people/person-123" - "run:Give the application its own identity" - "run:Pin the keys your application trusts" - # Stands in for the fence under "Install the Python client", the - # documented install of the released client package. - "python-client" - "run:Start the local services|2" - "run-fails:Start the local services|3" - "run:Read the definitions once" - "run:Pin the procedure" - "save:Write the relying procedure|python|1|age_check.py" - "run:Run it" - "run-fails:Refuse before reading" - "run:Stop the local services" - ) - # What the relying application actually did. Both refusals already - # exit non-zero, so what is held here is the reason: an unnamed caller - # refused for want of a registered client, and an unverifiable response - # refused before anything was read. The two answers prove the right - # subject was resolved rather than a constant returned, the pinning line - # proves the subject binding is still recorded, and the assurance - # profile is the trust level a relying party reads off the deployment. - SPEC_ASSERTS=( - "evidencectl: the active project requires a registered client selected with --client" - '"assuranceProfile": "local"' - "person-123 is_adult=True" - "person-456 is_adult=False" - "pinned binding recorded in subject-bindings.json" - "unverifiable response, nothing read (policy)" - ) - ;; - connect-a-sqlite-extract) - SPEC_STEPS=( - "run:Create and prove the starter" - ) - # The scaffold and the fixture run hold themselves up: a starter that - # failed to scaffold ends the journey at the next command, and the - # fixture run exits non-zero on any case it cannot prove. - SPEC_ASSERTS=() - ;; - issue-fhir-evidence-as-vcs) - SPEC_STEPS=( - "run:Select live synthetic records|1" - "save:Select live synthetic records|python|1|discover-fhir-records.py" - "fhir-mock" - "run:Select live synthetic records|2" - "save:Run a live FHIR read-through adapter|python|1|fhir-read-through.py" - "run:Run a live FHIR read-through adapter" - "track-pid:fhir-read-through.pid" - "save:Describe the exact FHIR reads|yaml|1|fhir-smart-r4.openapi.yaml" - "run:Describe the exact FHIR reads" - "save:Author the patient coverage question|yaml|1|questions/fhir-coverage-status.yaml" - "save:Author the patient coverage question|rhai|1|derivations/fhir-coverage-status.rhai" - "save:Author the healthcare-establishment question|yaml|1|questions/fhir-healthcare-establishment.yaml" - "save:Author the healthcare-establishment question|rhai|1|derivations/fhir-healthcare-establishment.rhai" - "run:Start the project" - "run:Request the patient coverage credential" - "run:Request the healthcare-establishment credential" - "run:Inspect the audit and clean up" - ) - SPEC_ASSERTS=( - "VERIFIED" - "ACCESS AUTHORIZED fhir-healthcare-establishment healthcare-establishment-verification requester=" - "DISCLOSURE RELEASED healthcare_provider_record_active" - ) - ;; - *) - printf '%s is not a registered Evidence tutorial\n' "$1" >&2 - exit 2 - ;; - esac -} - -# --------------------------------------------------------------------------- -# Arguments -# --------------------------------------------------------------------------- - -DRY_RUN=0 -ONLY="" -while (($# > 0)); do - case "$1" in - --dry-run) - DRY_RUN=1 - shift - ;; - --only) - if (($# < 2)); then - printf -- '--only needs a tutorial slug\n' >&2 - exit 2 - fi - ONLY="$2" - shift 2 - ;; - *) - printf 'unknown argument: %s (expected --dry-run or --only )\n' "$1" >&2 - exit 2 - ;; - esac -done - -if [[ -n "$ONLY" ]]; then - # load_spec exits on an unregistered slug, which is the check we want here. - load_spec "$ONLY" - # Every follow-up begins from the project first-evidence-assertion builds. - # A full run gets that from the list order; --only has to name it. - case "$ONLY" in - request-evidence-as-sd-jwt-vc | return-a-governed-value | \ - refuse-unsafe-evidence-requests | verify-an-assertion-as-a-consumer | \ - control-who-can-request-evidence | request-evidence-from-an-application) - EVIDENCE_TUTORIALS=(first-evidence-assertion "$ONLY") - ;; - *) EVIDENCE_TUTORIALS=("$ONLY") ;; - esac -fi - -WORK_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/evidence-tutorial.XXXXXX")" -cleanup() { - local exit_code=$? - set +e - chmod -R u+w "$WORK_ROOT" 2>/dev/null - rm -rf "$WORK_ROOT" - if ((exit_code == 0)); then - printf 'Evidence tutorial gate: PASS\n' - else - printf 'Evidence tutorial gate: FAIL (exit %d)\n' "$exit_code" >&2 - fi -} -trap cleanup EXIT -trap 'exit 130' HUP INT TERM - -# --------------------------------------------------------------------------- -# Toolset under test -# --------------------------------------------------------------------------- - -resolve_profile_dir() { - case "$BUILD_PROFILE" in - ci | release) printf '%s' "$BUILD_PROFILE" ;; - *) - printf 'unsupported tutorial Cargo profile: %s (expected ci or release)\n' \ - "$BUILD_PROFILE" >&2 - exit 1 - ;; - esac -} - -SHIM_DIR="$WORK_ROOT/bin" - -prepare_toolset() { - if [[ -z "${EVIDENCE_BIN:-}" || -z "${EVIDENCECTL_BIN:-}" || \ - -z "${EVIDENCE_OID4VCI_BIN:-}" ]]; then - local profile_dir - profile_dir="$(resolve_profile_dir)" - (cd "$REPO_ROOT" && CARGO_TARGET_DIR="$TARGET_DIR" \ - cargo build --locked --profile "$BUILD_PROFILE" \ - -p registry-evidence -p registry-evidencectl \ - -p registry-evidence-oid4vci) - EVIDENCE_BIN="$TARGET_DIR/$profile_dir/evidence" - EVIDENCECTL_BIN="$TARGET_DIR/$profile_dir/evidencectl" - EVIDENCE_OID4VCI_BIN="$TARGET_DIR/$profile_dir/evidence-oid4vci" - fi - export EVIDENCE_OID4VCI_BIN - local bin - for bin in "$EVIDENCE_BIN" "$EVIDENCECTL_BIN" "$EVIDENCE_OID4VCI_BIN"; do - # Absoluteness first: the reader journey runs from its own directory and - # reaches the binaries through symlinks, so a relative path resolves - # against the wrong directory and would otherwise surface much later, - # mid-journey, as "command not found". - if [[ "$bin" != /* ]]; then - printf 'toolset binary path must be absolute: %s\n' "$bin" >&2 - exit 1 - fi - if [[ ! -x "$bin" ]]; then - printf 'toolset binary not executable: %s\n' "$bin" >&2 - exit 1 - fi - done - - # The tutorials call the binaries by name, so serve them from a shim dir. - mkdir -p "$SHIM_DIR" - ln -s "$EVIDENCE_BIN" "$SHIM_DIR/evidence" - ln -s "$EVIDENCECTL_BIN" "$SHIM_DIR/evidencectl" - ln -s "$EVIDENCE_OID4VCI_BIN" "$SHIM_DIR/evidence-oid4vci" -} - -# The unified client package, unpacked once for whichever tutorials import it. -# -# The documented install resolves the package from an index at the running -# runtime's version, which is the right instruction for a reader and the wrong -# one for this gate: it needs the network, and it would prove a released client -# rather than the one in this checkout. Importing a package assembled from this -# checkout instead is what makes a client regression fail this gate on the -# commit that introduces it. The documented install fence it stands in for is -# reported as unexecuted, so its version selector stays a reviewer's call -# rather than this gate's. -# The bindings inside are built for the stable ABI, so a wheel assembled -# outside this script imports under any CPython the replay userland carries, -# exactly as EVIDENCE_BIN's siblings let CI mount prebuilt binaries. Assembling -# one needs a build toolchain the replay userland does not carry, so this gate -# never assembles: the caller names a wheel, or the gate stops here. -REGISTRY_CLIENT_PY_WHEEL="${REGISTRY_CLIENT_PY_WHEEL:-}" - -prepare_python_client() { - if [[ -z "$REGISTRY_CLIENT_PY_WHEEL" ]]; then - printf 'REGISTRY_CLIENT_PY_WHEEL is unset: name a client wheel assembled with %s\n' \ - 'release/scripts/assemble-registry-client-packages.py' >&2 - exit 1 - fi - if [[ "$REGISTRY_CLIENT_PY_WHEEL" != /* ]]; then - printf 'client wheel path must be absolute: %s\n' "$REGISTRY_CLIENT_PY_WHEEL" >&2 - exit 1 - fi - if [[ ! -f "$REGISTRY_CLIENT_PY_WHEEL" ]]; then - printf 'client wheel not found: %s; assemble one with %s\n' \ - "$REGISTRY_CLIENT_PY_WHEEL" \ - 'release/scripts/assemble-registry-client-packages.py' >&2 - exit 1 - fi -} - -# --------------------------------------------------------------------------- -# Journey assembly -# --------------------------------------------------------------------------- - -# Resolve a heading address to the sh fence numbers it names, in document -# order, space separated. -# -# An address is a heading, optionally followed by | to name one -# fence under it. Addressing by heading rather than by position is what lets a -# writer add or remove a command block without touching a spec, and it is what -# stops an inserted block from silently moving a later step onto the wrong -# command. -resolve_fences() { - local slug="$1" address="$2" fence_dir="$3" - local heading="$address" occurrence="" - if [[ "$address" == *'|'* ]]; then - heading="${address%%|*}" - occurrence="${address##*|}" - if [[ ! "$occurrence" =~ ^[1-9][0-9]*$ ]]; then - printf 'tutorial spec error in %s: fence occurrence must be a positive integer: %s\n' \ - "$slug" "$address" >&2 - exit 2 - fi - fi - local matched - matched="$(awk -F '\t' -v want="$heading" -v want_occurrence="$occurrence" ' - $3 != want { next } - want_occurrence != "" && $2 != want_occurrence + 0 { next } - { printf "%s ", $1 } - ' "$fence_dir/index.tsv")" - matched="${matched% }" - if [[ -z "$matched" ]]; then - printf 'tutorial drift in %s: no sh fence answers to "%s"\n' "$slug" "$address" >&2 - printf 'A step names a heading the page no longer carries, or an occurrence under it that no longer exists.\n' >&2 - printf 'Renaming a heading is a structural edit to the journey; walk it again, then name the new heading in %s.\n' \ - "${BASH_SOURCE[0]}" >&2 - printf 'The page currently holds these sh fences:\n' >&2 - awk -F '\t' '{ printf " fence %s, occurrence %s under \"%s\"\n", $1, $2, $3 }' \ - "$fence_dir/index.tsv" >&2 - exit 1 - fi - printf '%s\n' "$matched" -} - -# Resolve a heading address that must name exactly one sh fence. -resolve_one_fence() { - local slug="$1" address="$2" fence_dir="$3" step_kind="$4" - local matched - matched="$(resolve_fences "$slug" "$address" "$fence_dir")" || exit $? - local -a numbers - read -r -a numbers <<<"$matched" - if ((${#numbers[@]} != 1)); then - printf 'tutorial spec error in %s: a %s step runs one fence, but "%s" names %d; add |\n' \ - "$slug" "$step_kind" "$address" "${#numbers[@]}" >&2 - exit 2 - fi - printf '%s\n' "${numbers[0]}" -} - -# Emit the sh fences named by a run: step, in document order. -emit_run_step() { - local slug="$1" address="$2" fence_dir="$3" - local matched - matched="$(resolve_fences "$slug" "$address" "$fence_dir")" || exit $? - local -a numbers - read -r -a numbers <<<"$matched" - local number - for number in "${numbers[@]}"; do - printf '\nprintf "==> %s fence %s\\n"\n' "$slug" "$number" - cat "$fence_dir/fence-$number.sh" - done -} - -# Emit one sh fence the tutorial documents as refused, and require it to fail. -# -# A refusal the tutorial teaches is as much a documented outcome as a success, -# so replaying it means asserting the non-zero exit rather than tolerating it: -# a fence that starts succeeding has stopped teaching what the page says. -# -# The fence runs on its own line rather than as an `if` condition, because bash -# suppresses errexit throughout a condition, subshells included, even one that -# sets it itself. A fence that refuses on its first command and then prints -# would run that print and report success, which is neither what the reader -# sees nor what the page documents. `set +e` around the run keeps the failure -# from ending the journey, and reinstates errexit for the steps after it. -emit_run_fails_step() { - local slug="$1" address="$2" fence_dir="$3" - local number - number="$(resolve_one_fence "$slug" "$address" "$fence_dir" run-fails)" || exit $? - printf '\nprintf "==> %s fence %s (documented refusal)\\n"\n' "$slug" "$number" - printf 'set +e\n' - printf '( set -e\n' - cat "$fence_dir/fence-$number.sh" - printf ')\nrefusal_status=$?\nset -e\n' - printf 'if ((refusal_status == 0))\nthen\n' - printf ' printf "tutorial drift in %s: fence %s succeeded, but the page documents a refusal\\n" >&2\n' \ - "$slug" "$number" - printf ' exit 1\n' - printf 'fi\n' -} - -# Put the client package assembled from this checkout where the tutorial's -# install fence puts the released one: on the import path of the shell the -# reader's commands run in. The replay userland carries unzip and no installer, -# so the wheel is unpacked rather than installed, which is enough because the -# package declares no dependencies of its own. -emit_python_client_step() { - local slug="$1" - printf '\nprintf "==> %s unpack the client package assembled from this checkout\\n"\n' "$slug" - printf 'mkdir -p client-package\n' - printf 'unzip -q -o %q -d client-package\n' "$REGISTRY_CLIENT_PY_WHEEL" - # shellcheck disable=SC2016 # PYTHONPATH expands in the emitted script - printf 'PYTHONPATH="$PWD/client-package${PYTHONPATH:+:$PYTHONPATH}"\nexport PYTHONPATH\n' -} - -# Emit a documented before/after fence pair applied to a file the reader edits. -# -# Both fences are read out of the tutorial here, while the journey is being -# assembled, so a pair the tutorial no longer carries fails by name before the -# reader's first command runs. -emit_edit_step() { - local slug="$1" spec="$2" tutorial_file="$3" edit_dir="$4" - local IFS='|' - # shellcheck disable=SC2206 # deliberate split on the field separator - local parts=($spec) - if ((${#parts[@]} != 7)); then - printf 'tutorial spec error in %s: edit step needs 7 fields, got %d: %s\n' \ - "$slug" "${#parts[@]}" "$spec" >&2 - exit 2 - fi - EDIT_INDEX=$((EDIT_INDEX + 1)) - local before after - before="$(printf '%s/edit-%02d-before' "$edit_dir" "$EDIT_INDEX")" - after="$(printf '%s/edit-%02d-after' "$edit_dir" "$EDIT_INDEX")" - if ! bash "$FENCE" write-fence "$tutorial_file" \ - "${parts[0]}" "${parts[1]}" "${parts[2]}" "$before" || - ! bash "$FENCE" write-fence "$tutorial_file" \ - "${parts[3]}" "${parts[4]}" "${parts[5]}" "$after"; then - printf 'tutorial drift in %s: edit step names a fence the tutorial no longer carries: %s\n' \ - "$slug" "$spec" >&2 - exit 1 - fi - printf '\nprintf "==> %s edit %s\\n"\n' "$slug" "${parts[6]}" - # shellcheck disable=SC2016 # FENCE expands in the emitted script - printf 'bash "$FENCE" replace-block %q %q %q\n' "${parts[6]}" "$before" "$after" -} - -# Save a documented non-shell fence as the file the reader is instructed to -# create. The maintained Markdown remains the single source of those bytes. -emit_save_step() { - local slug="$1" spec="$2" - local IFS='|' - # shellcheck disable=SC2206 # deliberate split on the field separator - local parts=($spec) - if ((${#parts[@]} != 4)); then - printf 'tutorial spec error in %s: save step needs 4 fields, got %d: %s\n' \ - "$slug" "${#parts[@]}" "$spec" >&2 - exit 2 - fi - printf '\nprintf "==> %s save %s\\n"\n' "$slug" "${parts[3]}" - # shellcheck disable=SC2016 # FENCE and TUTORIAL expand in the emitted script - printf 'bash "$FENCE" write-fence "$TUTORIAL" %q %q %q %q\n' \ - "${parts[0]}" "${parts[1]}" "${parts[2]}" "${parts[3]}" -} - -# A tutorial may ask the reader to leave one foreground command running in a -# second terminal. CI runs that exact one-line command in the background and -# retains its PID for cleanup. -emit_background_step() { - local slug="$1" address="$2" fence_dir="$3" - local number - number="$(resolve_one_fence "$slug" "$address" "$fence_dir" background)" || exit $? - local fence="$fence_dir/fence-$number.sh" - if [[ "$(wc -l <"$fence")" -ne 1 ]]; then - printf 'tutorial spec error in %s: a background step needs one sh line, but fence %s under "%s" holds more\n' \ - "$slug" "$number" "$address" >&2 - exit 2 - fi - local command - IFS= read -r command <"$fence" - printf '\nprintf "==> %s fence %s (background)\\n"\n' "$slug" "$number" - printf '%s &\n' "$command" - printf 'BACKGROUND_PIDS+=("$!")\n' -} - -# Stop the foreground command the page told the reader to leave running in -# another terminal. This models Ctrl+C without adding a shell fence that a -# reader would never type. -emit_stop_background_step() { - local slug="$1" - printf '\nprintf "==> %s stop the previous background fence\\n"\n' "$slug" - printf 'if ((${#BACKGROUND_PIDS[@]} == 0)); then printf "tutorial spec error in %s: no background fence to stop\\n" >&2; exit 2; fi\n' "$slug" - printf 'background_index=$((${#BACKGROUND_PIDS[@]} - 1))\n' - printf 'background_pid="${BACKGROUND_PIDS[$background_index]}"\n' - printf 'kill "$background_pid" >/dev/null 2>&1 || true\n' - printf 'wait "$background_pid" >/dev/null 2>&1 || true\n' - printf 'unset "BACKGROUND_PIDS[$background_index]"\n' -} - -emit_wait_http_step() { - local url="$1" - printf '\nfor attempt in {1..50}; do\n' - printf ' if curl -fs %q >/dev/null 2>&1; then break; fi\n' "$url" - printf ' if [[ "$attempt" -eq 50 ]]; then printf "tutorial service did not become ready\\n" >&2; exit 1; fi\n' - printf ' sleep 0.1\n' - printf 'done\n' -} - -emit_fhir_mock_step() { - printf '\nprintf "==> start sanitized local FHIR mock\\n"\n' - printf '%q >%q 2>&1 &\n' "$FHIR_TUTORIAL_MOCK" "$WORK_ROOT/fhir-tutorial-mock.log" - printf 'BACKGROUND_PIDS+=("$!")\n' - printf 'for attempt in {1..50}; do\n' - printf ' if curl --noproxy "*" -fs http://127.0.0.1:8003/healthz >/dev/null 2>&1; then break; fi\n' - printf ' if [[ "$attempt" -eq 50 ]]; then printf "sanitized FHIR mock did not become ready\\n" >&2; exit 1; fi\n' - printf ' sleep 0.1\n' - printf 'done\n' -} - -emit_track_pid_step() { - local path="$1" - printf '\ntracked_pid="$(cat %q)"\n' "$path" - printf 'if [[ ! "$tracked_pid" =~ ^[1-9][0-9]*$ ]]; then printf %q >&2; exit 1; fi\n' \ - "invalid tracked PID in $path\n" - printf 'BACKGROUND_PIDS+=("$tracked_pid")\n' -} - -emit_journey() { - local slug="$1" fence_dir="$2" tutorial_file="$3" - local edit_dir="$WORK_ROOT/edits/$slug" - mkdir -p "$edit_dir" - EDIT_INDEX=0 - printf 'set -euo pipefail\n' - printf 'FENCE=%q\n' "$FENCE" - printf 'TUTORIAL=%q\n' "$tutorial_file" - printf 'BACKGROUND_PIDS=()\n' - printf 'cleanup_journey() {\n' - printf ' if [[ -S .evidence/dev/control.sock ]]; then evidencectl dev stop >/dev/null 2>&1 || true; fi\n' - printf ' local pid\n' - printf ' for pid in "${BACKGROUND_PIDS[@]}"; do kill "$pid" >/dev/null 2>&1 || true; wait "$pid" >/dev/null 2>&1 || true; done\n' - printf '}\n' - printf 'trap cleanup_journey EXIT\n' - printf 'trap "exit 130" HUP INT TERM\n' - local step - for step in ${SPEC_STEPS[@]+"${SPEC_STEPS[@]}"}; do - case "$step" in - run:*) emit_run_step "$slug" "${step#run:}" "$fence_dir" ;; - run-fails:*) emit_run_fails_step "$slug" "${step#run-fails:}" "$fence_dir" ;; - python-client) emit_python_client_step "$slug" ;; - fhir-mock) emit_fhir_mock_step ;; - track-pid:*) emit_track_pid_step "${step#track-pid:}" ;; - edit:*) emit_edit_step "$slug" "${step#edit:}" "$tutorial_file" "$edit_dir" ;; - save:*) emit_save_step "$slug" "${step#save:}" ;; - background:*) emit_background_step "$slug" "${step#background:}" "$fence_dir" ;; - stop-background) emit_stop_background_step "$slug" ;; - wait-http:*) emit_wait_http_step "${step#wait-http:}" ;; - *) - printf 'tutorial spec error in %s: unknown step: %s\n' "$slug" "$step" >&2 - exit 2 - ;; - esac - done -} - -# Resolve every fence-addressing step into EXECUTED_FENCES, in step order. -# -# This runs before the replay and in --dry-run, so a heading a spec names but -# the page no longer carries fails by name in seconds, without a toolchain. -resolve_journey_fences() { - local slug="$1" fence_dir="$2" - EXECUTED_FENCES=() - local step matched number - local -a numbers - for step in ${SPEC_STEPS[@]+"${SPEC_STEPS[@]}"}; do - case "$step" in - run:*) matched="$(resolve_fences "$slug" "${step#run:}" "$fence_dir")" || exit $? ;; - run-fails:*) - matched="$(resolve_one_fence "$slug" "${step#run-fails:}" "$fence_dir" run-fails)" || exit $? - ;; - background:*) - matched="$(resolve_one_fence "$slug" "${step#background:}" "$fence_dir" background)" || exit $? - ;; - *) continue ;; - esac - read -r -a numbers <<<"$matched" - for number in "${numbers[@]}"; do - if ! in_list "$number" ${EXECUTED_FENCES[@]+"${EXECUTED_FENCES[@]}"}; then - EXECUTED_FENCES+=("$number") - fi - done - done -} - -# Name the sh fences the journey never runs. -# -# This is information for a reviewer, not a rule: an install one-liner or a -# recovery block a reader only reaches on a bad day is documented and -# unverified, and saying so is more use than pinning its text would be. -report_unexecuted_fences() { - local slug="$1" fence_dir="$2" - local number occurrence heading first_line - while IFS=$'\t' read -r number occurrence heading; do - if in_list "$number" ${EXECUTED_FENCES[@]+"${EXECUTED_FENCES[@]}"}; then - continue - fi - first_line="" - IFS= read -r first_line <"$fence_dir/fence-$number.sh" || true - printf ' not executed: fence %s under "%s": %s\n' "$number" "$heading" "$first_line" - done <"$fence_dir/index.tsv" -} - -# Hold the behaviours a successful exit does not already prove. -# -# Read the SPEC_ASSERTS note in the header before adding an entry here. This -# holds outcomes, never the transcript: a page is free to reword everything -# around the line, and the line itself is only here because losing it would -# leave the journey green. -assert_transcript() { - local slug="$1" run_log="$2" - local expected - for expected in ${SPEC_ASSERTS[@]+"${SPEC_ASSERTS[@]}"}; do - if ! grep -F -q -- "$expected" "$run_log"; then - printf 'tutorial behaviour drift in %s: the replay ran, but its transcript never showed "%s"\n' \ - "$slug" "$expected" >&2 - printf 'Every command exited zero, so this is the kind of regression only this assertion catches.\n' >&2 - exit 1 - fi - done -} - -# The sanitized OID4VCI runner may fall back to Cargo when CI has not supplied -# its prebuilt interoperability test. Keep that build in this gate's target -# directory without changing the documented Cargo behavior of other tutorials. -run_journey_script() { - local slug="$1" reader_dir="$2" run_script="$3" - if [[ "$slug" == "run-oid4vci-interoperability-checks" ]]; then - (cd "$reader_dir" && PATH="$SHIM_DIR:$PATH" CARGO_TARGET_DIR="$TARGET_DIR" bash "$run_script") - elif [[ "$slug" == "issue-fhir-evidence-as-vcs" ]]; then - ( - unset CARGO_TARGET_DIR - cd "$reader_dir" - PATH="$SHIM_DIR:$PATH" \ - FHIR_TUTORIAL_TEST_BASE_URL="http://127.0.0.1:8003" \ - bash "$run_script" - ) - else - ( - unset CARGO_TARGET_DIR - cd "$reader_dir" - PATH="$SHIM_DIR:$PATH" bash "$run_script" - ) - fi -} - -# --------------------------------------------------------------------------- -# Replay -# --------------------------------------------------------------------------- - -if ((DRY_RUN == 0)) && ((${#EVIDENCE_TUTORIALS[@]} > 0)); then - prepare_toolset -fi - -for slug in "${EVIDENCE_TUTORIALS[@]}"; do - load_spec "$slug" - tutorial_file="$DOCS_ROOT/$slug.mdx" - if [[ ! -f "$tutorial_file" ]]; then - printf 'Evidence tutorial not found: %s\n' "$tutorial_file" >&2 - exit 1 - fi - - # Extract every sh fence, in order, into numbered files, and index each one - # by the heading it sits under and its occurrence there. Heading - # attribution matches the fence helper the save and edit steps use, so one - # address means the same thing everywhere in a spec: a level-2 heading opens - # a section, and occurrences are counted per heading. - fence_dir="$WORK_ROOT/fences/$slug" - mkdir -p "$fence_dir" - : >"$fence_dir/index.tsv" - fence_count="$(awk -v outdir="$fence_dir" -v index_file="$fence_dir/index.tsv" ' - in_fence == 0 && /^##[ \t]+/ { - heading = $0 - sub(/^##[ \t]+/, "", heading) - sub(/[ \t]+$/, "", heading) - next - } - in_fence == 0 && /^```[A-Za-z0-9_-]+$/ { - in_fence = 1 - capture = ($0 == "```sh") - if (capture) { - count += 1 - occurrence[heading] += 1 - printf "%02d\t%d\t%s\n", count, occurrence[heading], heading > index_file - } - next - } - in_fence && /^```$/ { in_fence = 0; capture = 0; next } - in_fence && capture { print > (outdir "/fence-" sprintf("%02d", count) ".sh") } - END { print count + 0 } - ' "$tutorial_file")" - - resolve_journey_fences "$slug" "$fence_dir" - - printf '%s: %s sh fences, %s executed\n' \ - "$slug" "$fence_count" "${#EXECUTED_FENCES[@]}" - report_unexecuted_fences "$slug" "$fence_dir" - - if ((DRY_RUN)); then - continue - fi - - # Replay the journey in one shell so `cd` persists exactly as a reader - # experiences it, from a reader directory of this tutorial's own. - case "$slug" in - first-evidence-assertion) - reader_dir="$WORK_ROOT/reader/evidence-start" - ;; - run-oid4vci-interoperability-checks) - # The runner remains sourced from the checkout, but the copied adopter - # configuration belongs to a fresh writable reader directory. This also - # proves the journey in CI, where the checkout is mounted read-only. - reader_dir="$WORK_ROOT/reader/run-oid4vci-interoperability-checks" - ;; - request-evidence-as-sd-jwt-vc) - # This follow-up deliberately rewrites the starter project to explore a - # structured VC. Give it a copy so the other follow-ups still begin from - # the exact project produced by first-evidence-assertion. - reader_dir="$WORK_ROOT/reader/request-evidence-as-sd-jwt-vc" - cp -R "$WORK_ROOT/reader/evidence-start/first-evidence-assertion" "$reader_dir" - ;; - request-evidence-from-an-application) - # This follow-up gives the project its first access policy, which - # retires the unnamed development caller the other follow-ups still - # use, and writes a trusted key file one of them writes too. Both are - # the reader's own project to change, so it gets a copy, and it takes - # it here, before any follow-up has touched the starter project. - reader_dir="$WORK_ROOT/reader/request-evidence-from-an-application" - cp -R "$WORK_ROOT/reader/evidence-start/first-evidence-assertion" "$reader_dir" - ;; - return-a-governed-value) - reader_dir="$WORK_ROOT/reader/evidence-start/first-evidence-assertion" - ;; - control-who-can-request-evidence) - reader_dir="$WORK_ROOT/reader/evidence-start/first-evidence-assertion" - ;; - refuse-unsafe-evidence-requests) - reader_dir="$WORK_ROOT/reader/evidence-start/first-evidence-assertion" - ;; - verify-an-assertion-as-a-consumer) - reader_dir="$WORK_ROOT/reader/evidence-start/first-evidence-assertion" - ;; - *) reader_dir="$WORK_ROOT/reader/$slug" ;; - esac - mkdir -p "$reader_dir" - if [[ "$slug" == "run-oid4vci-interoperability-checks" ]]; then - ln -s "$REPO_ROOT/products" "$reader_dir/products" - ln -s "$REPO_ROOT/crates" "$reader_dir/crates" - ln -s "$REPO_ROOT/Cargo.toml" "$reader_dir/Cargo.toml" - ln -s "$REPO_ROOT/Cargo.lock" "$reader_dir/Cargo.lock" - fi - for step in ${SPEC_STEPS[@]+"${SPEC_STEPS[@]}"}; do - if [[ "$step" == "python-client" ]]; then - prepare_python_client - fi - done - run_script="$WORK_ROOT/run-$slug.sh" - emit_journey "$slug" "$fence_dir" "$tutorial_file" >"$run_script" - - run_log="$WORK_ROOT/run-$slug.log" - if ! run_journey_script "$slug" "$reader_dir" "$run_script" 2>&1 | - tee "$run_log"; then - printf 'tutorial %s failed; the transcript ends just before this line\n' \ - "$slug" >&2 - exit 1 - fi - - assert_transcript "$slug" "$run_log" -done - -if ((${#EVIDENCE_TUTORIALS[@]} == 1)); then - printf 'Checked 1 tutorial.\n' -else - printf 'Checked %d tutorials.\n' "${#EVIDENCE_TUTORIALS[@]}" -fi diff --git a/docs/site/scripts/check-evidence-tutorials.test.mjs b/docs/site/scripts/check-evidence-tutorials.test.mjs deleted file mode 100644 index d3fa31c3a9..0000000000 --- a/docs/site/scripts/check-evidence-tutorials.test.mjs +++ /dev/null @@ -1,804 +0,0 @@ -import assert from 'node:assert/strict'; -import { execFile } from 'node:child_process'; -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { dirname, join, resolve } from 'node:path'; -import test from 'node:test'; -import { fileURLToPath } from 'node:url'; -import { promisify } from 'node:util'; - -const execFileAsync = promisify(execFile); -const scriptDir = dirname(fileURLToPath(import.meta.url)); -const gate = resolve(scriptDir, 'check-evidence-tutorials.sh'); -const fenceHelper = resolve(scriptDir, 'evidence-tutorial-fence.sh'); -const fhirTutorial = resolve( - scriptDir, - '../src/content/docs/tutorials/issue-fhir-evidence-as-vcs.mdx', -); - -async function runGate(env = {}, args = ['--dry-run']) { - try { - const { stdout, stderr } = await execFileAsync('bash', [gate, ...args], { - env: { ...process.env, ...env }, - }); - return { code: 0, output: `${stdout}${stderr}` }; - } catch (error) { - return { code: error.code ?? 1, output: `${error.stdout}${error.stderr}` }; - } -} - -async function runShell(script) { - try { - const { stdout, stderr } = await execFileAsync('bash', ['-c', script]); - return { code: 0, output: `${stdout}${stderr}` }; - } catch (error) { - return { code: error.code ?? 1, output: `${error.stdout}${error.stderr}` }; - } -} - -// Counts are reported, never required: a writer who adds or removes a command -// block under an existing heading changes these numbers and neither the gate -// nor this test may object. Only the registration is asserted. -test('the dry-run gate resolves every registered Evidence tutorial', async () => { - const { code, output } = await runGate(); - assert.equal(code, 0, output); - for (const slug of [ - 'first-evidence-assertion', - 'request-evidence-as-sd-jwt-vc', - 'run-oid4vci-interoperability-checks', - 'request-evidence-from-an-application', - 'return-a-governed-value', - 'assert-a-role-bound-relationship', - 'refuse-unsafe-evidence-requests', - 'verify-an-assertion-as-a-consumer', - 'control-who-can-request-evidence', - 'issue-fhir-evidence-as-vcs', - 'connect-a-sqlite-extract', - ]) { - assert.match(output, new RegExp(`${slug}: \\d+ sh fences, \\d+ executed`, 'u')); - } - assert.match(output, /Checked 11 tutorials\./u); -}); - -// The unexecuted surface is information a reviewer needs, not a rule: the -// install one-liner and the port-conflict recovery block are documented and -// never replayed, so the gate says so rather than pinning their text. -test('the gate names the sh fences it did not execute', async () => { - const { code, output } = await runGate({}, ['--dry-run', '--only', 'first-evidence-assertion']); - assert.equal(code, 0, output); - assert.match(output, /not executed: fence 01 under "Install Evidence Gateway"/u); - assert.match(output, /not executed: fence \d+ under "If local ports are already in use"/u); -}); - -test('--only accepts the current first Evidence tutorial', async () => { - const { code, output } = await runGate({}, [ - '--dry-run', - '--only', - 'first-evidence-assertion', - ]); - assert.equal(code, 0, output); - assert.match(output, /Checked 1 tutorial\./u); - const source = await readFile(gate, 'utf8'); - const branch = source.match(/\n\tfirst-evidence-assertion\)[\s\S]*?\n\t\t;;/u)?.[0]; - assert.ok(branch, 'the first Evidence replay spec must exist'); - assert.match(branch, /stop-background/u); - assert.match(branch, /run:Preview a synthetic source\|3/u); - assert.match(branch, /run:Try the SD-JWT VC serialization/u); -}); - -test('--only accepts the role-bound relationship follow-up', async () => { - const { code, output } = await runGate({}, [ - '--dry-run', - '--only', - 'assert-a-role-bound-relationship', - ]); - assert.equal(code, 0, output); - assert.match(output, /Checked 1 tutorial\./u); -}); - -test('--only accepts the deterministic FHIR tutorial replay', async () => { - const { code, output } = await runGate({}, [ - '--dry-run', - '--only', - 'issue-fhir-evidence-as-vcs', - ]); - assert.equal(code, 0, output); - assert.match(output, /issue-fhir-evidence-as-vcs: 10 sh fences, 10 executed/u); - assert.match(output, /Checked 1 tutorial\./u); -}); - -test('both FHIR tutorial clients bypass ambient proxies', async () => { - const source = await readFile(fhirTutorial, 'utf8'); - const proxyFreeOpeners = source.match( - /build_opener\(ProxyHandler\(\{\}\), NoRedirect\)/gu, - ); - assert.equal(proxyFreeOpeners?.length, 2); -}); - -test('the FHIR tutorial test origin refuses a remote endpoint', async () => { - const root = await mkdtemp(join(tmpdir(), 'fhir-tutorial-origin-test-')); - const discovery = join(root, 'discover-fhir-records.py'); - try { - await execFileAsync('bash', [ - fenceHelper, - 'write-fence', - fhirTutorial, - 'Select live synthetic records', - 'python', - '1', - discovery, - ]); - await assert.rejects( - execFileAsync('python3', [discovery], { - env: { - ...process.env, - FHIR_TUTORIAL_TEST_BASE_URL: 'https://example.com', - }, - }), - (error) => { - assert.match(error.stderr, /test origin must be numeric loopback HTTP/u); - return true; - }, - ); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('the FHIR replay tracks the read-through adapter for cleanup', async () => { - const source = await readFile(gate, 'utf8'); - const branch = source.match( - /\n\tissue-fhir-evidence-as-vcs\)[\s\S]*?\n\t\t;;/u, - )?.[0]; - assert.ok(branch, 'the FHIR replay spec must exist'); - assert.match( - branch, - /"run:Run a live FHIR read-through adapter"\s+"track-pid:fhir-read-through\.pid"/u, - ); - assert.match(source, /track-pid:\*\) emit_track_pid_step/u); - assert.match(source, /BACKGROUND_PIDS\+=\("\$tracked_pid"\)/u); -}); - -// Every follow-up below begins from the project first-evidence-assertion -// builds. A full run gets that from the registration order, so a --only that -// skipped it would fail on the reader directory rather than on the tutorial, -// and only for the person running one slug by hand. -for (const slug of [ - 'request-evidence-as-sd-jwt-vc', - 'request-evidence-from-an-application', - 'return-a-governed-value', - 'refuse-unsafe-evidence-requests', - 'verify-an-assertion-as-a-consumer', - 'control-who-can-request-evidence', -]) { - test(`--only runs the starter project before ${slug}`, async () => { - const { code, output } = await runGate({}, ['--dry-run', '--only', slug]); - assert.equal(code, 0, output); - const prerequisite = output.indexOf('first-evidence-assertion:'); - const followUp = output.indexOf(`${slug}:`); - assert.notEqual(prerequisite, -1, output); - assert.ok(followUp > prerequisite, output); - assert.match(output, /Checked 2 tutorials\./u); - }); -} - -// The application tutorial is the only registered replay that reaches the -// Evidence client SDK, and it reaches it through the Python binding. Losing -// either the registration or the substituted build would leave that path -// unproven while the gate still reported PASS. -test('the application tutorial replays the Python client from this checkout', async () => { - const source = await readFile(gate, 'utf8'); - assert.match(source, /^\trequest-evidence-from-an-application$/mu); - const branch = source.match( - /\n\trequest-evidence-from-an-application\)[\s\S]*?\n\t\t;;/u, - )?.[0]; - assert.ok(branch, 'the application replay spec must exist'); - assert.match(branch, /"python-client"/u); - assert.match(branch, /person-123 is_adult=True/u); - assert.match(branch, /person-456 is_adult=False/u); -}); - -// The tutorial installs the one maintained client distribution, so the gate -// has to import that same distribution: a per-product extension module would -// prove a package no reader can install, and would leave the namespaces the -// tutorial imports unexercised. -test('the application replay imports the assembled client package', async () => { - const source = await readFile(gate, 'utf8'); - const step = source.match(/\nemit_python_client_step\(\) \{\n[\s\S]*?\n\}\n/u)?.[0]; - assert.ok(step, 'the client step must exist'); - // The replay userland carries unzip and no installer, so the package is - // unpacked onto the import path rather than installed. - assert.match(step, /unzip/u); - assert.match(step, /PYTHONPATH/u); - assert.match(source, /REGISTRY_CLIENT_PY_WHEEL/u); - assert.doesNotMatch(source, /EVIDENCE_CLIENT_PY_LIB/u); - assert.doesNotMatch(source, /registry_evidence_client\.so/u); -}); - -// The gate cannot assemble the package itself: assembling needs a build -// toolchain the replay userland does not carry. Refusing early, by name, is -// what keeps that from surfacing as an import error twenty steps in. -test('the gate refuses a client package it cannot use', async () => { - const source = await readFile(gate, 'utf8'); - const prepare = source.match(/\nprepare_python_client\(\) \{\n[\s\S]*?\n\}\n/u)?.[0]; - assert.ok(prepare, 'the client preparation must exist'); - assert.match(prepare, /!= \/\*/u, 'the path must be required to be absolute'); - assert.match(prepare, /must be absolute/u); - assert.match(prepare, /-f "\$REGISTRY_CLIENT_PY_WHEEL"/u); - assert.match(prepare, /assemble-registry-client-packages/u); -}); - -test('the caller-access replay expects the privacy-safe refusal audit line', async () => { - const source = await readFile(gate, 'utf8'); - const branch = source.match( - /\n\tcontrol-who-can-request-evidence\)[\s\S]*?\n\t\t;;/u, - )?.[0]; - assert.ok(branch, 'the caller-access replay spec must exist'); - assert.match( - branch, - /"run:Revoke an application\|1"\s+"run:Revoke an application\|2"\s+"run-fails:Revoke an application\|3"\s+"run:Stop the final generation"\s+"run:Clean up"/u, - ); - assert.match(branch, /"ACCESS REFUSED requester="/u); - assert.match(branch, /"reason=not_authorized"/u); - assert.doesNotMatch(branch, /ACCESS AUTHORIZED age-bracket/u); -}); - -// EVIDENCE_TUTORIALS and EXCLUDED_EVIDENCE_TUTORIALS between them must -// account for every page under the tutorials directory, so a new page can -// never ship unreplayed and unexplained. Read the lists from the gate -// itself rather than restating them, so this test tracks the gate instead -// of drifting from it. -function extractBashArray(source, name) { - const match = source.match(new RegExp(`\\n${name}=\\(([\\s\\S]*?)\\n\\)`, 'u')); - assert.ok(match, `${name} array must exist in the gate`); - return match[1] - .split('\n') - .map((line) => line.split('#')[0].trim()) - .filter(Boolean); -} - -test('the tutorial coverage check fails on an unregistered page', async () => { - const source = await readFile(gate, 'utf8'); - const excluded = extractBashArray(source, 'EXCLUDED_EVIDENCE_TUTORIALS'); - const root = await mkdtemp(join(tmpdir(), 'evidence-tutorial-coverage-test-')); - try { - // Stub every already-excluded page so only the deliberately unregistered - // page below can trip the check. - for (const slug of excluded) { - await writeFile(join(root, `${slug}.mdx`), '---\ntitle: stub\n---\n'); - } - await writeFile(join(root, 'orphan-tutorial.mdx'), '---\ntitle: stub\n---\n'); - const { code, output } = await runGate({ EVIDENCE_TUTORIAL_DOCS_ROOT: root }); - assert.notEqual(code, 0, 'an unregistered tutorial page must fail the gate'); - assert.match(output, /tutorial coverage gap/u); - assert.match(output, /orphan-tutorial\.mdx/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -// --------------------------------------------------------------------------- -// Heading addressing -// --------------------------------------------------------------------------- - -// Build a tutorials directory the gate will accept: every excluded page must -// exist, and the one registered page under test is the real one, edited. -async function tutorialFixtureRoot(edit) { - const source = await readFile(gate, 'utf8'); - const excluded = extractBashArray(source, 'EXCLUDED_EVIDENCE_TUTORIALS'); - const root = await mkdtemp(join(tmpdir(), 'evidence-tutorial-heading-test-')); - for (const slug of excluded) { - await writeFile(join(root, `${slug}.mdx`), '---\ntitle: stub\n---\n'); - } - const page = await readFile( - resolve(scriptDir, '../src/content/docs/tutorials/first-evidence-assertion.mdx'), - 'utf8', - ); - await writeFile(join(root, 'first-evidence-assertion.mdx'), edit(page)); - return root; -} - -// The point of heading addressing. A writer who adds a command block under a -// heading the journey already runs must not have to touch the gate, and the -// added block must be replayed rather than silently skipped. -test('a command block added under a replayed heading needs no gate change', async () => { - const root = await tutorialFixtureRoot((page) => - page.replace( - '\n## Verify before reading\n', - '\n```sh\nevidencectl request list\n```\n\n## Verify before reading\n', - ), - ); - try { - const before = await runGate({}, ['--dry-run', '--only', 'first-evidence-assertion']); - assert.equal(before.code, 0, before.output); - const baseline = before.output.match( - /first-evidence-assertion: (\d+) sh fences, (\d+) executed/u, - ); - assert.ok(baseline, before.output); - - const { code, output } = await runGate( - { EVIDENCE_TUTORIAL_DOCS_ROOT: root }, - ['--dry-run', '--only', 'first-evidence-assertion'], - ); - assert.equal(code, 0, output); - const added = output.match(/first-evidence-assertion: (\d+) sh fences, (\d+) executed/u); - assert.ok(added, output); - assert.equal(Number(added[1]), Number(baseline[1]) + 1); - assert.equal(Number(added[2]), Number(baseline[2]) + 1); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -// The trade heading addressing makes: a renamed heading is a structural edit -// to the journey, so it fails, by name, before any command runs. -test('a renamed heading fails the gate by name', async () => { - const root = await tutorialFixtureRoot((page) => - page.replace('\n## Request an assertion\n', '\n## Ask for an assertion\n'), - ); - try { - const { code, output } = await runGate( - { EVIDENCE_TUTORIAL_DOCS_ROOT: root }, - ['--dry-run', '--only', 'first-evidence-assertion'], - ); - assert.notEqual(code, 0, 'a renamed heading must fail the gate'); - assert.match(output, /no sh fence answers to "Request an assertion"/u); - // The message has to be actionable: it names the headings the page does - // carry, so the fix is reading the list rather than the script. - assert.match(output, /Ask for an assertion/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -// A heading holding more than one sh fence cannot answer a step that runs -// exactly one command, so the gate says which suffix is missing. -test('a one-fence step under a multi-fence heading names the missing occurrence', async () => { - const source = await readFile(gate, 'utf8'); - const root = await mkdtemp(join(tmpdir(), 'evidence-occurrence-test-')); - try { - await writeFile(join(root, 'index.tsv'), '01\t1\tRun it\n02\t2\tRun it\n'); - const harness = join(root, 'resolve.sh'); - await writeFile( - harness, - [ - '#!/usr/bin/env bash', - 'set -euo pipefail', - await liftFunction(source, 'resolve_fences'), - await liftFunction(source, 'resolve_one_fence'), - 'resolve_one_fence tutorial "Run it" "$1" background', - '', - ].join('\n'), - ); - const { code, output } = await runShell(`bash ${harness} ${root}`); - assert.notEqual(code, 0, 'an ambiguous one-fence step must fail'); - assert.match(output, /names 2/u); - assert.match(output, /\|/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -// --------------------------------------------------------------------------- -// Behaviour assertions -// --------------------------------------------------------------------------- - -async function runAssertTranscript(asserts, transcript) { - const source = await readFile(gate, 'utf8'); - const root = await mkdtemp(join(tmpdir(), 'evidence-asserts-test-')); - const log = join(root, 'run.log'); - await writeFile(log, transcript); - const harness = join(root, 'assert.sh'); - await writeFile( - harness, - [ - '#!/usr/bin/env bash', - 'set -euo pipefail', - await liftFunction(source, 'assert_transcript'), - `SPEC_ASSERTS=(${asserts.map((entry) => `'${entry}'`).join(' ')})`, - `assert_transcript tutorial '${log}'`, - '', - ].join('\n'), - ); - try { - return await runShell(`bash ${harness}`); - } finally { - await rm(root, { recursive: true, force: true }); - } -} - -test('a retained behaviour assertion missing from the transcript fails', async () => { - const { code, output } = await runAssertTranscript( - ['VERIFIED', 'DISCLOSURE RELEASED is_adult'], - '==> fence 12\nVERIFIED\n==> fence 19\nACCESS AUTHORIZED adult-status age-check requester=x\n', - ); - assert.notEqual(code, 0, 'a missing behaviour must fail the gate'); - assert.match(output, /DISCLOSURE RELEASED is_adult/u); -}); - -test('a transcript showing every retained behaviour passes', async () => { - const { code, output } = await runAssertTranscript( - ['VERIFIED', 'DISCLOSURE RELEASED is_adult'], - 'VERIFIED\nDISCLOSURE RELEASED is_adult\n', - ); - assert.equal(code, 0, output); -}); - -// Every retained assertion has to earn its place by regressing silently. The -// two the gate must never lose are the refusal that actually refused and the -// tamper that was actually caught, and both must be words a tool printed. A -// page that echoes its own verdict asserts nothing: the echo survives the -// regression it was supposed to catch and leaves the transcript quietly clean. -test('the refusal tutorial still asserts the refusal and the tamper', async () => { - const source = await readFile(gate, 'utf8'); - const branch = source.match( - /\n\trefuse-unsafe-evidence-requests\)[\s\S]*?\n\t\t;;/u, - )?.[0]; - assert.ok(branch, 'the refusal replay spec must exist'); - assert.match(branch, /"HTTP 403"/u); - assert.match(branch, /"evidencectl: Evidence response verification failed"/u); - // Startup chatter a successful exit already proves does not belong here. - assert.doesNotMatch(branch, /Evidence ready at/u); - assert.doesNotMatch(branch, /Prepared request:/u); - assert.doesNotMatch(branch, /Local Evidence stopped/u); -}); - -// This gate proves the documented commands still run. It does not police what -// a page says, and the two arrays below are how it used to: one pinned how -// many command blocks a page held, the other pinned strings the page had to -// keep. Both made ordinary prose edits fail CI, and neither verified anything -// replay does not already verify. Reintroducing either is the regression this -// test exists to catch. -test('the gate pins neither fence counts nor page strings', async () => { - const source = await readFile(gate, 'utf8'); - assert.doesNotMatch(source, /SPEC_FENCES/u); - assert.doesNotMatch(source, /SPEC_LITERALS/u); -}); - -test('--only refuses a slug that is not registered', async () => { - const { code, output } = await runGate({}, ['--dry-run', '--only', 'no-such-tutorial']); - assert.notEqual(code, 0, 'an unregistered slug must fail the gate'); - assert.match(output, /not a registered Evidence tutorial/u); -}); - -test('--only refuses an unpublished legacy tutorial', async () => { - const { code, output } = await runGate({}, [ - '--dry-run', - '--only', - 'serve-assertions-over-http', - ]); - assert.notEqual(code, 0, 'an unpublished tutorial must not be registered'); - assert.match(output, /not a registered Evidence tutorial/u); -}); - -// The replay runs inside a clean Debian userland holding a shell, coreutils -// and the toolset under test. An interpreter the container does not carry -// fails mid-journey, where the transcript makes it look like a tutorial -// defect, so the gate and everything it emits stay on that floor. -test('the gate depends on no interpreter beyond the replay userland', async () => { - const source = await readFile(gate, 'utf8'); - const offenders = source - .split('\n') - .map((line, index) => [index + 1, line]) - // `python-client` is a step name and `python-module` is the directory the - // application tutorial imports from. Both are data the gate writes or - // matches, never an interpreter it runs, so they are removed before the - // line is judged rather than exempting whole lines that carry them. - .map(([number, line]) => [number, line.replaceAll(/python-(?:client|module)/gu, '')]) - .filter(([, line]) => /\b(?:node|npm|npx|python3?|ruby|perl)\b/u.test(line)) - // A save step names the Markdown fence language as data. It extracts that - // fence with the shell helper and does not execute the named interpreter. - .filter(([, line]) => !/^\s*"save:[^"]+\|[^|]+\|\d+\|[^"]+",?$/u.test(line)); - assert.deepEqual(offenders, [], 'the gate must not reach for an interpreter'); -}); - -async function replayCargoTarget(slug) { - const source = await readFile(gate, 'utf8'); - const runner = source.match(/\nrun_journey_script\(\) \{\n[\s\S]*?\n\}\n/u)?.[0]; - assert.ok(runner, 'the journey runner must exist'); - const root = await mkdtemp(join(tmpdir(), 'evidence-cargo-target-test-')); - const journey = join(root, 'journey.sh'); - const harness = join(root, 'run.sh'); - await writeFile(journey, 'printf "%s\\n" "${CARGO_TARGET_DIR-unset}"\n'); - await writeFile( - harness, - [ - '#!/usr/bin/env bash', - 'set -euo pipefail', - runner, - 'SHIM_DIR="$1"', - 'TARGET_DIR="$2"', - 'run_journey_script "$3" "$1" "$4"', - '', - ].join('\n'), - ); - try { - const expectedTarget = join(root, 'oid4vci-target'); - const { stdout } = await execFileAsync( - 'bash', - [harness, root, expectedTarget, slug, journey], - { - env: { ...process.env, CARGO_TARGET_DIR: join(root, 'inherited-target') }, - }, - ); - return { output: stdout.trim(), expectedTarget }; - } finally { - await rm(root, { recursive: true, force: true }); - } -} - -test('the OID4VCI replay receives the gate Cargo target directory', async () => { - const { output, expectedTarget } = await replayCargoTarget( - 'run-oid4vci-interoperability-checks', - ); - assert.equal(output, expectedTarget); -}); - -test('other tutorial replays do not receive a Cargo target directory', async () => { - const { output } = await replayCargoTarget('first-evidence-assertion'); - assert.equal(output, 'unset'); -}); - -async function runFence(args) { - try { - const { stdout, stderr } = await execFileAsync('bash', [fenceHelper, ...args]); - return { code: 0, output: `${stdout}${stderr}` }; - } catch (error) { - return { code: error.code ?? 1, output: `${error.stdout}${error.stderr}` }; - } -} - -const fenceFixture = [ - '---', - 'title: A tutorial', - '---', - '', - '## Add a narrower selector profile', - '', - 'Before:', - '', - '```yaml', - '', - 'selectors:', - ' - kind: broad', - '', - '```', - '', - 'After:', - '', - '```yaml', - 'selectors:', - ' - kind: narrow', - '```', - '', - '## Run it', - '', - '```sh', - 'evidencectl check', - '```', - '', -].join('\n'); - -async function fenceScratch() { - const root = await mkdtemp(join(tmpdir(), 'evidence-fence-test-')); - await writeFile(join(root, 'tutorial.mdx'), fenceFixture); - return root; -} - -// Lift one named function out of the gate. Sourcing the gate would run it, so -// the tests below exercise the shipped text of the function instead of -// restating it. -async function liftFunction(source, name) { - const lifted = source.match( - new RegExp(`\\n${name}\\(\\) \\{\\n[\\s\\S]*?\\n\\}\\n`, 'u'), - )?.[0]; - assert.ok(lifted, `${name} must exist in the gate`); - return lifted; -} - -// Run the gate's own run-fails emitter over one fence addressed by heading, -// and return the journey lines it emits. -async function emitRunFailsStep(fenceBody) { - const source = await readFile(gate, 'utf8'); - const emitter = [ - await liftFunction(source, 'resolve_fences'), - await liftFunction(source, 'resolve_one_fence'), - await liftFunction(source, 'emit_run_fails_step'), - ].join('\n'); - const root = await mkdtemp(join(tmpdir(), 'evidence-refusal-test-')); - await writeFile(join(root, 'fence-09.sh'), fenceBody); - await writeFile(join(root, 'index.tsv'), '09\t1\tRefuse before reading\n'); - const harness = join(root, 'emit.sh'); - await writeFile( - harness, - [ - '#!/usr/bin/env bash', - 'set -euo pipefail', - emitter, - 'emit_run_fails_step tutorial "Refuse before reading" "$1"', - '', - ].join('\n'), - ); - const { stdout } = await execFileAsync('bash', [harness, root]); - return { root, journey: `set -euo pipefail\n${stdout}` }; -} - -// A refusal fence that prints after the command that refuses is the shape the -// pages actually carry: the reader sees the error, then the state it left -// behind. Bash suppresses errexit for everything inside an `if` condition, -// subshells included, so an emitter that tested the fence there would run the -// trailing line, read the whole fence as a success, and report drift on a -// tutorial that is doing exactly what it documents. -test('a documented refusal is accepted even when the fence prints after it', async () => { - const { root, journey } = await emitRunFailsStep( - 'false\nprintf "kept going\\n"\n', - ); - try { - const { code, output } = await runShell(journey); - assert.equal(code, 0, output); - assert.doesNotMatch(output, /kept going/u); - assert.doesNotMatch(output, /tutorial drift/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('a refusal fence that starts succeeding is reported as drift', async () => { - const { root, journey } = await emitRunFailsStep('true\n'); - try { - const { code, output } = await runShell(journey); - assert.notEqual(code, 0, 'a fence that no longer refuses must fail the gate'); - assert.match(output, /tutorial drift/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -// The steps after a documented refusal still run under the journey's errexit, -// so a later failure ends the journey where it happened instead of being -// carried past. -test('errexit is back in force after a documented refusal', async () => { - const { root, journey } = await emitRunFailsStep('false\n'); - try { - const { code, output } = await runShell(`${journey}\nfalse\nprintf "past it\\n"\n`); - assert.notEqual(code, 0, 'the journey must stop at the failure after the refusal'); - assert.doesNotMatch(output, /past it/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('write-fence extracts one fence by heading, language and occurrence', async () => { - const root = await fenceScratch(); - try { - const out = join(root, 'before.yaml'); - const { code, output } = await runFence([ - 'write-fence', - join(root, 'tutorial.mdx'), - 'Add a narrower selector profile', - 'yaml', - '1', - out, - ]); - assert.equal(code, 0, output); - // Blank lines at the edges of a fence are presentation, so they are - // trimmed exactly as the published fence renders. - assert.equal(await readFile(out, 'utf8'), 'selectors:\n - kind: broad\n'); - - const second = join(root, 'after.yaml'); - assert.equal((await runFence([ - 'write-fence', - join(root, 'tutorial.mdx'), - 'Add a narrower selector profile', - 'yaml', - '2', - second, - ])).code, 0); - assert.equal(await readFile(second, 'utf8'), 'selectors:\n - kind: narrow\n'); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('write-fence counts occurrences per heading and language', async () => { - const root = await fenceScratch(); - try { - const out = join(root, 'sh.txt'); - // The sh fence under a later heading is that heading's first, not the - // document's third. - const { code, output } = await runFence([ - 'write-fence', - join(root, 'tutorial.mdx'), - 'Run it', - 'sh', - '1', - out, - ]); - assert.equal(code, 0, output); - assert.equal(await readFile(out, 'utf8'), 'evidencectl check\n'); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('write-fence names the fence it could not find', async () => { - const root = await fenceScratch(); - try { - const { code, output } = await runFence([ - 'write-fence', - join(root, 'tutorial.mdx'), - 'Add a narrower selector profile', - 'yaml', - '3', - join(root, 'missing.yaml'), - ]); - assert.notEqual(code, 0, 'a missing fence must fail'); - assert.match(output, /missing yaml fence 3 under "Add a narrower selector profile"/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('replace-block applies a documented pair to the reader file', async () => { - const root = await fenceScratch(); - try { - const target = join(root, 'evidence.yaml'); - await writeFile(target, 'version: 1\nselectors:\n - kind: broad\ntrailer: keep\n'); - await writeFile(join(root, 'b'), 'selectors:\n - kind: broad\n'); - await writeFile(join(root, 'a'), 'selectors:\n - kind: narrow\n'); - const { code, output } = await runFence([ - 'replace-block', - target, - join(root, 'b'), - join(root, 'a'), - ]); - assert.equal(code, 0, output); - assert.equal( - await readFile(target, 'utf8'), - 'version: 1\nselectors:\n - kind: narrow\ntrailer: keep\n', - ); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('replace-block refuses a block that is not in the target exactly once', async () => { - const root = await fenceScratch(); - try { - const target = join(root, 'evidence.yaml'); - await writeFile(join(root, 'b'), 'kind: broad\n'); - await writeFile(join(root, 'a'), 'kind: narrow\n'); - - await writeFile(target, 'kind: broad\nkind: broad\n'); - const twice = await runFence(['replace-block', target, join(root, 'b'), join(root, 'a')]); - assert.notEqual(twice.code, 0, 'an ambiguous edit must fail'); - assert.match(twice.output, /found 2/u); - - await writeFile(target, 'kind: other\n'); - const never = await runFence(['replace-block', target, join(root, 'b'), join(root, 'a')]); - assert.notEqual(never.code, 0, 'an edit with nothing to change must fail'); - assert.match(never.output, /found 0/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('replace-block refuses a pair that changes nothing', async () => { - const root = await fenceScratch(); - try { - const target = join(root, 'evidence.yaml'); - await writeFile(target, 'kind: broad\n'); - await writeFile(join(root, 'b'), 'kind: broad\n'); - await writeFile(join(root, 'a'), 'kind: broad\n'); - const { code, output } = await runFence([ - 'replace-block', - target, - join(root, 'b'), - join(root, 'a'), - ]); - assert.notEqual(code, 0, 'a pair that changes nothing is a spec error'); - assert.match(output, /must change the target/u); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); diff --git a/docs/site/scripts/run-tutorial.mjs b/docs/site/scripts/run-tutorial.mjs index f068f753b0..81815c64bf 100644 --- a/docs/site/scripts/run-tutorial.mjs +++ b/docs/site/scripts/run-tutorial.mjs @@ -1,8 +1,8 @@ #!/usr/bin/env node // Replay a tutorial page the way a reader follows it. // -// node scripts/run-tutorial.mjs [--dry-run] [--toolset breg|casework|none] ... -// node scripts/run-tutorial.mjs [--dry-run] --gate breg|casework +// node scripts/run-tutorial.mjs [--dry-run] [--toolset breg|casework|evidence|none] ... +// node scripts/run-tutorial.mjs [--dry-run] --gate breg|casework|evidence // // The page is the specification (see tutorial-runner/page.mjs): its sh fences // run in document order in one bash shell, from an empty reader directory @@ -21,8 +21,15 @@ // tutorial_test.checkout, the reader directory starts as a copy of this // checkout instead (tutorial-runner/checkout.mjs). // +// A test-file block writes its file from the shell's current directory. A +// test-background fence runs beside the journey until its ready URL answers, +// and stays running until the next background fence starts or its page ends +// (tutorial-runner/background.mjs). test-cwd moves the shell to a directory +// under the reader directory before its fence runs. +// // The toolset puts the product binaries under test on PATH and stops any -// service the journey left running, whether it passed or failed. +// service the journey left running, whether it passed or failed. Any process +// the journey started and left behind is stopped when it ends. // // With --gate, the pages come from their own frontmatter instead of the // command line (tutorial-runner/gate.mjs): every page under start/ or @@ -40,6 +47,7 @@ import { tmpdir } from 'node:os'; import { basename, dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { stopGroup } from './tutorial-runner/background.mjs'; import { checkExcerpt } from './tutorial-runner/excerpt.mjs'; import { checkExpectation } from './tutorial-runner/expect.mjs'; import { copyCheckout } from './tutorial-runner/checkout.mjs'; @@ -48,9 +56,10 @@ import { readJourney } from './tutorial-runner/page.mjs'; import { TOOLSETS, ToolsetError } from './tutorial-runner/toolsets.mjs'; const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); -const USAGE = 'usage: run-tutorial.mjs [--dry-run] [--toolset breg|casework|none] ...\n run-tutorial.mjs [--dry-run] --gate breg|casework'; +const USAGE = 'usage: run-tutorial.mjs [--dry-run] [--toolset breg|casework|evidence|none] ...\n run-tutorial.mjs [--dry-run] --gate breg|casework|evidence'; const DOCS_ROOT = process.env.TUTORIAL_DOCS_ROOT ?? resolve(dirname(fileURLToPath(import.meta.url)), '../src/content/docs'); const APPLY_EDIT = join(dirname(fileURLToPath(import.meta.url)), 'tutorial-runner/apply-edit.mjs'); +const BACKGROUND = join(dirname(fileURLToPath(import.meta.url)), 'tutorial-runner/background.mjs'); function usageError(message) { console.error(`${message}\n${USAGE}`); @@ -78,7 +87,7 @@ function parseArgs(argv) { const at = (step) => `${step.page ? `${step.page} ` : ''}line ${step.line}`; const where = (step) => `${at(step)}${step.heading ? ` (${step.heading})` : ''}`; -const BLOCK_NAMES = { edit: 'the edit', excerpt: 'the excerpt' }; +const BLOCK_NAMES = { edit: 'the edit', excerpt: 'the excerpt', file: 'the file' }; const blockName = (step) => BLOCK_NAMES[step.kind] ?? 'the sh fence'; const quote = (text) => `'${text.replaceAll("'", "'\\''")}'`; const outName = (index) => `${String(index).padStart(3, '0')}.out`; @@ -86,10 +95,15 @@ const outName = (index) => `${String(index).padStart(3, '0')}.out`; function printPlan(steps, checkout) { if (checkout) console.log('start in a copy of the checkout'); for (const step of steps) { - const exit = step.exit === undefined ? '' : ` (expects exit ${step.exit})`; - if (step.kind === 'run') console.log(`run ${where(step)}: ${step.code.split('\n')[0]}${exit}`); + const notes = []; + if (step.cwd) notes.push(`in ${step.cwd}`); + if (step.exit !== undefined) notes.push(`expects exit ${step.exit}`); + if (step.background) notes.push(`in the background until ${step.background} answers`); + const note = notes.length > 0 ? ` (${notes.join(', ')})` : ''; + if (step.kind === 'run') console.log(`run ${where(step)}: ${step.code.split('\n')[0]}${note}`); else if (step.kind === 'skip') console.log(`skip ${where(step)}: ${step.reason}`); else if (step.kind === 'edit') console.log(`edit ${where(step)}: ${step.path}`); + else if (step.kind === 'file') console.log(`file ${where(step)}: ${step.path}`); else if (step.kind === 'excerpt' && step.path) console.log(`excerpt ${at(step)}: ${step.path}`); else if (step.kind === 'excerpt') console.log(`excerpt ${at(step)}: checks line ${steps[step.runIndex].line}`); else console.log(`expect ${at(step)}: checks line ${steps[step.runIndex].line}`); @@ -110,10 +124,17 @@ function printPlan(steps, checkout) { // reaches its end leaves `page-N.done`; one whose fence ran `exit` does not, // and the journey stops there. // +// A background fence starts with job control on, which gives it a process +// group of its own that can be stopped whole. The file `background` holds the +// group and output file of the one running now; `backgrounds` lists every +// group started, for the cleanup after a journey that stopped early. +// // The script names every harness file by its literal path and sets no shell // variable, so a page's own variables neither see nor clobber the harness. -async function journeyScript(pages, outDir) { +async function journeyScript(pages, outDir, readerDir) { const file = (name) => quote(join(outDir, name)); + const node = quote(process.execPath); + const stopBackground = `${node} ${quote(BACKGROUND)} stop ${file('background')}`; const lines = ['set -euo pipefail', "trap 'exit 130' HUP INT TERM"]; let index = 0; for (const [pageIndex, steps] of pages.entries()) { @@ -121,6 +142,8 @@ async function journeyScript(pages, outDir) { for (const step of steps) { const out = file(outName(index)); lines.push(`printf '%s\n' ${index} >${file('current')}`); + const cd = step.cwd ? `cd -- ${quote(join(readerDir, step.cwd))}` : undefined; + if (cd && !step.background) lines.push(cd); if (step.kind === 'skip') { lines.push(`printf '%s\\n' ${quote(`skip ${where(step)}: ${step.reason}`)}`); } else if (step.kind === 'edit') { @@ -133,6 +156,23 @@ async function journeyScript(pages, outDir) { lines.push(`printf '\\n%s\\n' ${quote(`==> ${where(step)}`)}`); lines.push(`cat -- ${quote(step.path)} >${out} 2>&1 ${where(step)}`)}`); + lines.push(`cp -- ${quote(staged)} ${quote(step.path)} >${out} 2>&1 ${where(step)} (background)`)}`); + lines.push(`: >${out}`, 'set -m'); + // The other terminal's directory is its own, so a test-cwd here leaves + // the reader's shell where it stands. + lines.push(`{\n${cd ? `${cd}\n` : ''}${step.code}\n} >>${out} 2>&1 ${file('background')}`); + lines.push(`printf '%s\\n' "$!" >>${file('backgrounds')}`, 'set +m'); + lines.push(`${node} ${quote(BACKGROUND)} ready ${quote(step.background)} ${file('background')} >>${out} 2>&1`); + lines.push(`printf 'ready: %s\\n' ${quote(step.background)}`); } else if (step.kind === 'run' && step.exit === undefined) { lines.push(`printf '\\n%s\\n' ${quote(`==> ${where(step)}`)}`); lines.push(`{\n${step.code}\n} >${out} 2>&1 ${done}`, ')', `[[ -e ${done} ]] || exit 0`); + lines.push(stopBackground, `: >${done}`, ')', `[[ -e ${done} ]] || exit 0`); } lines.push(`printf "\\n" >${file('complete')}`); return `${lines.join('\n')}\n`; @@ -163,8 +203,8 @@ async function journeyScript(pages, outDir) { // command a fence is running and not only the shell waiting for it, which // would run its trap only once that command ended. onSpawn receives a // function that sends a signal to the whole group. -function runScript(scriptPath, readerDir, binDir, onSpawn) { - const env = { ...process.env, PATH: `${binDir}:${process.env.PATH}` }; +function runScript(scriptPath, readerDir, binDir, toolsetEnv, onSpawn) { + const env = { ...process.env, ...toolsetEnv, PATH: `${binDir}:${process.env.PATH}` }; // A reader has no CARGO_TARGET_DIR pointing into this checkout. delete env.CARGO_TARGET_DIR; return new Promise((resolvePromise, reject) => { @@ -173,7 +213,10 @@ function runScript(scriptPath, readerDir, binDir, onSpawn) { if (child.exitCode === null && child.signalCode === null) process.kill(-child.pid, signal); }); child.on('error', reject); - child.on('close', (code, signal) => resolvePromise(signal ? 130 : code)); + child.on('close', (code, signal) => { + // Whatever the journey started and left behind is stopped with it. + stopGroup(child.pid).then(() => resolvePromise(signal ? 130 : code), reject); + }); }); } @@ -233,6 +276,18 @@ async function checkBlocks(steps, outDir) { return failures; } +// Stop every background fence a journey that ended early left running. +async function stopBackgrounds(outDir) { + let text; + try { + text = await readFile(join(outDir, 'backgrounds'), 'utf8'); + } catch (error) { + if (error.code === 'ENOENT') return; + throw error; + } + for (const group of text.trim().split('\n').filter(Boolean)) await stopGroup(Number(group)); +} + async function replay(pages, toolset, checkout) { const steps = pages.flat(); const workRoot = await realpath(await mkdtemp(join(tmpdir(), 'tutorial-run.'))); @@ -253,12 +308,12 @@ async function replay(pages, toolset, checkout) { let status = 1; let prepared = false; try { - await toolset.prepare({ repoRoot: REPO_ROOT, binDir }); + const toolsetEnv = await toolset.prepare({ repoRoot: REPO_ROOT, binDir, workRoot }); prepared = true; if (checkout) await copyCheckout(REPO_ROOT, readerDir); const scriptPath = join(workRoot, 'journey.sh'); - await writeFile(scriptPath, await journeyScript(pages, outDir)); - const code = await runScript(scriptPath, readerDir, binDir, (send) => { + await writeFile(scriptPath, await journeyScript(pages, outDir, readerDir)); + const code = await runScript(scriptPath, readerDir, binDir, toolsetEnv, (send) => { signalJourney = send; }); if (interrupted || code === 130) { @@ -288,6 +343,7 @@ async function replay(pages, toolset, checkout) { } finally { for (const signal of signals) process.off(signal, onSignal); try { + await stopBackgrounds(outDir); unlock(workRoot); const stoppedAll = prepared ? await toolset.teardown({ readerDir, binDir }) : true; if (stoppedAll) { diff --git a/docs/site/scripts/tutorial-runner/background.mjs b/docs/site/scripts/tutorial-runner/background.mjs new file mode 100644 index 0000000000..71bd5f9d7f --- /dev/null +++ b/docs/site/scripts/tutorial-runner/background.mjs @@ -0,0 +1,114 @@ +#!/usr/bin/env node +// The command a page leaves running in another terminal (test-background). +// +// The journey script starts a background fence in its own process group and +// records ` ` in a state file. This helper then waits for +// the fence's ready URL to answer with success, or stops the group the state +// file names: +// +// background.mjs ready wait until the URL answers 2xx +// background.mjs stop stop the group, show its output +// +// A fence whose group ends before its URL answers, or that does not answer +// within READY_TIMEOUT_MS, fails the wait. A group that has already ended by +// the time stop is called fails too: a background command promises to keep +// running until it is stopped, so one that exited on its own is a failure +// even though it answered while it was up. + +import { readFile, writeFile } from 'node:fs/promises'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { pathToFileURL } from 'node:url'; + +const READY_TIMEOUT_MS = 60_000; +const STOP_GRACE_MS = 10_000; + +export function groupAlive(group) { + try { + process.kill(-group, 0); + return true; + } catch (error) { + if (error.code === 'ESRCH') return false; + throw error; + } +} + +// Send TERM to the process group, and KILL once the grace period has passed. +// Returns once no process of the group is left, so a port it held is free. +export async function stopGroup(group) { + if (!groupAlive(group)) return; + process.kill(-group, 'SIGTERM'); + const deadline = Date.now() + STOP_GRACE_MS; + while (groupAlive(group)) { + if (Date.now() > deadline) { + process.kill(-group, 'SIGKILL'); + while (groupAlive(group)) await sleep(50); + return; + } + await sleep(50); + } +} + +async function readState(stateFile) { + let text; + try { + text = (await readFile(stateFile, 'utf8')).trim(); + } catch (error) { + if (error.code === 'ENOENT') return undefined; + throw error; + } + if (text === '') return undefined; + const [group, output] = text.split('\t'); + return { group: Number(group), output }; +} + +async function ready(url, stateFile) { + const { group } = await readState(stateFile); + const deadline = Date.now() + READY_TIMEOUT_MS; + for (;;) { + // The group is checked first, so a service that already held the port + // cannot answer for a command that has ended. + if (!groupAlive(group)) { + console.error(`the command ended before ${url} answered`); + return 1; + } + try { + const response = await fetch(url, { signal: AbortSignal.timeout(2000) }); + await response.body?.cancel(); + if (response.ok && groupAlive(group)) return 0; + } catch { + // Not answering yet: the command may still be starting. + } + if (Date.now() > deadline) { + console.error(`${url} did not answer within ${READY_TIMEOUT_MS / 1000} seconds`); + return 1; + } + await sleep(100); + } +} + +async function stop(stateFile) { + const state = await readState(stateFile); + if (!state) return 0; + // Checked before stopping it, so a group that already ended on its own is + // told apart from one this call is the one to stop. + const alreadyEnded = !groupAlive(state.group); + await stopGroup(state.group); + const output = await readFile(state.output, 'utf8'); + await writeFile(stateFile, ''); + if (alreadyEnded) { + console.error(`the background command had already exited, although it must keep running until it is stopped${output === '' ? '' : '; it printed:'}`); + process.stdout.write(output); + return 1; + } + console.log(`\nstopped the background command${output === '' ? '' : ', which printed:'}`); + process.stdout.write(output); + return 0; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const [command, ...args] = process.argv.slice(2); + if (command === 'ready' && args.length === 2) process.exit(await ready(...args)); + if (command === 'stop' && args.length === 1) process.exit(await stop(...args)); + console.error('usage: background.mjs ready | stop '); + process.exit(2); +} diff --git a/docs/site/scripts/tutorial-runner/gate.mjs b/docs/site/scripts/tutorial-runner/gate.mjs index 16946333c1..36c4fadea5 100644 --- a/docs/site/scripts/tutorial-runner/gate.mjs +++ b/docs/site/scripts/tutorial-runner/gate.mjs @@ -20,7 +20,9 @@ // run both, and those pages belong to its gate rather than to each. A page that // runs the commands under a toolset that does not serve them, or replays none // of them, is an error too, so neither the declaration nor test-skip can take a -// page out of the gate that owns its commands. +// page out of the gate that owns its commands. The one exception is a page no +// toolset can replay because it runs the commands of two that neither serves +// both: it declares either one, with a skip reason. // A new tutorial therefore fails the gate on the commit that adds it, until it // is replayed or says why not. @@ -115,6 +117,11 @@ export async function planGate(docsRoot, toolset, toolsets) { continue; } if (runs && !serves(declaration.toolset)) { + // A page running the commands of two toolsets that neither serves both + // can only be replayed by neither, so it is skipped in the gate of the + // one it declares. + const runsDeclared = commands.some((code) => toolsets[declaration.toolset].commands?.test(code)); + if (declaration.skip !== undefined && runsDeclared) continue; errors.push( `${slug}.mdx runs ${toolset} commands but declares toolset ${declaration.toolset}, which does not serve them; declare toolset ${toolset}`, ); diff --git a/docs/site/scripts/tutorial-runner/gate.test.mjs b/docs/site/scripts/tutorial-runner/gate.test.mjs index 4e40bf7930..aad5dfb9d2 100644 --- a/docs/site/scripts/tutorial-runner/gate.test.mjs +++ b/docs/site/scripts/tutorial-runner/gate.test.mjs @@ -149,3 +149,22 @@ test('a page running the toolset under another toolset, or skipping every one of ]); }); }); + +test('a skipped page running the commands of two toolsets, neither serving the other, may declare either', async () => { + const toolsets = { ...TOOLSETS, evidence: { commands: /(^|[^\w./-])(evidencectl|evidence)([^\w-]|$)/mu } }; + const both = '```sh\nbregctl check project\nevidencectl source add\n```\n'; + const pages = { + 'tutorials/composed': { frontmatter: 'tutorial_test:\n toolset: breg\n skip: needs a container\n', body: both }, + 'tutorials/replayed': { frontmatter: 'tutorial_test:\n toolset: breg\n', body: both }, + }; + await withDocs(pages, async (root) => { + const evidence = await planGate(root, 'evidence', toolsets); + assert.deepEqual(evidence.errors, [ + 'tutorials/replayed.mdx runs evidence commands but declares toolset breg, which does not serve them; declare toolset evidence', + ]); + assert.deepEqual(evidence.skipped, []); + const breg = await planGate(root, 'breg', toolsets); + assert.deepEqual(breg.errors, []); + assert.deepEqual(breg.skipped, [{ slug: 'tutorials/composed', reason: 'needs a container' }]); + }); +}); diff --git a/docs/site/scripts/tutorial-runner/page.mjs b/docs/site/scripts/tutorial-runner/page.mjs index cb778c9d02..427c943bb6 100644 --- a/docs/site/scripts/tutorial-runner/page.mjs +++ b/docs/site/scripts/tutorial-runner/page.mjs @@ -17,6 +17,14 @@ // stands at that point of the journey; bare, part of the output of the // nearest sh fence above it. // +// A block titled with a file path and marked test-file is the whole file the +// page asks the reader to create or replace in their editor. +// +// test-background="" on an sh fence is a command the page leaves running +// while the reader carries on in another terminal: the journey goes on once +// the URL answers. test-cwd="" on an sh fence names the directory, inside +// the reader directory, that the page tells the reader to return to first. +// // The page is parsed as Markdown rather than MDX, as check-draft-links.mjs // does: fences, including fences indented inside list items, parse the same // way, and JSX or comment lines read as paragraphs the journey never runs. @@ -26,7 +34,16 @@ import remarkParse from 'remark-parse'; import { unified } from 'unified'; const parser = unified().use(remarkParse).use(remarkGfm); -const ANNOTATIONS = new Set(['test-skip', 'test-expect', 'test-exit', 'test-edit', 'test-excerpt']); +const ANNOTATIONS = new Set([ + 'test-skip', + 'test-expect', + 'test-exit', + 'test-edit', + 'test-excerpt', + 'test-file', + 'test-background', + 'test-cwd', +]); // Parse the test- tokens of a fence meta string: bare flags and key="value" // pairs. Other tokens belong to Expressive Code and are ignored here. @@ -82,10 +99,11 @@ function* walk(node) { } // Return { steps, errors }. A step is one of -// { kind: 'run', line, heading, code, exit? } +// { kind: 'run', line, heading, code, exit?, background?, cwd? } // { kind: 'skip', line, heading, code, reason } // { kind: 'expect', line, heading, format, text, runIndex } // { kind: 'edit', line, heading, path, before, after } +// { kind: 'file', line, heading, path, text } // { kind: 'excerpt', line, heading, format, text, runIndex } // { kind: 'excerpt', line, heading, format, text, path } // where runIndex is the index in steps of the fence whose output it checks. @@ -110,6 +128,9 @@ export function readJourney(text) { const exit = annotations['test-exit']; const edit = annotations['test-edit']; const excerpt = annotations['test-excerpt']; + const file = annotations['test-file']; + const background = annotations['test-background']; + const cwd = annotations['test-cwd']; if (node.lang === 'sh') { if (expect !== undefined) { @@ -122,18 +143,48 @@ export function readJourney(text) { if (excerpt !== undefined) { errors.push(`line ${line}: test-excerpt belongs on a block the page shows, not on an sh fence`); } + if (file !== undefined) errors.push(`line ${line}: test-file belongs on a block showing the file, not on an sh fence`); const step = { kind: skip === undefined ? 'run' : 'skip', line, heading, code: node.value }; if (typeof skip === 'string' && skip !== '') step.reason = skip; if (exit !== undefined) { if (typeof exit === 'string' && /^\d+$/u.test(exit)) step.exit = Number(exit); else errors.push(`line ${line}: test-exit takes an exit status, as test-exit="1"`); } + if (background !== undefined) { + if (typeof background !== 'string' || !/^https?:\/\/\S+$/u.test(background)) { + errors.push( + `line ${line}: test-background takes the URL that answers once the command is ready, as test-background="http://127.0.0.1:4010/"`, + ); + } else if (exit !== undefined) { + errors.push(`line ${line}: a test-background fence keeps running, so it cannot also be test-exit`); + } else if (skip !== undefined) { + errors.push(`line ${line}: a test-background fence is run, so it cannot also be test-skip`); + } else step.background = background; + } + if (cwd !== undefined) { + if (typeof cwd !== 'string' || cwd === '' || cwd.startsWith('/') || cwd.split('/').includes('..')) { + errors.push(`line ${line}: test-cwd takes a directory inside the reader directory, as test-cwd="first-project"`); + } else step.cwd = cwd; + } lastCommand = steps.push(step) - 1; continue; } if (skip !== undefined) errors.push(`line ${line}: test-skip applies only to sh fences`); if (exit !== undefined) errors.push(`line ${line}: test-exit applies only to sh fences`); + if (background !== undefined) errors.push(`line ${line}: test-background applies only to sh fences`); + if (cwd !== undefined) errors.push(`line ${line}: test-cwd applies only to sh fences`); + if ([file, edit, expect, excerpt].filter((value) => value !== undefined).length > 1) { + errors.push(`line ${line}: a block is one of test-file, test-edit, test-expect, or test-excerpt`); + continue; + } + if (file !== undefined) { + const path = titleOf(node.meta); + if (node.lang === 'diff') errors.push(`line ${line}: test-file takes the whole file; a diff block is test-edit`); + else if (!path) errors.push(`line ${line}: test-file needs the file path, as title=""`); + else steps.push({ kind: 'file', line, heading, path, text: `${node.value}\n` }); + continue; + } if (edit !== undefined) { const path = titleOf(node.meta); const sides = node.lang === 'diff' ? diffSides(node.value) : undefined; @@ -153,10 +204,6 @@ export function readJourney(text) { continue; } if (expect === undefined && excerpt === undefined) continue; - if (expect !== undefined && excerpt !== undefined) { - errors.push(`line ${line}: a block is either test-expect or test-excerpt, not both`); - continue; - } const format = node.lang === 'json' ? 'json' : 'text'; if (typeof excerpt === 'string' && excerpt !== '') { steps.push({ kind: 'excerpt', line, heading, format, text: node.value, path: excerpt }); diff --git a/docs/site/scripts/tutorial-runner/page.test.mjs b/docs/site/scripts/tutorial-runner/page.test.mjs index f11e625ddb..885f8f1a36 100644 --- a/docs/site/scripts/tutorial-runner/page.test.mjs +++ b/docs/site/scripts/tutorial-runner/page.test.mjs @@ -250,6 +250,61 @@ test('test-excerpt mistakes are errors that name the line', () => { 'line 1: test-excerpt has no sh fence above it to check', 'line 5: test-excerpt belongs on a block the page shows, not on an sh fence', 'line 13: test-excerpt checks the output of the sh fence at line 9, which is skipped', - 'line 17: a block is either test-expect or test-excerpt, not both', + 'line 17: a block is one of test-file, test-edit, test-expect, or test-excerpt', + ]); +}); + +test('a titled block marked test-file is the whole file the page asks the reader to create', () => { + const { steps, errors } = readJourney( + '## Ask\n\nOpen `questions/q.yaml` and add:\n\n```yaml title="questions/q.yaml" test-file\nid: q\n```\n', + ); + assert.deepEqual(errors, []); + assert.deepEqual(steps, [{ kind: 'file', line: 5, heading: 'Ask', path: 'questions/q.yaml', text: 'id: q\n' }]); +}); + +test('test-file mistakes are errors that name the line', () => { + const { errors } = readJourney( + '```yaml test-file\na: 1\n```\n\n```sh title="x.sh" test-file\ntrue\n```\n\n' + + '```diff title="a.yaml" test-file\n+a\n```\n\n```yaml title="a.yaml" test-file test-excerpt\na\n```\n', + ); + assert.deepEqual(errors, [ + 'line 1: test-file needs the file path, as title=""', + 'line 5: test-file belongs on a block showing the file, not on an sh fence', + 'line 9: test-file takes the whole file; a diff block is test-edit', + 'line 13: a block is one of test-file, test-edit, test-expect, or test-excerpt', + ]); +}); + +test('test-background leaves an sh fence running until its ready URL answers, and test-cwd says where a fence runs', () => { + const { steps, errors } = readJourney( + '```sh test-background="http://127.0.0.1:4010/health" test-cwd="first"\nserve\n```\n\n' + + '```text test-expect\nready\n```\n\n```sh test-cwd="first/project"\nls\n```\n', + ); + assert.deepEqual(errors, []); + assert.deepEqual(steps[0], { + kind: 'run', + line: 1, + heading: '', + code: 'serve', + background: 'http://127.0.0.1:4010/health', + cwd: 'first', + }); + assert.equal(steps[1].runIndex, 0); + assert.equal(steps[2].cwd, 'first/project'); +}); + +test('test-background and test-cwd mistakes are errors that name the line', () => { + const { errors } = readJourney( + '```sh test-background\nserve\n```\n\n```sh test-background="http://x/" test-exit="1"\nserve\n```\n\n' + + '```sh test-background="http://x/" test-skip="offline"\nserve\n```\n\n' + + '```sh test-cwd="/abs"\nls\n```\n\n```sh test-cwd="../up"\nls\n```\n\n```text test-cwd="a"\nx\n```\n', + ); + assert.deepEqual(errors, [ + 'line 1: test-background takes the URL that answers once the command is ready, as test-background="http://127.0.0.1:4010/"', + 'line 5: a test-background fence keeps running, so it cannot also be test-exit', + 'line 9: a test-background fence is run, so it cannot also be test-skip', + 'line 13: test-cwd takes a directory inside the reader directory, as test-cwd="first-project"', + 'line 17: test-cwd takes a directory inside the reader directory, as test-cwd="first-project"', + 'line 21: test-cwd applies only to sh fences', ]); }); diff --git a/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs b/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs index 213681affb..ec55787377 100644 --- a/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs +++ b/docs/site/scripts/tutorial-runner/run-tutorial.test.mjs @@ -2,6 +2,8 @@ import assert from 'node:assert/strict'; import { execFile, spawn } from 'node:child_process'; import { existsSync } from 'node:fs'; import { chmod, mkdir, mkdtemp, readdir, readFile, realpath, rm, writeFile } from 'node:fs/promises'; +import { createServer as createHttpServer } from 'node:http'; +import { createServer } from 'node:net'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import test from 'node:test'; @@ -472,3 +474,241 @@ test('a gate dry run builds nothing and starts nothing', async () => { assert.equal(existsSync(join(bin, 'calls')), false, output); }); }); + +// A loopback port nothing listens on right now. +async function freePort() { + const server = createServer(); + await new Promise((resolvePromise) => server.listen(0, '127.0.0.1', resolvePromise)); + const { port } = server.address(); + await new Promise((resolvePromise) => server.close(resolvePromise)); + return port; +} + +const alive = (pid) => { + try { + process.kill(pid, 0); + return true; + } catch (error) { + if (error.code === 'ESRCH') return false; + throw error; + } +}; + +test('a test-file block writes the whole file where the reader stands', async () => { + const body = + '## Write\n\n' + + fence('sh', 'mkdir -p work/questions\ncd work\necho stale >questions/q.yaml') + + 'Open `questions/q.yaml` and add:\n\n' + + fence('yaml title="questions/q.yaml" test-file', 'id: q\npurpose: check') + + fence('sh', 'cat questions/q.yaml') + + fence('text test-expect', 'id: q\npurpose: check'); + await withPage(body, async ({ page }) => { + const { code, output } = await run([page]); + assert.equal(code, 0, output); + assert.match(output, /wrote questions\/q\.yaml/u); + assert.match(output, /tutorial PASS/u); + const plan = await run(['--dry-run', page]); + assert.match(plan.output, /file line 15 \(Write\): questions\/q\.yaml/u); + }); +}); + +test('a test-file block whose directory does not exist stops the journey', async () => { + const body = '## Write\n\n' + fence('yaml title="missing/q.yaml" test-file', 'id: q') + fence('sh', 'echo never'); + await withPage(body, async ({ page }) => { + const { code, output } = await run([page]); + assert.equal(code, 1, output); + assert.match(output, /the file at line 7 \(Write\) failed/u); + assert.doesNotMatch(output, /^never$/mu); + }); +}); + +test('a background fence runs until the next one starts or the page ends, and test-cwd says where a fence runs', async () => { + const port = await freePort(); + const url = `http://127.0.0.1:${port}/`; + const serve = `python3 -m http.server ${port} --bind 127.0.0.1`; + const body = + '## Serve\n\n' + + fence('sh', 'mkdir -p site/one site/two other\necho first >site/one/index.html\necho second >site/two/index.html\ncd other') + + fence(`sh test-background="${url}" test-cwd="site/one"`, serve) + + fence('sh', `curl -fsS ${url}\npwd | sed 's|.*/||'`) + + fence('text test-expect', 'first\nother') + + fence(`sh test-background="${url}" test-cwd="site/two"`, serve) + + fence('sh test-cwd="site"', `curl -fsS ${url}\npwd | sed 's|.*/||'`) + + fence('text test-expect', 'second\nsite'); + await withPage(body, async ({ page }) => { + const { code, output } = await run([page]); + assert.equal(code, 0, output); + assert.match(output, /tutorial PASS/u); + await assert.rejects(fetch(url), 'the page end must stop the background fence'); + const plan = await run(['--dry-run', page]); + assert.match(plan.output, new RegExp(`run line 14 \\(Serve\\): ${serve} \\(in site/one, in the background until ${url} answers\\)`, 'u')); + }); +}); + +test('a background fence that ends before it is ready stops the journey and shows what it printed', async () => { + const port = await freePort(); + const body = '## Serve\n\n' + fence(`sh test-background="http://127.0.0.1:${port}/"`, 'echo "port taken" >&2\nfalse') + fence('sh', 'echo never'); + await withPage(body, async ({ page }) => { + const { code, output } = await run([page]); + assert.equal(code, 1, output); + assert.match(output, /port taken/u); + assert.match(output, /the sh fence at line 7 \(Serve\) failed/u); + assert.doesNotMatch(output, /^never$/mu); + }); +}); + +test('a background fence is ready only once its URL answers with success', async () => { + const port = await freePort(); + const url = `http://127.0.0.1:${port}/ready.txt`; + const body = + '## Serve\n\n' + + fence(`sh test-background="${url}"`, `(sleep 1; echo ok >ready.txt) &\nexec python3 -m http.server ${port} --bind 127.0.0.1 2>/dev/null`) + + fence('sh', 'cat ready.txt') + + fence('text test-expect', 'ok'); + await withPage(body, async ({ page }) => { + const { code, output } = await run([page]); + assert.equal(code, 0, output); + assert.match(output, /tutorial PASS/u); + }); +}); + +test('a background fence that ends is not ready although another service answers its URL', async () => { + const port = await freePort(); + const stale = createHttpServer((request, response) => response.end('stale')); + await new Promise((resolvePromise) => stale.listen(port, '127.0.0.1', resolvePromise)); + try { + const body = '## Serve\n\n' + fence(`sh test-background="http://127.0.0.1:${port}/"`, 'echo "port taken" >&2\nfalse') + fence('sh', 'echo never'); + await withPage(body, async ({ page }) => { + const { code, output } = await run([page]); + assert.equal(code, 1, output); + assert.match(output, /port taken/u); + assert.match(output, /the sh fence at line 7 \(Serve\) failed/u); + assert.doesNotMatch(output, /^never$/mu); + }); + } finally { + await new Promise((resolvePromise) => stale.close(resolvePromise)); + } +}); + +test('an expectation on a background fence checks what it printed while it ran', async () => { + const port = await freePort(); + const url = `http://127.0.0.1:${port}/`; + const body = + '## Serve\n\n' + + fence(`sh test-background="${url}"`, `echo "ready on ${port}"\nexec python3 -m http.server ${port} --bind 127.0.0.1 2>/dev/null`) + + fence('text test-expect', 'ready on ') + + fence('sh', `curl -fsS -o /dev/null ${url}`); + await withPage(body, async ({ page }) => { + const { code, output } = await run([page]); + assert.equal(code, 0, output); + assert.match(output, /expect line 12: ok/u); + }); +}); + +test('a background command that exits on its own before it is stopped fails the journey', async () => { + const port = await freePort(); + const url = `http://127.0.0.1:${port}/`; + const serve = `python3 -m http.server ${port} --bind 127.0.0.1`; + const body = + '## Serve\n\n' + + fence( + `sh test-background="${url}"`, + `${serve} &\nserver=$!\nsleep 1\nkill "$server"\nwait "$server" 2>/dev/null || true\necho "the flaky server has exited"`, + ) + + fence('sh', 'sleep 1.5\necho after'); + await withPage(body, async ({ page }) => { + const { code, output } = await run([page]); + assert.equal(code, 1, output); + assert.match(output, /the flaky server has exited/u, 'the background command must be named by what it printed'); + assert.match(output, /background command.*exited/isu); + assert.match(output, /tutorial FAIL/u); + }); +}); + +test('a journey that ends, passing or failing, leaves no process it started running', async () => { + const port = await freePort(); + const body = (pids, last) => + '## Start\n\n' + + fence('sh', `sleep 300 &\necho "$!" >>'${pids}'`) + + fence(`sh test-background="http://127.0.0.1:${port}/"`, `echo "$BASHPID" >>'${pids}'\nexec python3 -m http.server ${port} --bind 127.0.0.1`) + + fence('sh', last); + for (const [last, expected] of [['true', 0], ['false', 1]]) { + await withPage('', async ({ dir, page }) => { + const pids = join(dir, 'pids'); + await writeFile(page, `---\ntitle: t\n---\n\n${body(pids, last)}`); + const { code, output } = await run([page]); + assert.equal(code, expected, output); + const started = (await readFile(pids, 'utf8')).trim().split('\n').map(Number); + assert.equal(started.length, 2, output); + for (const pid of started) assert.equal(alive(pid), false, `process ${pid} outlived the journey\n${output}`); + }); + } +}); + +// A wheel holding one package with no dependencies, as the assembled client is. +async function fakeWheel(dir) { + const wheel = join(dir, 'client.whl'); + await execFileAsync('python3', [ + '-c', + 'import sys, zipfile\nwith zipfile.ZipFile(sys.argv[1], "w") as z: z.writestr("tutorial_fake_client/__init__.py", "NAME = \\"unpacked\\"\\n")', + wheel, + ]); + return wheel; +} + +async function fakeEvidenceBinaries(dir, calls) { + const env = {}; + for (const [name, variable] of [ + ['evidence', 'EVIDENCE_BIN'], + ['evidencectl', 'EVIDENCECTL_BIN'], + ['evidence-oid4vci', 'EVIDENCE_OID4VCI_BIN'], + ]) { + await writeFile(join(dir, name), `#!/bin/sh\nprintf '%s %s\\n' ${name} "$*" >>'${calls}'\n`); + await chmod(join(dir, name), 0o755); + env[variable] = join(dir, name); + } + return env; +} + +test('the evidence toolset serves its binaries, the client package, and the FHIR mock, and stops dev sessions', async () => { + const body = + '## Start\n\n' + + fence( + 'sh', + 'evidence --version\nevidencectl dev\n"$EVIDENCE_OID4VCI_BIN" --version\n' + + 'python3 -c "import tutorial_fake_client; print(tutorial_fake_client.NAME)"\n' + + 'python3 -c "import os, urllib.request; print(urllib.request.urlopen(os.environ[\'FHIR_TUTORIAL_TEST_BASE_URL\'] + \'/healthz\').status)"\n' + + 'mkdir -p work/project/.evidence/dev\ntouch work/project/.evidence/dev/control.sock', + ) + + fence('text test-expect', 'unpacked\n200') + + fence('sh', 'false'); + await withPage(body, async ({ dir, page }) => { + const calls = join(dir, 'calls.log'); + const env = await fakeEvidenceBinaries(dir, calls); + const { code, output } = await run(['--toolset', 'evidence', page], { + ...env, + REGISTRY_CLIENT_PY_WHEEL: await fakeWheel(dir), + }); + assert.equal(code, 1, output); + const log = (await readFile(calls, 'utf8')).trim().split('\n'); + assert.deepEqual(log.slice(0, 3), ['evidence --version', 'evidencectl dev', 'evidence-oid4vci --version']); + assert.match(log[3], /^evidencectl dev stop --project \/\S+\/work\/project$/u); + assert.equal(log.length, 4); + await assert.rejects(fetch('http://127.0.0.1:8003/healthz'), 'the FHIR mock must stop with the journey'); + }); +}); + +test('the evidence toolset needs the client wheel as an absolute path to a file', async () => { + await withPage('## A\n\n' + fence('sh', 'true'), async ({ dir, page }) => { + const env = await fakeEvidenceBinaries(dir, join(dir, 'calls.log')); + for (const [wheel, message] of [ + ['', /REGISTRY_CLIENT_PY_WHEEL is unset: name a client wheel assembled with release\/scripts\/assemble-registry-client-packages\.py/u], + ['client.whl', /REGISTRY_CLIENT_PY_WHEEL must be an absolute path: client\.whl/u], + [join(dir, 'missing.whl'), /client wheel not found: \/\S+\/missing\.whl/u], + ]) { + const { code, output } = await run(['--toolset', 'evidence', page], { ...env, REGISTRY_CLIENT_PY_WHEEL: wheel }); + assert.equal(code, 2, output); + assert.match(output, message); + } + }); +}); diff --git a/docs/site/scripts/tutorial-runner/toolsets.mjs b/docs/site/scripts/tutorial-runner/toolsets.mjs index 4a9f819a99..c77381fdd7 100644 --- a/docs/site/scripts/tutorial-runner/toolsets.mjs +++ b/docs/site/scripts/tutorial-runner/toolsets.mjs @@ -5,10 +5,14 @@ // the journey left behind, whether it passed or failed. This is the only // product-specific code in the runner. -import { spawnSync } from 'node:child_process'; -import { accessSync, constants } from 'node:fs'; +import { spawn, spawnSync } from 'node:child_process'; +import { accessSync, closeSync, constants, existsSync, openSync, readFileSync, statSync } from 'node:fs'; import { mkdir, readdir, symlink } from 'node:fs/promises'; -import { dirname, isAbsolute, join } from 'node:path'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { fileURLToPath } from 'node:url'; + +import { stopGroup } from './background.mjs'; export class ToolsetError extends Error {} @@ -24,12 +28,23 @@ function checkBinary(variable, path) { // A product toolset: its binaries, built from this checkout unless every one // of their variables names an exact candidate or released binary, served by // name from binDir; and its local development sessions, stopped in the order -// given when the journey ends. -function productToolset({ label, binaries, cargoArgs, profileVariable, targetName, sessions, ...rest }) { +// given when the journey ends, each with the arguments stopArgs gives for its +// project directory. +function productToolset({ + label, + binaries, + cargoArgs, + profileVariable, + targetName, + sessions, + stopArgs = (project) => ['dev', 'stop', project, '--remove'], + ...rest +}) { const variables = binaries.map(([, variable]) => variable); return { ...rest, + // Returns the variables the journey's environment adds, if any. async prepare({ repoRoot, binDir }) { const given = binaries.map(([, variable]) => process.env[variable] || undefined); if (given.some(Boolean) && !given.every(Boolean)) { @@ -55,12 +70,13 @@ function productToolset({ label, binaries, cargoArgs, profileVariable, targetNam binaries.forEach(([, variable], i) => checkBinary(variable, paths[i])); await mkdir(binDir, { recursive: true }); for (const [i, [name]] of binaries.entries()) await symlink(paths[i], join(binDir, name)); + return {}; }, // Stop every local development session the journey started and reclaim - // its container and volume. A journey that fails halfway leaves its - // sessions running, and deleting the reader directory alone would orphan - // their containers. Stopping with --remove is idempotent. Returns false + // its container and volume, if it has one. A journey that fails halfway + // leaves its sessions running, and deleting the reader directory alone + // would orphan them. Stopping is idempotent. Returns false // when a session could not be stopped, so its project is kept for a second // attempt. async teardown({ readerDir, binDir }) { @@ -75,7 +91,7 @@ function productToolset({ label, binaries, cargoArgs, profileVariable, targetNam for (const [tool, state] of sessions) { for (const entry of entries.filter((path) => path.endsWith(state)).sort()) { const project = dirname(dirname(dirname(join(readerDir, entry)))); - const stop = spawnSync(join(binDir, tool), ['dev', 'stop', project, '--remove'], { encoding: 'utf8' }); + const stop = spawnSync(join(binDir, tool), stopArgs(project), { encoding: 'utf8' }); if (stop.status === 0) { console.log(`stopped the local development session in ${project}`); } else { @@ -94,8 +110,9 @@ function productToolset({ label, binaries, cargoArgs, profileVariable, targetNam const breg = productToolset({ label: 'breg and bregctl', // A page whose sh fences match this runs the toolset, so its gate must - // cover it (tutorial-runner/gate.mjs). - commands: /(^|[^\w-])(bregctl|breg)([^\w-]|$)/mu, + // cover it (tutorial-runner/gate.mjs). A path such as .breg/dev or + // tutorial-work/breg names no command. + commands: /(^|[^\w./-])(bregctl|breg)([^\w./-]|$)/mu, binaries: [ ['breg', 'BREG_BIN'], ['bregctl', 'BREGCTL_BIN'], @@ -112,7 +129,7 @@ const breg = productToolset({ // so they no longer reconcile against a registry that is stopping. const casework = productToolset({ label: 'casework, caseworkctl, breg, and bregctl', - commands: /(^|[^\w-])(caseworkctl|casework)([^\w-]|$)/mu, + commands: /(^|[^\w./-])(caseworkctl|casework)([^\w./-]|$)/mu, includes: ['breg'], binaries: [ ['casework', 'CASEWORK_BIN'], @@ -129,12 +146,117 @@ const casework = productToolset({ ], }); +// Evidence: evidence, evidencectl, and evidence-oid4vci. Two things a reader +// sets up themselves are set up here instead: +// +// - The Python client package a reader installs with pip is unpacked from +// REGISTRY_CLIENT_PY_WHEEL, a wheel assembled from this checkout with +// release/scripts/assemble-registry-client-packages.py, onto PYTHONPATH. +// The package declares no dependencies, so unpacking is enough, and its +// bindings are built for the stable ABI, so it imports under any CPython. +// - The FHIR server a page reads is replaced by the sanitized local mock in +// fixtures/fhir-tutorial-mock.py, at FHIR_TUTORIAL_TEST_BASE_URL, for every +// journey. +// +// EVIDENCE_OID4VCI_BIN names the served evidence-oid4vci for the +// interoperability checks, which also take EVIDENCE_OID4VCI_INTEROP_TEST_BIN +// from the environment when it names a prebuilt interoperability test. +const FHIR_MOCK = resolve(dirname(fileURLToPath(import.meta.url)), '../fixtures/fhir-tutorial-mock.py'); +const FHIR_BASE_URL = 'http://127.0.0.1:8003'; +const FHIR_READY_TIMEOUT_MS = 30_000; +const WHEEL_HINT = 'release/scripts/assemble-registry-client-packages.py'; + +function clientWheel() { + const wheel = process.env.REGISTRY_CLIENT_PY_WHEEL ?? ''; + if (wheel === '') throw new ToolsetError(`REGISTRY_CLIENT_PY_WHEEL is unset: name a client wheel assembled with ${WHEEL_HINT}`); + if (!isAbsolute(wheel)) throw new ToolsetError(`REGISTRY_CLIENT_PY_WHEEL must be an absolute path: ${wheel}`); + if (!existsSync(wheel) || !statSync(wheel).isFile()) { + throw new ToolsetError(`client wheel not found: ${wheel}; assemble one with ${WHEEL_HINT}`); + } + return wheel; +} + +// Start the FHIR mock in its own process group and wait until it answers. +// Returns its process group. +async function startFhirMock(workRoot) { + const log = join(workRoot, 'fhir-tutorial-mock.log'); + const fd = openSync(log, 'w'); + const child = spawn('python3', [FHIR_MOCK], { detached: true, stdio: ['ignore', fd, fd] }); + closeSync(fd); + const failed = async (why) => { + await stopGroup(child.pid); + return new ToolsetError(`the FHIR tutorial mock ${why}:\n${readFileSync(log, 'utf8')}`); + }; + const deadline = Date.now() + FHIR_READY_TIMEOUT_MS; + for (;;) { + if (child.exitCode !== null || child.signalCode !== null) throw await failed(`ended before ${FHIR_BASE_URL} answered`); + try { + const response = await fetch(`${FHIR_BASE_URL}/healthz`, { signal: AbortSignal.timeout(2000) }); + if (response.ok) break; + } catch { + // Not answering yet: the mock may still be starting. + } + if (Date.now() > deadline) throw await failed(`did not answer within ${FHIR_READY_TIMEOUT_MS / 1000} seconds`); + await sleep(100); + } + // Another process may hold the port and have answered in its place. + await sleep(100); + if (child.exitCode !== null || child.signalCode !== null) throw await failed(`could not serve ${FHIR_BASE_URL}`); + return child.pid; +} + +const evidenceProduct = productToolset({ + label: 'evidence, evidencectl, and evidence-oid4vci', + // Not a path such as .evidence/dev, which a page may name; but a page that + // runs Evidence's own checks from a checkout runs Evidence. + commands: /(^|[^\w./-])(evidencectl|evidence-oid4vci|evidence)([^\w./-]|$)|(^|\s)products\/evidence\/scripts\//mu, + binaries: [ + ['evidence', 'EVIDENCE_BIN'], + ['evidencectl', 'EVIDENCECTL_BIN'], + ['evidence-oid4vci', 'EVIDENCE_OID4VCI_BIN'], + ], + cargoArgs: ['-p', 'registry-evidence', '-p', 'registry-evidencectl', '-p', 'registry-evidence-oid4vci'], + profileVariable: 'EVIDENCE_TUTORIAL_CARGO_PROFILE', + targetName: 'evidence-tutorial-source', + sessions: [['evidencectl', '.evidence/dev/control.sock']], + stopArgs: (project) => ['dev', 'stop', '--project', project], +}); + +let fhirMock; +const evidence = { + ...evidenceProduct, + + async prepare({ repoRoot, binDir, workRoot }) { + const wheel = clientWheel(); + await evidenceProduct.prepare({ repoRoot, binDir }); + const clientPackage = join(workRoot, 'client-package'); + const unpack = spawnSync('python3', ['-m', 'zipfile', '-e', wheel, clientPackage], { encoding: 'utf8' }); + if (unpack.status !== 0) throw new ToolsetError(`could not unpack ${wheel}:\n${unpack.stderr}`); + fhirMock = await startFhirMock(workRoot); + const pythonPath = process.env.PYTHONPATH ? `${clientPackage}:${process.env.PYTHONPATH}` : clientPackage; + return { + PYTHONPATH: pythonPath, + EVIDENCE_OID4VCI_BIN: join(binDir, 'evidence-oid4vci'), + FHIR_TUTORIAL_TEST_BASE_URL: FHIR_BASE_URL, + }; + }, + + async teardown(context) { + const stoppedAll = await evidenceProduct.teardown(context); + if (fhirMock !== undefined) await stopGroup(fhirMock); + fhirMock = undefined; + return stoppedAll; + }, +}; + // No product binaries: the journey runs against what is already on PATH. const none = { - async prepare() {}, + async prepare() { + return {}; + }, async teardown() { return true; }, }; -export const TOOLSETS = { breg, casework, none }; +export const TOOLSETS = { breg, casework, evidence, none }; diff --git a/docs/site/scripts/tutorial-runner/toolsets.test.mjs b/docs/site/scripts/tutorial-runner/toolsets.test.mjs new file mode 100644 index 0000000000..7a3eeb1c95 --- /dev/null +++ b/docs/site/scripts/tutorial-runner/toolsets.test.mjs @@ -0,0 +1,29 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { TOOLSETS } from './toolsets.mjs'; + +// A toolset's commands pattern decides which pages its gate must replay, so it +// matches a command a fence runs and not a path or file that shares its name. +const CASES = { + breg: { + runs: ['bregctl init .', 'bregctl dev start .', 'breg --version', 'version=$(bregctl --version)', 'bregctl dev stop .; echo done'], + names: ['cd tutorial-work/breg', 'cat .breg/dev/state.json', 'ls ./breg', 'cat breg.yaml', 'cd breg-demo', 'ls breg/'], + }, + casework: { + runs: ['caseworkctl dev start .', 'casework --version', 'caseworkctl check . | tail -1'], + names: ['cd tutorial-work/casework', 'cat .casework/dev/state.json', 'cat casework.yaml', 'ls casework/'], + }, + evidence: { + runs: ['evidencectl init .', 'evidence --version', 'evidence-oid4vci --help', 'products/evidence/scripts/check-contracts.sh'], + names: ['cd ~/work/evidence', 'ls .evidence/clients', 'cat evidence.yaml', 'ls evidence/'], + }, +}; + +for (const [name, { runs, names }] of Object.entries(CASES)) { + test(`the ${name} toolset matches the commands a fence runs, not paths that share their name`, () => { + const { commands } = TOOLSETS[name]; + for (const code of runs) assert.equal(commands.test(code), true, `${name} must match: ${code}`); + for (const code of names) assert.equal(commands.test(code), false, `${name} must not match: ${code}`); + }); +} diff --git a/docs/site/src/content/docs/tutorials/assert-a-role-bound-relationship.mdx b/docs/site/src/content/docs/tutorials/assert-a-role-bound-relationship.mdx index 1bdc6b4a16..2f0d242cdf 100644 --- a/docs/site/src/content/docs/tutorials/assert-a-role-bound-relationship.mdx +++ b/docs/site/src/content/docs/tutorials/assert-a-role-bound-relationship.mdx @@ -11,6 +11,8 @@ persona: - assertion provider locale: en standards_referenced: [] +tutorial_test: + toolset: evidence --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -37,7 +39,7 @@ mkdir role-bound-relationship cd role-bound-relationship ``` -```python +```python title="registry.py" test-file import json from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import unquote, urlsplit @@ -98,7 +100,7 @@ ThreadingHTTPServer(("127.0.0.1", 8002), Registry).serve_forever() Start it in one terminal and leave it running: -```sh +```sh test-background="http://127.0.0.1:8002/openapi.json" python3 registry.py ``` @@ -121,7 +123,7 @@ cd parent-relationship Create the question definition: -```yaml +```yaml title="questions/parent-relationship.yaml" test-file id: parent-relationship question: Is the candidate registered as a parent of the child? purpose: relationship-check @@ -153,7 +155,7 @@ arguments. The source decision is the only fact that reaches the derivation. Create the answer logic: -```rhai +```rhai title="derivations/parent-relationship.rhai" test-file fn answer(facts, selectors, context) { #{ relationship_confirmed: @@ -172,7 +174,7 @@ and both subject bindings before signing. evidencectl dev start . ``` -```text +```text test-excerpt Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -213,7 +215,7 @@ evidencectl verify parent-relationship.jws.json \ --output parent-relationship.verified.json ``` -```text +```text test-expect VERIFIED ``` @@ -224,7 +226,7 @@ python3 -m json.tool parent-relationship.verified.json The verified assertion has two pseudonymous subject bindings, one for `child` and one for `candidate-parent`, plus this supported value: -```json +```json test-excerpt { "providesValueFor": "urn:registrystack:evidence:local:concept:parent-relationship:relationship_confirmed", "value": true @@ -242,6 +244,13 @@ evidencectl audit show --last-operation evidencectl dev clean ``` +```text test-expect +Local Evidence stopped +ACCESS AUTHORIZED parent-relationship relationship-check requester= +DISCLOSURE RELEASED relationship_confirmed +Removed stopped local Evidence state +``` + The audit identifies the authorized question, purpose, requester pseudonym, and disclosed concept. It does not record either source identifier or the boolean value. diff --git a/docs/site/src/content/docs/tutorials/build-and-deploy-evidence-project.mdx b/docs/site/src/content/docs/tutorials/build-and-deploy-evidence-project.mdx index 5f554c71c7..95a7655079 100644 --- a/docs/site/src/content/docs/tutorials/build-and-deploy-evidence-project.mdx +++ b/docs/site/src/content/docs/tutorials/build-and-deploy-evidence-project.mdx @@ -12,6 +12,9 @@ persona: - operator locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: drift-checked by evidence-production-build-docs.test.mjs; needs a production build environment --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; diff --git a/docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx b/docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx index 2bbd95f22f..04495f22b3 100644 --- a/docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx +++ b/docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx @@ -12,6 +12,8 @@ persona: - operator locale: en standards_referenced: [] +tutorial_test: + toolset: evidence --- Use a SQLite extract when the authority can publish an immutable snapshot but cannot offer a @@ -139,7 +141,7 @@ Create a complete governed target by following [Build and deploy an Evidence Gat project](../build-and-deploy-evidence-project/). Then build the editable project and run the checks against the exact candidate and mounted extract: -```sh +```sh test-skip="needs the complete governed deployment target from Build and deploy an Evidence Gateway project" evidencectl package registry-status \ --target registry-status/deployment-targets/staging \ --output candidate-staging diff --git a/docs/site/src/content/docs/tutorials/connect-an-institution-source.mdx b/docs/site/src/content/docs/tutorials/connect-an-institution-source.mdx index 8669d95806..6ab5feaccc 100644 --- a/docs/site/src/content/docs/tutorials/connect-an-institution-source.mdx +++ b/docs/site/src/content/docs/tutorials/connect-an-institution-source.mdx @@ -11,6 +11,9 @@ persona: - assertion provider locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: how-to against the reader's own OpenAPI source; no fixed scenario to replay --- If this is your first Evidence Gateway project, complete diff --git a/docs/site/src/content/docs/tutorials/control-who-can-request-evidence.mdx b/docs/site/src/content/docs/tutorials/control-who-can-request-evidence.mdx index 70e697e36b..841cb46422 100644 --- a/docs/site/src/content/docs/tutorials/control-who-can-request-evidence.mdx +++ b/docs/site/src/content/docs/tutorials/control-who-can-request-evidence.mdx @@ -11,6 +11,9 @@ persona: - assertion provider locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + after: tutorials/return-a-governed-value --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -65,13 +68,13 @@ A client change affects the next local generation. Stopping that generation also In one terminal, return to the working directory from the first tutorial and serve the checked synthetic cases: -```sh +```sh test-cwd="first-evidence-assertion" test-background="http://127.0.0.1:4010/people/person-123" evidencectl source mock serve --config adult-status/mocks/source.yaml ``` Leave the source mock running. In another terminal, enter the project containing both questions: -```sh +```sh test-cwd="first-evidence-assertion" cd adult-status ``` @@ -94,7 +97,7 @@ evidencectl access policy add age-checks --question adult-status evidencectl access policy add service-routing --question age-bracket ``` -```text +```text test-expect Added access policy age-checks for adult-status. Added access policy service-routing for age-bracket. ``` @@ -125,7 +128,7 @@ evidencectl access client add age-checker \ --generate-local-key ``` -```text +```text test-expect Added client age-checker with policy age-checks. ``` @@ -143,7 +146,7 @@ Compile the questions and both access policies, then start Evidence Gateway and evidencectl dev start . ``` -```text +```text test-excerpt Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -182,7 +185,7 @@ curl --silent --show-error --fail-with-body \ --write-out 'HTTP %{http_code}\n' ``` -```text +```text test-expect HTTP 200 ``` @@ -194,7 +197,7 @@ evidencectl verify age-checker-allowed.jws.json \ --output age-checker-allowed.verified.json ``` -```text +```text test-expect VERIFIED ``` @@ -213,7 +216,7 @@ evidencectl access client add service-router \ --generate-local-key ``` -```text +```text test-excerpt Added client service-router with policy service-routing. ``` @@ -264,7 +267,7 @@ evidencectl verify service-router-allowed.jws.json \ --output service-router-allowed.verified.json ``` -```text +```text test-expect HTTP 200 VERIFIED ``` @@ -306,7 +309,7 @@ curl --silent --show-error \ --write-out 'HTTP %{http_code}\n' ``` -```text +```text test-expect HTTP 403 ``` @@ -316,14 +319,14 @@ Inspect the safe problem response: python3 -m json.tool age-checker-refused.json ``` -```json +```json test-expect { "type": "https://id.registrystack.org/problems/registry-evidence/evidence/denied", "title": "Evidence request is not permitted", "status": 403, "detail": "the Evidence request is not permitted", "code": "evidence.denied", - "traceId": "<32-lowercase-hex-trace-id>" + "traceId": "" } ``` @@ -342,7 +345,7 @@ evidencectl dev stop evidencectl audit show --last-operation ``` -```text +```text test-expect Local Evidence stopped ACCESS REFUSED requester= reason=not_authorized ``` @@ -358,8 +361,8 @@ evidencectl access client revoke age-checker evidencectl dev start . ``` -```text -Revoked client age-checker. +```text test-expect +Revoked client age-checker (removed local private key .evidence/clients/age-checker). Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -369,7 +372,7 @@ revoked client. Try to prepare a fresh request as the revoked client: -```sh +```sh test-exit="1" evidencectl request prepare adult-status \ --purpose age-check \ --subject person_id=person-123 \ @@ -377,7 +380,7 @@ evidencectl request prepare adult-status \ --name age-checker-revoked ``` -```text +```text test-expect evidencectl: unknown or revoked active client age-checker ``` @@ -393,7 +396,7 @@ Stop the local services: evidencectl dev stop ``` -```text +```text test-expect Local Evidence stopped ``` diff --git a/docs/site/src/content/docs/tutorials/deploy-evidence-from-breg.mdx b/docs/site/src/content/docs/tutorials/deploy-evidence-from-breg.mdx index 0cee282fd9..3b1dd26e9e 100644 --- a/docs/site/src/content/docs/tutorials/deploy-evidence-from-breg.mdx +++ b/docs/site/src/content/docs/tutorials/deploy-evidence-from-breg.mdx @@ -13,6 +13,9 @@ persona: - operator locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: operated target handoff; native composition and production build tests cover offline candidates, target-host checks need provisioned dependencies --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; diff --git a/docs/site/src/content/docs/tutorials/first-evidence-assertion.mdx b/docs/site/src/content/docs/tutorials/first-evidence-assertion.mdx index 5c674ba53d..cc19859e76 100644 --- a/docs/site/src/content/docs/tutorials/first-evidence-assertion.mdx +++ b/docs/site/src/content/docs/tutorials/first-evidence-assertion.mdx @@ -11,6 +11,8 @@ persona: - assertion provider locale: en standards_referenced: [] +tutorial_test: + toolset: evidence --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -52,7 +54,7 @@ released assertion contains only the governed answer and an opaque subject bindi Install the latest Evidence Gateway toolset: -```sh +```sh test-skip="installs the released toolset; the replay serves the binaries under test" curl -fsSL https://github.com/registrystack/registry-stack/releases/latest/download/evidencectl-install.sh | bash evidencectl --version ``` @@ -77,7 +79,7 @@ cd first-evidence-assertion Open `tutorial-source.openapi.yaml` in your editor and add this OpenAPI description: -```yaml +```yaml title="tutorial-source.openapi.yaml" test-file openapi: 3.1.0 info: title: Tutorial registry @@ -136,11 +138,11 @@ Three details connect this contract to Evidence Gateway: Start a write-free preview directly from the OpenAPI description: -```sh +```sh test-background="http://127.0.0.1:4010/people/person-123" evidencectl source mock serve --openapi tutorial-source.openapi.yaml ``` -```text +```text test-excerpt Source mock ready: mode=ephemeral origin=http://127.0.0.1:4010 contract=evidencectl-source-mock-v1 seed=0 asOf=2025-01-01 digest=sha256: served=1 skipped=0 Next: evidencectl source mock generate --openapi ``` @@ -155,7 +157,7 @@ curl -s http://127.0.0.1:4010/people/person-123 | python3 -c \ 'import json,sys; print(sorted(json.load(sys.stdin)))' ``` -```text +```text test-expect ['date_of_birth', 'name', 'person_id'] ``` @@ -201,7 +203,7 @@ Later institution integrations use the reusable source, selector, adapter, and s Open `questions/adult-status.yaml` in your editor and add the question definition: -```yaml +```yaml title="questions/adult-status.yaml" test-file id: adult-status question: Is the person at least 18 years old? purpose: age-check @@ -244,7 +246,7 @@ matching the purpose declared here. Open `derivations/adult-status.rhai` in your editor and add the answer logic. Rhai is the bounded scripting language Evidence Gateway uses for requirement-specific derivations: -```rhai +```rhai title="derivations/adult-status.rhai" test-file fn answer(facts, selectors, context) { let born = parse_date(required(facts.date_of_birth, "date_of_birth_missing")); let adult_on = add_calendar_years(born, 18); @@ -275,7 +277,7 @@ mkdir -p mocks/cases Open `mocks/source.yaml` and add the three request cases: -```yaml +```yaml title="mocks/source.yaml" test-file version: 1 openapi: ../source.openapi.yaml operations: @@ -304,7 +306,7 @@ Add the exact synthetic bodies. ### `mocks/cases/person-123.json` -```json +```json title="mocks/cases/person-123.json" test-file { "person_id": "person-123", "name": "Amina Example", @@ -314,7 +316,7 @@ Add the exact synthetic bodies. ### `mocks/cases/person-456.json` -```json +```json title="mocks/cases/person-456.json" test-file { "person_id": "person-456", "name": "Mateo Example", @@ -324,7 +326,7 @@ Add the exact synthetic bodies. ### `mocks/cases/person-789.json` -```json +```json title="mocks/cases/person-789.json" test-file { "person_id": "person-789", "name": "Noor Example", @@ -338,7 +340,7 @@ Check the plan and every authored body offline: evidencectl source mock check --config mocks/source.yaml ``` -```text +```text test-expect Mock plan valid: operations=1 cases=3 ``` @@ -350,11 +352,11 @@ checked bytes. Start the materialized source and leave it running: -```sh +```sh test-background="http://127.0.0.1:4010/people/person-123" evidencectl source mock serve --config mocks/source.yaml ``` -```text +```text test-excerpt Source mock ready: mode=materialized origin=http://127.0.0.1:4010 served=3 skipped=0 ``` @@ -364,7 +366,7 @@ Start Evidence Gateway and the pinned local issuer: evidencectl dev start . ``` -```text +```text test-excerpt Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -391,7 +393,7 @@ before a response exists. `evidencectl` separately asks the local issuer for sho authorization. It sends no HTTP request to Evidence Gateway and does not contact the registry. The command creates exactly these owner-only artifacts: -```text +```text test-expect Prepared request: .evidence/requests/first-assertion/request.json Prepared verification context: .evidence/requests/first-assertion/verification.json Prepared authorization: .evidence/requests/first-assertion/authorization.curl @@ -417,7 +419,7 @@ curl --silent --show-error --fail-with-body \ --write-out 'HTTP %{http_code}\n' ``` -```text +```text test-expect HTTP 200 ``` @@ -438,7 +440,7 @@ evidencectl verify assertion.jws.json \ --output verified.json ``` -```text +```text test-expect VERIFIED ``` @@ -451,13 +453,13 @@ python3 -m json.tool verified.json The verified document contains generated identifiers, timestamps, and a pseudonymous subject binding. The fields relevant to this question look like this excerpt: -```json +```json test-excerpt { "assuranceProfile": "local", "purpose": "age-check", "subjects": [ { - "binding": "urn:evidence:subject:v1_…", + "binding": "urn:evidence:subject:v1_", "role": "person" } ], @@ -496,7 +498,7 @@ evidencectl request prepare adult-status \ --name first-vc ``` -```text +```text test-expect Prepared request: .evidence/requests/first-vc/request.json Prepared verification context: .evidence/requests/first-vc/verification.json Prepared authorization: .evidence/requests/first-vc/authorization.curl @@ -516,7 +518,7 @@ curl --silent --show-error --fail-with-body \ --write-out 'HTTP %{http_code}\n' ``` -```text +```text test-expect HTTP 200 ``` @@ -528,7 +530,7 @@ evidencectl verify assertion.sd-jwt \ --output verified-vc.json ``` -```text +```text test-expect VERIFIED ``` @@ -553,7 +555,7 @@ Stop Evidence Gateway and the local issuer before reading the completed audit ch evidencectl dev stop ``` -```text +```text test-expect Local Evidence stopped ``` @@ -565,7 +567,7 @@ Verify the completed audit chain and show its last operation: evidencectl audit show --last-operation ``` -```text +```text test-expect ACCESS AUTHORIZED adult-status age-check requester= DISCLOSURE RELEASED is_adult ``` @@ -578,7 +580,7 @@ record or access token. Choose two different unused ports when you start the local services: -```sh +```sh test-skip="the alternative for busy ports; the replay uses the default ports" evidencectl dev --evidence-port 8180 --issuer-port 8181 start . ``` @@ -603,7 +605,7 @@ Remove the stopped local generation, including the sealed bundle that Evidence G evidencectl dev clean ``` -```text +```text test-expect Removed stopped local Evidence state ``` diff --git a/docs/site/src/content/docs/tutorials/integrate-evidence-candidate-with-docker-compose.mdx b/docs/site/src/content/docs/tutorials/integrate-evidence-candidate-with-docker-compose.mdx index 57db988992..aaf11e2adc 100644 --- a/docs/site/src/content/docs/tutorials/integrate-evidence-candidate-with-docker-compose.mdx +++ b/docs/site/src/content/docs/tutorials/integrate-evidence-candidate-with-docker-compose.mdx @@ -11,6 +11,9 @@ persona: - operator locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: drift-checked by evidence-production-build-docs.test.mjs; needs Docker Compose --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; diff --git a/docs/site/src/content/docs/tutorials/issue-a-birth-certificate-vc-from-opencrvs.mdx b/docs/site/src/content/docs/tutorials/issue-a-birth-certificate-vc-from-opencrvs.mdx index 663d4de18a..da312f7203 100644 --- a/docs/site/src/content/docs/tutorials/issue-a-birth-certificate-vc-from-opencrvs.mdx +++ b/docs/site/src/content/docs/tutorials/issue-a-birth-certificate-vc-from-opencrvs.mdx @@ -13,6 +13,9 @@ persona: - assertion provider locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: needs the public OpenCRVS Farajaland demo; live and opt-in, not replayed in CI --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; diff --git a/docs/site/src/content/docs/tutorials/issue-fhir-evidence-as-vcs.mdx b/docs/site/src/content/docs/tutorials/issue-fhir-evidence-as-vcs.mdx index e27a31d7b5..5d617652fa 100644 --- a/docs/site/src/content/docs/tutorials/issue-fhir-evidence-as-vcs.mdx +++ b/docs/site/src/content/docs/tutorials/issue-fhir-evidence-as-vcs.mdx @@ -13,6 +13,8 @@ locale: en standards_referenced: - fhir-r4 - sd-jwt-vc +tutorial_test: + toolset: evidence --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -73,7 +75,7 @@ record whose beneficiary resolves to a Patient that exists, and one active Organ healthcare provider. It writes only their selectors to two owner-only local files, and prints neither identifiers nor FHIR resources: -```python +```python title="discover-fhir-records.py" test-file import json import os import re @@ -244,7 +246,7 @@ print("Organization selector file: ready") python3 discover-fhir-records.py ``` -```text +```text test-expect Coverage selector file: ready Organization selector file: ready ``` @@ -264,7 +266,7 @@ selectors never reach a log file. Create `fhir-read-through.py`: -```python +```python title="fhir-read-through.py" test-file import json import os import re @@ -383,7 +385,7 @@ done curl --silent --show-error --fail http://127.0.0.1:8000/healthz ``` -```text +```text test-expect ready ``` @@ -402,7 +404,7 @@ origin, paths, response media type, projected fields, and bounds are all stated Save this reviewed subset as `fhir-smart-r4.openapi.yaml`: -```yaml +```yaml title="fhir-smart-r4.openapi.yaml" test-file openapi: 3.1.0 info: title: SMART Health IT public FHIR R4 tutorial subset @@ -488,7 +490,7 @@ evidencectl init fhir-record-evidence \ cd fhir-record-evidence ``` -```text +```text test-excerpt Created an editable OpenAPI authoring project in fhir-record-evidence ``` @@ -500,7 +502,7 @@ authoring only; a deployment must use a reviewed authenticated HTTPS source. Create `questions/fhir-coverage-status.yaml`: -```yaml +```yaml title="questions/fhir-coverage-status.yaml" test-file id: fhir-coverage-status question: Does this coverage record report active coverage for the selected patient? purpose: coverage-record-verification @@ -534,7 +536,7 @@ Patient resource. Create `derivations/fhir-coverage-status.rhai`: -```rhai +```rhai title="derivations/fhir-coverage-status.rhai" test-file fn answer(facts, selectors, context) { let coverage_id = selectors["coverage-record"]["values"]["coverage_id"]; let patient_id = selectors["patient"]["values"]["patient_id"]; @@ -560,7 +562,7 @@ the payor, subscriber identifier, coverage class, period, and every other source Create `questions/fhir-healthcare-establishment.yaml`: -```yaml +```yaml title="questions/fhir-healthcare-establishment.yaml" test-file id: fhir-healthcare-establishment question: Does this organization record report an active healthcare provider? purpose: healthcare-establishment-verification @@ -589,7 +591,7 @@ disclosure: Create `derivations/fhir-healthcare-establishment.rhai`: -```rhai +```rhai title="derivations/fhir-healthcare-establishment.rhai" test-file fn answer(facts, selectors, context) { let organization_id = selectors["organization"]["values"]["organization_id"]; if required(facts.resource_id, "organization_id_missing") != organization_id { @@ -621,7 +623,7 @@ Compile the two questions and start Evidence Gateway with its stock identity pro evidencectl dev start . ``` -```text +```text test-excerpt Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -644,7 +646,7 @@ evidencectl request prepare fhir-coverage-status \ --name fhir-coverage-vc ``` -```text +```text test-expect Prepared request: .evidence/requests/fhir-coverage-vc/request.json Prepared verification context: .evidence/requests/fhir-coverage-vc/verification.json Prepared authorization: .evidence/requests/fhir-coverage-vc/authorization.curl @@ -672,7 +674,7 @@ evidencectl verify fhir-coverage.sd-jwt \ --output fhir-coverage.verified.json ``` -```text +```text test-expect HTTP 200 VERIFIED ``` @@ -685,7 +687,7 @@ python3 -m json.tool fhir-coverage.verified.json The verified payload carries one supported value: -```json +```json test-excerpt { "providesValueFor": "urn:registrystack:evidence:local:concept:fhir-coverage-status:coverage_record_reports_active", "value": true @@ -722,8 +724,10 @@ evidencectl verify fhir-healthcare-establishment.sd-jwt \ --output fhir-healthcare-establishment.verified.json ``` -```text +```text test-expect Prepared request: .evidence/requests/fhir-healthcare-establishment-vc/request.json +Prepared verification context: .evidence/requests/fhir-healthcare-establishment-vc/verification.json +Prepared authorization: .evidence/requests/fhir-healthcare-establishment-vc/authorization.curl HTTP 200 VERIFIED ``` @@ -751,6 +755,13 @@ rm -f \ fhir-organization-subjects.json ``` +```text test-expect +Local Evidence stopped +ACCESS AUTHORIZED fhir-healthcare-establishment healthcare-establishment-verification requester= +DISCLOSURE RELEASED healthcare_provider_record_active +Removed stopped local Evidence state +``` + The audit names the authorized question, the purpose it was authorized under, the requester pseudonym, and the concept that was released. Read it for what it leaves out: no selected FHIR identifier, no source resource, and not even the boolean value itself. An operator diff --git a/docs/site/src/content/docs/tutorials/issue-immunization-evidence-from-dhis2.mdx b/docs/site/src/content/docs/tutorials/issue-immunization-evidence-from-dhis2.mdx index e63805502e..147819b493 100644 --- a/docs/site/src/content/docs/tutorials/issue-immunization-evidence-from-dhis2.mdx +++ b/docs/site/src/content/docs/tutorials/issue-immunization-evidence-from-dhis2.mdx @@ -11,6 +11,9 @@ persona: - assertion provider locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: needs the public DHIS2 demo; live and opt-in, not replayed in CI --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; diff --git a/docs/site/src/content/docs/tutorials/manage-evidence-verifier-trust.mdx b/docs/site/src/content/docs/tutorials/manage-evidence-verifier-trust.mdx index 0688427f98..03b79551bd 100644 --- a/docs/site/src/content/docs/tutorials/manage-evidence-verifier-trust.mdx +++ b/docs/site/src/content/docs/tutorials/manage-evidence-verifier-trust.mdx @@ -11,6 +11,9 @@ persona: - consumer or verifier locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: how-to against the reader's own deployment; no fixed scenario to replay --- A valid signature proves control of a private key. Your consumer still decides which provider, diff --git a/docs/site/src/content/docs/tutorials/move-evidence-to-production-signing.mdx b/docs/site/src/content/docs/tutorials/move-evidence-to-production-signing.mdx index 2d59881126..99035780d2 100644 --- a/docs/site/src/content/docs/tutorials/move-evidence-to-production-signing.mdx +++ b/docs/site/src/content/docs/tutorials/move-evidence-to-production-signing.mdx @@ -11,6 +11,9 @@ persona: - operator locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: drift-checked by evidence-production-build-docs.test.mjs; needs a Transit signer --- Use this procedure to move Evidence Gateway from disposable local signing to a strict deployment. diff --git a/docs/site/src/content/docs/tutorials/prove-an-evidence-project.mdx b/docs/site/src/content/docs/tutorials/prove-an-evidence-project.mdx index 04b0ad9a6c..b192763344 100644 --- a/docs/site/src/content/docs/tutorials/prove-an-evidence-project.mdx +++ b/docs/site/src/content/docs/tutorials/prove-an-evidence-project.mdx @@ -12,6 +12,9 @@ persona: - operator locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: how-to against the reader's own project; no fixed scenario to replay --- Start with an editable project: the directory `evidencectl init` created, holding `questions/`, diff --git a/docs/site/src/content/docs/tutorials/refuse-unsafe-evidence-requests.mdx b/docs/site/src/content/docs/tutorials/refuse-unsafe-evidence-requests.mdx index 42d2366214..f6054387d4 100644 --- a/docs/site/src/content/docs/tutorials/refuse-unsafe-evidence-requests.mdx +++ b/docs/site/src/content/docs/tutorials/refuse-unsafe-evidence-requests.mdx @@ -12,6 +12,9 @@ persona: - consumer or verifier locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + after: tutorials/return-a-governed-value --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -42,18 +45,18 @@ you completed that tutorial. Without it, preparation stops with In one terminal, return to the `first-evidence-assertion` directory and serve the checked cases: -```sh +```sh test-cwd="first-evidence-assertion" test-background="http://127.0.0.1:4010/people/person-123" evidencectl source mock serve --config adult-status/mocks/source.yaml ``` In another terminal, enter the existing project and start a fresh local generation: -```sh +```sh test-cwd="first-evidence-assertion" cd adult-status evidencectl dev start . ``` -```text +```text test-excerpt Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -106,7 +109,7 @@ curl --silent --show-error \ --write-out 'HTTP %{http_code}\n' ``` -```text +```text test-expect HTTP 403 ``` @@ -116,7 +119,15 @@ Inspect the public problem: python3 -m json.tool unauthorized-response.json ``` -The response identifies only the closed `evidence.denied` problem and public trace ID. It +The response identifies only the closed `evidence.denied` problem and public trace ID: + +```json test-excerpt +{ + "code": "evidence.denied" +} +``` + +It does not reveal source facts, selector values, grants, or credentials. The registry terminal shows no new `GET /people/person-123` because authorization failed before source access. @@ -143,7 +154,7 @@ evidencectl verify authorized-response.jws.json \ --output authorized-response.verified.json ``` -```text +```text test-expect VERIFIED ``` @@ -169,13 +180,13 @@ PY Try to verify the changed response: -```sh +```sh test-exit="1" evidencectl verify tampered-response.jws.json \ --context .evidence/requests/refusal-check/verification.json \ --output tampered-response.verified.json ``` -```text +```text test-expect evidencectl: Evidence response verification failed ``` @@ -193,7 +204,7 @@ evidencectl dev stop evidencectl dev clean ``` -```text +```text test-expect Local Evidence stopped Removed stopped local Evidence state ``` diff --git a/docs/site/src/content/docs/tutorials/request-a-holder-bound-credential.mdx b/docs/site/src/content/docs/tutorials/request-a-holder-bound-credential.mdx index 08b6a2f777..f6e32c7c16 100644 --- a/docs/site/src/content/docs/tutorials/request-a-holder-bound-credential.mdx +++ b/docs/site/src/content/docs/tutorials/request-a-holder-bound-credential.mdx @@ -14,6 +14,9 @@ locale: en standards_referenced: - sd-jwt-vc draft: true +tutorial_test: + toolset: evidence + skip: draft, hidden from the sidebar; no verified wallet flow exists to replay --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; diff --git a/docs/site/src/content/docs/tutorials/request-evidence-as-sd-jwt-vc.mdx b/docs/site/src/content/docs/tutorials/request-evidence-as-sd-jwt-vc.mdx index c78ba04fb5..109ad6f60b 100644 --- a/docs/site/src/content/docs/tutorials/request-evidence-as-sd-jwt-vc.mdx +++ b/docs/site/src/content/docs/tutorials/request-evidence-as-sd-jwt-vc.mdx @@ -13,6 +13,9 @@ persona: locale: en standards_referenced: - sd-jwt-vc +tutorial_test: + toolset: evidence + after: tutorials/first-evidence-assertion --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -53,13 +56,13 @@ concept URI. Omitting `responseFormats` keeps the project at signed JWS only. In one terminal, return to the `first-evidence-assertion` directory and serve the checked cases again: -```sh +```sh test-cwd="first-evidence-assertion" test-background="http://127.0.0.1:4010/people/person-123" evidencectl source mock serve --config adult-status/mocks/source.yaml ``` Leave it running. In another terminal, enter the existing Evidence Gateway project: -```sh +```sh test-cwd="first-evidence-assertion" cd adult-status ``` @@ -80,7 +83,7 @@ Start a fresh local generation: evidencectl dev start . ``` -```text +```text test-excerpt Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -110,7 +113,7 @@ curl --silent --show-error --fail-with-body \ --write-out 'HTTP %{http_code}\n' ``` -```text +```text test-expect HTTP 200 ``` @@ -123,7 +126,7 @@ evidencectl verify scalar.sd-jwt \ --output scalar.verified.json ``` -```text +```text test-expect VERIFIED ``` @@ -156,7 +159,7 @@ for disclosure in (part for part in parts[1:] if part): PY ``` -```text +```text test-expect typ: dc+sd-jwt vct: urn:registrystack:evidence:local:evidence-type:adult-status disclosure: urn:registrystack:evidence:local:concept:adult-status:is_adult @@ -200,13 +203,13 @@ PY Verification must fail and must not create trusted output: -```sh +```sh test-exit="1" evidencectl verify scalar-tampered.sd-jwt \ --context .evidence/requests/scalar-vc/verification.json \ --output scalar-tampered.verified.json ``` -```text +```text test-expect evidencectl: Evidence response verification failed ``` @@ -221,7 +224,7 @@ evaluated against. This is a second governed question, not a request-time option Create `schemas/adult-assessment.yaml`: -```yaml +```yaml title="schemas/adult-assessment.yaml" test-file $schema: https://json-schema.org/draft/2020-12/schema $id: urn:registrystack:evidence:local:schema:adult-assessment:v1 type: object @@ -237,7 +240,7 @@ properties: Create `questions/adult-assessment.yaml`: -```yaml +```yaml title="questions/adult-assessment.yaml" test-file id: adult-assessment question: What adult assessment applies to this person? purpose: age-assessment-review @@ -271,7 +274,7 @@ object would remain one atomic direct-field disclosure. Create `derivations/adult-assessment.rhai`: -```rhai +```rhai title="derivations/adult-assessment.rhai" test-file fn answer(facts, selectors, context) { let born = parse_date(required(facts.date_of_birth, "date_of_birth_missing")); let adult_on = add_calendar_years(born, 18); @@ -326,7 +329,7 @@ evidencectl verify structured.sd-jwt \ --output structured.verified.json ``` -```text +```text test-expect VERIFIED ``` @@ -349,7 +352,7 @@ for name in sorted(names): PY ``` -```text +```text test-expect disclosure: criterion disclosure: isAdult ``` @@ -369,7 +372,7 @@ evidencectl audit show --last-operation evidencectl dev clean ``` -```text +```text test-expect Local Evidence stopped ACCESS AUTHORIZED adult-assessment age-assessment-review requester= DISCLOSURE RELEASED adult_assessment diff --git a/docs/site/src/content/docs/tutorials/request-evidence-from-an-application.mdx b/docs/site/src/content/docs/tutorials/request-evidence-from-an-application.mdx index 5d884e8761..5c7975d4b5 100644 --- a/docs/site/src/content/docs/tutorials/request-evidence-from-an-application.mdx +++ b/docs/site/src/content/docs/tutorials/request-evidence-from-an-application.mdx @@ -11,6 +11,9 @@ persona: - consumer or verifier locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + after: tutorials/first-evidence-assertion --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -62,7 +65,7 @@ library refuses every response that does not match it. Enter the existing project: -```sh +```sh test-cwd="first-evidence-assertion" cd adult-status ``` @@ -76,7 +79,7 @@ evidencectl access client add age-check-app \ --generate-local-key ``` -```text +```text test-expect Added access policy app-age-checks for adult-status. Added client age-check-app with policy app-age-checks. ``` @@ -96,7 +99,7 @@ state for itself: grep evidenceAudience access/clients/age-check-app.yaml ``` -```text +```text test-expect evidenceAudience: urn:registrystack:evidence:local:client:age-check-app ``` @@ -114,7 +117,7 @@ from the project's own retained public signing key: evidencectl jwks --output trusted-issuer-keys.json secrets/signing-p256-public.jwk.json ``` -```text +```text test-expect wrote trusted-issuer-keys.json ``` @@ -130,7 +133,7 @@ The client is the `evidence` namespace of the maintained `registry-stack-client` version from the runtime you installed rather than typing one, and install the client at that exact version. From inside the project directory: -```sh +```sh test-skip="the gate provides the client package built from this checkout, which is not published yet" version="$(evidencectl --version | awk '{print $2}')" python3 -m venv .venv . .venv/bin/activate @@ -157,11 +160,11 @@ Evidence reads the source record through the materialized source mock from the f cleanup told you to stop it. In another terminal, return to the `first-evidence-assertion` directory and serve the same checked cases again. Leave it running: -```sh +```sh test-cwd="first-evidence-assertion" test-background="http://127.0.0.1:4010/people/person-123" evidencectl source mock serve --config adult-status/mocks/source.yaml ``` -```text +```text test-excerpt Source mock ready: mode=materialized origin=http://127.0.0.1:4010 served=3 skipped=0 ``` @@ -172,7 +175,7 @@ start Evidence and the pinned local issuer: evidencectl dev start . ``` -```text +```text test-excerpt Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -183,14 +186,14 @@ later, as a failed evidence request. The access policy is now part of the running generation, so a terminal request names a client too. Confirm that the project no longer accepts an unnamed one: -```sh +```sh test-exit="1" evidencectl request prepare adult-status \ --purpose age-check \ --subject person_id=person-123 \ --name unnamed-caller ``` -```text +```text test-expect evidencectl: the active project requires a registered client selected with --client ``` @@ -231,25 +234,33 @@ print(document) PY ``` -```json +```json test-expect { "assuranceProfile": "local", + "audience": "urn:registrystack:evidence:local:client:age-check-app", "definitions": [ { "concepts": [ { + "concept": "urn:registrystack:evidence:local:concept:adult-status:is_adult", "form": "boolean", - "id": "urn:registrystack:evidence:local:concept:adult-status:is_adult" + "handle": "is_adult", + "required": true } ], "configurationRevision": "sha256:", "evidenceType": "urn:registrystack:evidence:local:evidence-type:adult-status", + "handle": "adult-status", "kind": "criterion", "purpose": "age-check", "referenceFrameworks": [ "urn:registrystack:evidence:local:framework:adult-status" ], "requirement": "urn:registrystack:evidence:local:requirement:adult-status", + "responseFormats": [ + "signed-jws", + "sd-jwt-vc" + ], "subjects": [ { "cardinality": "one", @@ -377,7 +388,7 @@ print(document) PY ``` -```json +```json test-expect { "audience": "urn:registrystack:evidence:local:client:age-check-app", "clock_skew_seconds": 30, @@ -454,7 +465,7 @@ must be asked, moved. Open `age_check.py` in your editor and add the application. It loads the pinned procedure and never calls discovery again: -```python +```python title="age_check.py" test-file import fcntl import json import os @@ -591,7 +602,7 @@ umask 077 python3 age_check.py ``` -```text +```text test-expect person-123 is_adult=True pinned binding recorded in subject-bindings.json ``` @@ -603,7 +614,7 @@ again: python3 age_check.py ``` -```text +```text test-expect person-123 is_adult=True pinned binding recorded in subject-bindings.json ``` @@ -628,7 +639,7 @@ Ask about a different record: python3 age_check.py person-456 ``` -```text +```text test-expect person-456 is_adult=False pinned binding recorded in subject-bindings.json ``` @@ -640,7 +651,7 @@ The registry holds a name and a date of birth for both people. Neither answer co Change one stored binding to prove that the application, not the deployment, decides what it accepts: -```sh +```sh test-exit="1" python3 - <<'PY' import json from pathlib import Path @@ -652,7 +663,7 @@ PY python3 age_check.py person-123 ``` -```text +```text test-expect unverifiable response, nothing read (policy): the Evidence response failed verification: Evidence payload does not match the relying procedure ``` @@ -695,7 +706,7 @@ evidencectl dev stop evidencectl dev clean ``` -```text +```text test-expect Local Evidence stopped Removed stopped local Evidence state ``` diff --git a/docs/site/src/content/docs/tutorials/return-a-governed-value.mdx b/docs/site/src/content/docs/tutorials/return-a-governed-value.mdx index c4a55820e0..6d0c1a90a0 100644 --- a/docs/site/src/content/docs/tutorials/return-a-governed-value.mdx +++ b/docs/site/src/content/docs/tutorials/return-a-governed-value.mdx @@ -11,6 +11,9 @@ persona: - assertion provider locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + after: tutorials/first-evidence-assertion --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -52,7 +55,7 @@ age. In one terminal, return to the `first-evidence-assertion` directory and serve the same checked cases again: -```sh +```sh test-cwd="first-evidence-assertion" test-background="http://127.0.0.1:4010/people/person-123" evidencectl source mock serve --config adult-status/mocks/source.yaml ``` @@ -64,7 +67,7 @@ registry lookup from the first tutorial. In another terminal, enter the existing project: -```sh +```sh test-cwd="first-evidence-assertion" cd adult-status ``` @@ -72,7 +75,7 @@ cd adult-status Create a second question definition alongside `questions/adult-status.yaml`: -```yaml +```yaml title="questions/age-bracket.yaml" test-file id: age-bracket question: Which age bracket does this person belong to? purpose: service-path-selection @@ -103,7 +106,7 @@ codelist enforced by the Evidence Gateway runtime. Create this new derivation alongside `derivations/adult-status.rhai`: -```rhai +```rhai title="derivations/age-bracket.rhai" test-file fn answer(facts, selectors, context) { let born = parse_date(required(facts.date_of_birth, "date_of_birth_missing")); if compare_dates(context.legal_local_date, add_calendar_years(born, 18)) < 0 { @@ -129,7 +132,7 @@ Capture the edited project in a new immutable local generation: evidencectl dev start . ``` -```text +```text test-excerpt Evidence ready at http://127.0.0.1:8080 Issuer ready at http://127.0.0.1:8081 ``` @@ -170,7 +173,7 @@ evidencectl verify age-bracket.jws.json \ --output age-bracket.verified.json ``` -```text +```text test-expect VERIFIED ``` @@ -182,7 +185,7 @@ python3 -m json.tool age-bracket.verified.json The relevant supported value has this shape: -```json +```json test-excerpt { "providesValueFor": "urn:registrystack:evidence:local:concept:age-bracket:age_bracket", "value": "under-18" @@ -210,7 +213,7 @@ Inspect the last verified operation: evidencectl audit show --last-operation ``` -```text +```text test-expect ACCESS AUTHORIZED age-bracket service-path-selection requester= DISCLOSURE RELEASED age_bracket ``` @@ -232,7 +235,7 @@ Return to the registry terminal and press `Ctrl+C`. the first tutorial's services were left running. Stop and remove that session, then start the new generation: -```sh +```sh test-skip="recovers a session the first tutorial left running; the replay stopped it" evidencectl dev stop evidencectl dev clean ``` diff --git a/docs/site/src/content/docs/tutorials/rotate-evidence-signing-keys.mdx b/docs/site/src/content/docs/tutorials/rotate-evidence-signing-keys.mdx index fc9cc55ee1..4a91ffc77d 100644 --- a/docs/site/src/content/docs/tutorials/rotate-evidence-signing-keys.mdx +++ b/docs/site/src/content/docs/tutorials/rotate-evidence-signing-keys.mdx @@ -11,6 +11,9 @@ persona: - operator locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: drift-checked by evidence-production-build-docs.test.mjs; needs a deployed signing key --- Use this procedure after diff --git a/docs/site/src/content/docs/tutorials/run-oid4vci-interoperability-checks.mdx b/docs/site/src/content/docs/tutorials/run-oid4vci-interoperability-checks.mdx index 5c043d29ec..a2d64cf508 100644 --- a/docs/site/src/content/docs/tutorials/run-oid4vci-interoperability-checks.mdx +++ b/docs/site/src/content/docs/tutorials/run-oid4vci-interoperability-checks.mdx @@ -13,6 +13,9 @@ locale: en standards_referenced: - oid4vci - sd-jwt-vc +tutorial_test: + toolset: evidence + checkout: true --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -55,7 +58,7 @@ mkdir -p .tutorial/oid4vci-adopter Copy this complete loopback configuration exactly: -```yaml +```yaml title=".tutorial/oid4vci-adopter/oid4vci.yaml" test-file version: 1 validationMode: supervised-local-development credentialIssuer: http://127.0.0.1:18440 @@ -100,7 +103,7 @@ EVIDENCE_OID4VCI_ADOPTER_ROOT="$PWD/.tutorial/oid4vci-adopter" \ The successful run ends with this exact receipt: -```text +```text test-excerpt PASS: sanitized Inji OID4VCI profile and Registry-side interoperability tests ``` @@ -121,12 +124,12 @@ Before those compatibility cases, the runner invokes the actual `evidence-oid4vc listener, completes an authorized wallet flow through the published metadata, and verifies the returned holder-bound presentation independently. Its safe milestones include: -```text -CONFIG COPIED: complete configuration has no untracked inputs +```text test-excerpt CONFIG CHECKED: complete delivery configuration is valid METADATA INSPECTED: derived holder-bound batch ceiling is 4 SERVICE READY: health and readiness are available on the delivery listener METRICS PRIVATE: metrics exist only on the separate loopback listener +TASK GRANT REFUSED: deferred wallet state was not created PRESENTATION VERIFIED: public wallet flow returned holder-bound Evidence CLEANUP COMPLETE: generated private material was removed ``` @@ -168,7 +171,7 @@ Simulator. Before running it, install Java 17, Android SDK platform 34 with Buil full Xcode, and an iPhone 15 simulator. The runner requires `git`, `npm`, `java`, `xcodebuild`, and `xcrun` on `PATH`, plus network access. -```sh +```sh test-skip="the upstream runner needs macOS with Xcode and the iOS Simulator, Java, the Android SDK, and network access" EVIDENCE_INJI_OID4VCI=1 products/evidence/scripts/compat/inji-oid4vci-upstream.sh ``` diff --git a/docs/site/src/content/docs/tutorials/verify-a-registered-parent-with-opencrvs.mdx b/docs/site/src/content/docs/tutorials/verify-a-registered-parent-with-opencrvs.mdx index 3c24c37467..c262286db1 100644 --- a/docs/site/src/content/docs/tutorials/verify-a-registered-parent-with-opencrvs.mdx +++ b/docs/site/src/content/docs/tutorials/verify-a-registered-parent-with-opencrvs.mdx @@ -13,6 +13,9 @@ persona: - assertion provider locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + skip: needs the public OpenCRVS Farajaland demo; live and opt-in, not replayed in CI --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; diff --git a/docs/site/src/content/docs/tutorials/verify-an-assertion-as-a-consumer.mdx b/docs/site/src/content/docs/tutorials/verify-an-assertion-as-a-consumer.mdx index 993eb2a886..f28e2b6116 100644 --- a/docs/site/src/content/docs/tutorials/verify-an-assertion-as-a-consumer.mdx +++ b/docs/site/src/content/docs/tutorials/verify-an-assertion-as-a-consumer.mdx @@ -11,6 +11,9 @@ persona: - consumer or verifier locale: en standards_referenced: [] +tutorial_test: + toolset: evidence + after: tutorials/first-evidence-assertion --- import QuickstartMeta from '../../../components/QuickstartMeta.astro'; @@ -35,7 +38,7 @@ trust and request expectations support later offline review. Enter the existing project. No service or registry needs to be running: -```sh +```sh test-cwd="first-evidence-assertion" cd adult-status ``` @@ -88,15 +91,24 @@ evidence verify \ The result begins: -```text +```text test-excerpt verified-at: authentic: yes currently-valid: yes ``` -The command then prints the verified Evidence Gateway payload. It opens no listener, contacts no issuer, -fetches no discovery document, and calls no source. The named JWKS file is the complete trust set -for this verification. +The command then prints the verified Evidence Gateway payload, which carries the same answer the first +tutorial verified: + +```json test-excerpt +{ + "providesValueFor": "urn:registrystack:evidence:local:concept:adult-status:is_adult", + "value": true +} +``` + +It opens no listener, contacts no issuer, fetches no discovery document, and calls no source. The +named JWKS file is the complete trust set for this verification. `authentic` and `currently-valid` answer different questions. An expired stored response can remain authentic evidence of what was signed and accepted at an earlier decision time. It must not be used