feat(docs): replay the Evidence tutorials with the page runner

.github/scripts/ci_changes.py

@@ -178,46 +178,45 @@
     "docs/site/src/content/docs/tutorials/publish-governed-sqlite-registry.mdx",
 )
 
-# Every input the Evidence tutorial gate replays or is built from. The tutorial
-# pages and helper scripts here must stay in step with the gate's own registry
-# and the helpers it invokes, which test_ci_changes.py enforces: a tutorial or
-# helper CI does not watch is one that rots silently.
-EVIDENCE_TUTORIAL_INPUTS = frozenset(
-    {
-        "Cargo.lock",
-        "Cargo.toml",
-        "docs/site/package-lock.json",
-        "docs/site/package.json",
-        "docs/site/scripts/check-evidence-tutorials.sh",
-        "docs/site/scripts/check-evidence-tutorials.test.mjs",
-        "docs/site/scripts/evidence-tutorial-fence.sh",
-        "docs/site/scripts/fixtures/fhir-tutorial-mock.py",
-        "docs/site/src/content/docs/tutorials/assert-a-role-bound-relationship.mdx",
-        "docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx",
-        "docs/site/src/content/docs/tutorials/control-who-can-request-evidence.mdx",
-        "docs/site/src/content/docs/tutorials/first-evidence-assertion.mdx",
-        "docs/site/src/content/docs/tutorials/issue-fhir-evidence-as-vcs.mdx",
-        "docs/site/src/content/docs/tutorials/refuse-unsafe-evidence-requests.mdx",
-        "docs/site/src/content/docs/tutorials/request-evidence-as-sd-jwt-vc.mdx",
-        "docs/site/src/content/docs/tutorials/request-evidence-from-an-application.mdx",
-        "docs/site/src/content/docs/tutorials/run-oid4vci-interoperability-checks.mdx",
-        "docs/site/src/content/docs/tutorials/return-a-governed-value.mdx",
-        "docs/site/src/content/docs/tutorials/verify-an-assertion-as-a-consumer.mdx",
-        "products/evidence/fixtures/interoperability/inji-oid4vci/profile.json",
-        "products/evidence/fixtures/interoperability/inji-oid4vci/receipt.json",
-        "products/evidence/scripts/compat/inji-oid4vci-upstream.sh",
-        "products/evidence/scripts/compat/inji-oid4vci.sh",
-        # The application tutorial imports the maintained client package, and
-        # the job assembles that package from this commit with these scripts
-        # and this pinned build tool. A change to any of them changes what the
-        # replay imports.
-        "release/requirements/maturin-1.9.6.txt",
-        "release/scripts/assemble-registry-client-packages.py",
-        "release/scripts/assemble-registry-client-wheel.py",
-        "release/scripts/build-linux-python-client",
-        "release/scripts/zig-glibc-compiler",
-        "release/scripts/smoke-registry-client-package.py",
-    }
+# Every input the Evidence tutorial gate replays or is built from: the page
+# runner, the pages whose frontmatter it replays, the source mock the toolset
+# starts, and the build inputs of what the pages run. The replayed pages here
+# must stay in step with their tutorial_test frontmatter, which
+# test_ci_changes.py enforces: a tutorial CI does not watch is one that rots
+# silently.
+EVIDENCE_TUTORIAL_INPUTS = (
+    "Cargo.lock",
+    "Cargo.toml",
+    "docs/site/package-lock.json",
+    "docs/site/package.json",
+    "docs/site/scripts/run-tutorial.mjs",
+    "docs/site/scripts/tutorial-runner/**",
+    "docs/site/scripts/fixtures/fhir-tutorial-mock.py",
+    "docs/site/src/content/docs/tutorials/assert-a-role-bound-relationship.mdx",
+    "docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx",
+    "docs/site/src/content/docs/tutorials/control-who-can-request-evidence.mdx",
+    "docs/site/src/content/docs/tutorials/first-evidence-assertion.mdx",
+    "docs/site/src/content/docs/tutorials/issue-fhir-evidence-as-vcs.mdx",
+    "docs/site/src/content/docs/tutorials/refuse-unsafe-evidence-requests.mdx",
+    "docs/site/src/content/docs/tutorials/request-evidence-as-sd-jwt-vc.mdx",
+    "docs/site/src/content/docs/tutorials/request-evidence-from-an-application.mdx",
+    "docs/site/src/content/docs/tutorials/run-oid4vci-interoperability-checks.mdx",
+    "docs/site/src/content/docs/tutorials/return-a-governed-value.mdx",
+    "docs/site/src/content/docs/tutorials/verify-an-assertion-as-a-consumer.mdx",
+    "products/evidence/fixtures/interoperability/inji-oid4vci/profile.json",
+    "products/evidence/fixtures/interoperability/inji-oid4vci/receipt.json",
+    "products/evidence/scripts/compat/inji-oid4vci-upstream.sh",
+    "products/evidence/scripts/compat/inji-oid4vci.sh",
+    # The application tutorial imports the maintained client package, and
+    # the job assembles that package from this commit with these scripts
+    # and this pinned build tool. A change to any of them changes what the
+    # replay imports.
+    "release/requirements/maturin-1.9.6.txt",
+    "release/scripts/assemble-registry-client-packages.py",
+    "release/scripts/assemble-registry-client-wheel.py",
+    "release/scripts/build-linux-python-client",
+    "release/scripts/zig-glibc-compiler",
+    "release/scripts/smoke-registry-client-package.py",
 )
 
 # Every input the Base Registry Engine tutorial gate replays or is built from:
@@ -1216,7 +1215,7 @@ def classify(
 
     evidence_tutorial = (
         complete
-        or any(path in EVIDENCE_TUTORIAL_INPUTS for path in paths)
+        or any(matches(path, *EVIDENCE_TUTORIAL_INPUTS) for path in paths)
         or bool(
             affected & (EVIDENCE_TUTORIAL_PACKAGES | ASSEMBLED_PYTHON_CLIENT_PACKAGES)
         )

.github/scripts/test_ci_changes.py

@@ -829,52 +829,35 @@ def test_manifest_core_changes_select_breg_through_linked_code(
         self.assertIn("registry-breg", outputs["rust_packages"])
         self.assertIn("registry-manifest-core", outputs["rust_packages"])
 
-    def test_evidence_tutorial_inputs_cover_every_registered_tutorial(self) -> None:
-        # The gate's registry is the source of truth for which tutorials exist.
-        # A tutorial missing here would not trigger the job that replays it, so
-        # it could break without any pull request noticing.
-        gate = (
-            Path(__file__).resolve().parents[2]
-            / "docs/site/scripts/check-evidence-tutorials.sh"
-        )
-        registry = re.search(
-            r"^EVIDENCE_TUTORIALS=\((.*?)^\)", gate.read_text(), re.DOTALL | re.MULTILINE
-        )
-        if registry is None:
-            self.fail("the gate must declare EVIDENCE_TUTORIALS")
-        slugs = registry.group(1).split()
-        self.assertTrue(slugs, "the gate must register at least one tutorial")
+    def test_evidence_tutorial_inputs_cover_every_replayed_tutorial(self) -> None:
+        # Each page's tutorial_test frontmatter is the source of truth for
+        # which tutorials the gate replays. A replayed page missing here would
+        # not trigger the job that replays it, so it could break without any
+        # pull request noticing.
+        docs = Path(__file__).resolve().parents[2] / "docs/site/src/content/docs"
+        slugs = []
+        for section in ("start", "tutorials"):
+            for page in sorted((docs / section).glob("*.mdx")):
+                frontmatter = yaml.safe_load(page.read_text().split("---\n")[1])
+                declaration = frontmatter.get("tutorial_test") or {}
+                if declaration.get("toolset") == "evidence" and "skip" not in declaration:
+                    slugs.append(f"{section}/{page.stem}")
+        self.assertIn("tutorials/first-evidence-assertion", slugs)
         for slug in slugs:
             with self.subTest(slug=slug):
-                self.assertIn(
-                    f"docs/site/src/content/docs/tutorials/{slug}.mdx",
-                    EVIDENCE_TUTORIAL_INPUTS,
+                page = f"docs/site/src/content/docs/{slug}.mdx"
+                self.assertTrue(
+                    any(
+                        fnmatch.fnmatchcase(page, pattern)
+                        for pattern in EVIDENCE_TUTORIAL_INPUTS
+                    )
                 )
 
-    def test_evidence_tutorial_inputs_cover_every_helper_the_gate_invokes(self) -> None:
-        # Same reasoning as the tutorial registry above, one layer down. The gate
-        # delegates to sibling scripts, and a change to one of those changes what
-        # every tutorial replay does. A helper missing here routes the change
-        # past the job that would have caught it.
-        gate = (
-            Path(__file__).resolve().parents[2]
-            / "docs/site/scripts/check-evidence-tutorials.sh"
-        )
-        helpers = set(
-            re.findall(
-                r"\$SITE_ROOT/scripts/([A-Za-z0-9._/-]+\.(?:mjs|py|sh))",
-                gate.read_text(),
-            )
-        )
-        self.assertTrue(helpers, "the gate must invoke at least one helper")
-        for helper in sorted(helpers):
-            with self.subTest(helper=helper):
-                self.assertIn(f"docs/site/scripts/{helper}", EVIDENCE_TUTORIAL_INPUTS)
-
     def test_evidence_tutorial_routing(self) -> None:
         infrastructure = (
-            "docs/site/scripts/check-evidence-tutorials.sh",
-            "docs/site/scripts/check-evidence-tutorials.test.mjs",
+            "docs/site/scripts/run-tutorial.mjs",
+            "docs/site/scripts/tutorial-runner/toolsets.mjs",
+            "docs/site/scripts/fixtures/fhir-tutorial-mock.py",
             "docs/site/src/content/docs/tutorials/first-evidence-assertion.mdx",
             "docs/site/package.json",
         )

.github/workflows/ci.yml

@@ -1356,9 +1356,13 @@ jobs:
           cache-targets: false
           save-if: ${{ github.ref == 'refs/heads/main' }}
 
-      - name: Test the tutorial gate helpers
+      - name: Install docs dependencies
+        working-directory: docs/site
+        run: npm ci
+
+      - name: Test the tutorial runner
         working-directory: docs/site
-        run: npm run test:tutorial:evidence
+        run: npm run test:tutorial:runner
 
       - name: Check tutorial command drift
         working-directory: docs/site
@@ -1393,8 +1397,6 @@ jobs:
           "${RUNNER_TEMP}/maturin/bin/pip" install --quiet \
             --require-hashes --only-binary=:all: \
             --requirement "${GITHUB_WORKSPACE}/release/requirements/maturin-1.9.6.txt"
-          # The output stays inside the workspace, because the container step
-          # below mounts the workspace and nothing else.
           out_dir="${GITHUB_WORKSPACE}/target/evidence-tutorial-client"
           python3 release/scripts/assemble-registry-client-packages.py \
             --artifacts python \
@@ -1411,15 +1413,13 @@ jobs:
             exit 1
           fi
           # Prove installer metadata and native facade loading from this exact
-          # wheel. The clean-container tutorials separately exercise requests.
+          # wheel. The application tutorial separately exercises requests.
           python3 -m venv "${RUNNER_TEMP}/client-install-smoke"
           "${RUNNER_TEMP}/client-install-smoke/bin/pip" install \
             --no-index --no-deps "${wheel}"
           "${RUNNER_TEMP}/client-install-smoke/bin/python" -I \
             release/scripts/smoke-registry-client-package.py
-          # The gate reads it inside the container, at the mounted path.
-          printf 'TUTORIAL_CLIENT_WHEEL=/work/%s\n' \
-            "${wheel#"${GITHUB_WORKSPACE}/"}" >>"${GITHUB_ENV}"
+          printf 'REGISTRY_CLIENT_PY_WHEEL=%s\n' "${wheel}" >>"${GITHUB_ENV}"
 
       - name: Test the exact local Evidence lifecycle
         shell: bash
@@ -1438,10 +1438,7 @@ jobs:
           EVIDENCECTL_BIN: ${{ github.workspace }}/target/debug/evidencectl
           EVIDENCE_OID4VCI_BIN: ${{ github.workspace }}/target/debug/evidence-oid4vci
           EVIDENCE_OID4VCI_INTEROP_TEST_BIN: ${{ github.workspace }}/target/debug/inji-oid4vci-interop-test
-        run: |
-          set -euo pipefail
-          REGISTRY_CLIENT_PY_WHEEL="${GITHUB_WORKSPACE}/${TUTORIAL_CLIENT_WHEEL#/work/}" \
-            bash docs/site/scripts/check-evidence-tutorials.sh
+        run: node docs/site/scripts/run-tutorial.mjs --gate evidence
 
   breg-tutorial:
     name: Base Registry Engine tutorial from source

docs/site/package.json

@@ -47,9 +47,8 @@
     "check:tutorial:discovery:dry-run": "bash scripts/check-discovery-tutorial.sh --dry-run",
     "check:tutorial:relay": "bash scripts/check-relay-tutorial.sh",
     "check:tutorial:relay:dry-run": "bash scripts/check-relay-tutorial.sh --dry-run",
-    "test:tutorial:evidence": "node --test scripts/check-evidence-tutorials.test.mjs",
-    "check:tutorial:evidence": "bash scripts/check-evidence-tutorials.sh",
-    "check:tutorial:evidence:dry-run": "bash scripts/check-evidence-tutorials.sh --dry-run",
+    "check:tutorial:evidence": "node scripts/run-tutorial.mjs --gate evidence",
+    "check:tutorial:evidence:dry-run": "node scripts/run-tutorial.mjs --gate evidence --dry-run",
     "test:tutorial:runner": "node --test \"scripts/tutorial-runner/*.test.mjs\"",
     "check:tutorial:breg": "node scripts/run-tutorial.mjs --gate breg",
     "check:tutorial:breg:dry-run": "node scripts/run-tutorial.mjs --gate breg --dry-run",