From ae283b5f87aa172e26693bbe3a954caa7e5ed141 Mon Sep 17 00:00:00 2001 From: iperev Date: Thu, 22 Oct 2026 01:20:51 +0200 Subject: [PATCH 1/6] build(go): build with Go 1.27.3 to remove a reachable standard-library vulnerability govulncheck reports GO-2026-6809 (crypto/x509 distrusts compromised intermediate certificates) as reachable in Go 1.27.2 through the CLI's diagnostic output path, so the release candidate and every source gate fail at go:vulncheck. Go 1.27.3 contains the fix. - go.mod: toolchain go1.27.3; the go directive and every module requirement are unchanged. - release/change-record.v2.json: the toolchain addition now states that release binaries are built with Go 1.27.3 and why; the pinned copy in internal/tools/releasechange/record_test.go is regenerated from it. Proof on this tree: A local npm run check sequence on darwin/arm64 passed (exit 0) on tree f7cfc5f6bd9a with npm 12.2.0, CPython 3.15.0 and Go 1.27.3: 561/561 browser cases in Chromium, Firefox and WebKit with zero retries. One recorded deviation: the host firewall blocks Go-built binaries from the network, so the go:vulncheck step ran govulncheck against a vulnerability database snapshot fetched with curl (modified 2026-10-21T00:27:29Z) and found no vulnerabilities. With the same snapshot, Go 1.27.2 is reported affected by GO-2026-6809. --- go.mod | 2 +- internal/tools/releasechange/record_test.go | 2 +- release/change-record.v2.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 232c7401..bbc688b0 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/research-engineering/agentic-proofkit go 1.27 -toolchain go1.27.2 +toolchain go1.27.3 tool ( github.com/rhysd/actionlint/cmd/actionlint diff --git a/internal/tools/releasechange/record_test.go b/internal/tools/releasechange/record_test.go index abc79175..7f417c3f 100644 --- a/internal/tools/releasechange/record_test.go +++ b/internal/tools/releasechange/record_test.go @@ -241,7 +241,7 @@ var currentAdditions = []Change{ {ChangeID: "proofkit.workflow.agent-trajectory-witness", Summary: "REQ-PROOFKIT-WORKFLOW-020 adds a whole agent trajectory witness that drives a fixture repository through the public CLI only, from install bootstrap to current proof, with a route ledger, negative controls and an edge-local invalidation matrix; the installed npm carrier and the host wheel's module and console-script carriers execute it, while the CPython 3.11.0 floor smoke keeps its finite carrier cases. It proves CLI admission, linkage, currentness and route closure, not native test truth."}, {ChangeID: "proofkit.release-evidence.hardening", Summary: "Release tooling admits npm absence only from bounded strict E404 JSON, rejects failed or incomplete Go build-info metadata instead of emitting an empty SBOM dependency inventory, bounds the root-package verifier to 128 MiB compressed, 512 MiB expanded and 4096 entries, rejects duplicate wheel filenames in PyPI registry evidence, reads each wheel once in the Python package verifier and installs a private copy of the admitted bytes in its local consumer check, requires installed-CLI help on stdout with empty stderr, and passes the workflow credential to curl on stdin instead of argv. The release guide defines the provider-bound default signing policy; independently signed annotated tags remain optional. Source-checkout coverage tooling rejects partial, aborted, expired or failed native test observations."}, {ChangeID: "proofkit.browser.consumer-hosting-boundary", Summary: "Remote reading of specification views is documented as a consumer hosting concern: publish the existing --format html exports from a consumer-owned static host behind a consumer-owned access gateway, use rendered-artifact-freshness for hosted copies and secret-scan as an advisory byte check. requirement-browser-server and view keep serving only literal loopback hosts."}, - {ChangeID: "proofkit.toolchain.runtime-and-build-updates", Summary: "Release binaries are built with Go 1.27.2 instead of 1.27.1 and bundle golang.org/x/mod 0.42.0, golang.org/x/net 0.61.0, golang.org/x/sys 0.49.0, golang.org/x/text 0.43.0 and github.com/mattn/go-isatty 0.0.24; no runtime module was added or removed. LICENSE now also names the Go standard library among the bundled BSD-3-Clause components; its reproduced Go Authors notice already covered it. The development and CI toolchain moves to npm 12.2.0, Node 26.11.1, CPython 3.15.0, Playwright 1.63.0 and axe-core 4.14.0 with pinned GitHub Actions, and the Python launcher's POSIX path imports neither subprocess nor typing."}, + {ChangeID: "proofkit.toolchain.runtime-and-build-updates", Summary: "Release binaries are built with Go 1.27.3 instead of 1.27.1, whose crypto/x509 fix removes the reachable standard-library vulnerability GO-2026-6809 that govulncheck reports for 1.27.2, and bundle golang.org/x/mod 0.42.0, golang.org/x/net 0.61.0, golang.org/x/sys 0.49.0, golang.org/x/text 0.43.0 and github.com/mattn/go-isatty 0.0.24; no runtime module was added or removed. LICENSE now also names the Go standard library among the bundled BSD-3-Clause components; its reproduced Go Authors notice already covered it. The development and CI toolchain moves to npm 12.2.0, Node 26.11.1, CPython 3.15.0, Playwright 1.63.0 and axe-core 4.14.0 with pinned GitHub Actions, and the Python launcher's POSIX path imports neither subprocess nor typing."}, } var currentMigrationSteps = []string{ diff --git a/release/change-record.v2.json b/release/change-record.v2.json index 12b449c1..55484259 100644 --- a/release/change-record.v2.json +++ b/release/change-record.v2.json @@ -180,7 +180,7 @@ }, { "changeId": "proofkit.toolchain.runtime-and-build-updates", - "summary": "Release binaries are built with Go 1.27.2 instead of 1.27.1 and bundle golang.org/x/mod 0.42.0, golang.org/x/net 0.61.0, golang.org/x/sys 0.49.0, golang.org/x/text 0.43.0 and github.com/mattn/go-isatty 0.0.24; no runtime module was added or removed. LICENSE now also names the Go standard library among the bundled BSD-3-Clause components; its reproduced Go Authors notice already covered it. The development and CI toolchain moves to npm 12.2.0, Node 26.11.1, CPython 3.15.0, Playwright 1.63.0 and axe-core 4.14.0 with pinned GitHub Actions, and the Python launcher's POSIX path imports neither subprocess nor typing." + "summary": "Release binaries are built with Go 1.27.3 instead of 1.27.1, whose crypto/x509 fix removes the reachable standard-library vulnerability GO-2026-6809 that govulncheck reports for 1.27.2, and bundle golang.org/x/mod 0.42.0, golang.org/x/net 0.61.0, golang.org/x/sys 0.49.0, golang.org/x/text 0.43.0 and github.com/mattn/go-isatty 0.0.24; no runtime module was added or removed. LICENSE now also names the Go standard library among the bundled BSD-3-Clause components; its reproduced Go Authors notice already covered it. The development and CI toolchain moves to npm 12.2.0, Node 26.11.1, CPython 3.15.0, Playwright 1.63.0 and axe-core 4.14.0 with pinned GitHub Actions, and the Python launcher's POSIX path imports neither subprocess nor typing." } ], "migration": { From e672f6b16bbb530614d925b5e8548dd81e8f1f50 Mon Sep 17 00:00:00 2001 From: iperev Date: Thu, 22 Oct 2026 03:00:15 +0200 Subject: [PATCH 2/6] ci(platform): verify Linux arm64 on native ARM runners Add the required job "quality / platform smoke / linux-arm64" on the GitHub-hosted ubuntu-24.04-arm runner. Its first step fails unless uname -m reports aarch64. It then sets up the same pinned Node 26.11.1, verified npm, Go and Python 3.15.0 toolchains as the macOS platform jobs, runs the rootpath, repositorytransaction, processgroup and projectstatus packages natively, runs npm run npm:version and npm run platform:smoke, builds the complete current release wheel set, and runs the CPython 3.11.0 installed-wheel smoke, which selects the manylinux_2_17_aarch64 wheel and the pinned linux/arm64 CPython 3.11.0 image digest. The required aggregate needs the job and requires its success. The workflow oracles pin the job's name, runner, 15-minute timeout and exact ordered steps, the five required leaf jobs and the aggregate shell. They refuse mutants that move the job to another runner, drop, move or weaken its architecture check, drop or narrow its invariant, smoke, wheel-set or CPython 3.11.0 steps, or remove it from the aggregate needs or result checks. The Python runtime oracle counts the job as a pinned Python consumer. README states that required CI executes Linux arm64 on ubuntu-24.04-arm and x64 on ubuntu-24.04, that CI does not execute a host at the manylinux 2.17 floor, and that the CPython 3.11.0 smoke runs on both Linux architectures; its projection test follows. REQ-PROOFKIT-QUALITY-011 now names the Linux arm64 platform job, its architecture check and its CPython 3.11.0 step, and five required leaf jobs instead of four. CONTRIBUTING lists ubuntu-24.04-arm among the source-check targets. Proof on this tree: A local npm run check sequence on darwin/arm64 passed (exit 0) on tree 557350793e08 with npm 12.2.0, CPython 3.15.0 and Go 1.27.3: 561/561 browser cases in Chromium, Firefox and WebKit with zero retries. One recorded deviation: the host firewall blocks Go-built binaries from the network, so go:vulncheck ran against a vulnerability database snapshot fetched with curl (modified 2026-10-21T00:27:29Z): no vulnerabilities. The new arm64 job's success is proven only by hosted CI on this PR. --- .github/workflows/ci.yml | 57 +++ CONTRIBUTING.md | 12 +- README.md | 14 +- .../proofkit-supply-chain-quality/overview.md | 27 +- .../requirements.v2.json | 2 +- internal/tools/pythonpackage/metadata_test.go | 21 +- scripts/workflow_package_gate_oracle_test.go | 409 +++++++++++++++--- scripts/workflow_python_minimum_test.go | 173 +++++--- scripts/workflow_python_runtime_test.go | 12 +- 9 files changed, 578 insertions(+), 149 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ee24f87..345a9a6a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -258,6 +258,61 @@ jobs: npm run npm:version npm run platform:smoke + platform-smoke-linux-arm64: + name: quality / platform smoke / linux-arm64 + runs-on: ubuntu-24.04-arm + timeout-minutes: 15 + steps: + - name: Verify native Linux architecture + run: | + set -euo pipefail + machine="$(uname -m)" + printf 'uname -m=%s\n' "$machine" + test "$machine" = "aarch64" + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 26.11.1 + package-manager-cache: false + + - name: Install verified npm + uses: ./.github/actions/setup-verified-npm + + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + + - name: Setup Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.15.0' + + - name: Run Linux filesystem and process invariants + run: go test ./internal/kernel/rootpath ./internal/kernel/repositorytransaction ./internal/kernel/processgroup ./internal/command/projectstatus -count=1 + + - name: Run platform smoke + run: | + set -euo pipefail + npm run npm:version + npm run platform:smoke + + - name: Build release wheel set + run: | + set -euo pipefail + go run ./internal/tools/packagebuild all + go run ./internal/tools/pythonpackage build + + - name: Verify Python 3.11.0 installed wheel + run: go run ./internal/tools/pythonpackage verify-minimum + browser-runtime: name: quality / browser runtime runs-on: ubuntu-24.04 @@ -324,6 +379,7 @@ jobs: - source-quality - platform-smoke - platform-smoke-intel + - platform-smoke-linux-arm64 steps: - name: Verify required quality results run: | @@ -332,4 +388,5 @@ jobs: test "${{ needs.source-quality.result }}" = "success" test "${{ needs.platform-smoke.result }}" = "success" test "${{ needs.platform-smoke-intel.result }}" = "success" + test "${{ needs.platform-smoke-linux-arm64.result }}" = "success" printf 'OK: all required quality checks passed\n' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8245077f..059450b1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -115,12 +115,12 @@ SZOMB. A truncated record or a permission error remains an ordinary observation failure: the observer runs only before the sole reap, so the unreaped child still pins its PID and group and retained cleanup may signal them. Linux uses fresh `waitid(P_PID, WEXITED|WNOWAIT|WNOHANG)` records. Source-check targets -are the configured Ubuntu 24.04, macos-15 (arm64) and macos-15-intel (x86_64) -jobs and the native Linux wheel verifier profile, each requiring actual -host/API qualification. A CI label, cross-build or one local run is not native -qualification of another host. macOS 13 and 14 are not executed: no hosted -macOS 13 image exists, and GitHub retires `macos-14` on 2026-11-02 after failing -brownouts (actions/runner-images#13518). +are the configured Ubuntu 24.04 (x86_64), ubuntu-24.04-arm (arm64), macos-15 +(arm64) and macos-15-intel (x86_64) jobs and the native Linux wheel verifier +profile, each requiring actual host/API qualification. A CI label, cross-build +or one local run is not native qualification of another host. macOS 13 and 14 +are not executed: no hosted macOS 13 image exists, and GitHub retires +`macos-14` on 2026-11-02 after failing brownouts (actions/runner-images#13518). This package is outside the CLI dependency closure on all four shipped targets; consumer wheel tags and platform minima are unchanged. Import or payload changes require a fresh closure proof, and unsupported hosts fail before child Start. diff --git a/README.md b/README.md index 0eca95d6..431e9cd8 100644 --- a/README.md +++ b/README.md @@ -149,15 +149,17 @@ Supported binary targets are macOS 15 or later on arm64 or x64, executed by required CI on the GitHub-hosted `macos-15` and `macos-15-intel` runners. Darwin binaries declare a macOS 13.0 minimum in their Mach-O build metadata; macOS 13 and 14 are binary-compatible by that metadata only, not executed or verified. -Linux manylinux 2.17 or later is supported on arm64 or x64. Windows is unsupported. The Python -runner requires Python 3.11 or later and wraps the same Go CLI; it is not a -Python SDK. +Linux manylinux 2.17 or later is supported on arm64 or x64, executed by +required CI on the GitHub-hosted `ubuntu-24.04-arm` and `ubuntu-24.04` runners. +The manylinux 2.17 floor is the wheel-tag declaration; CI does not execute a +host at that floor. Windows is unsupported. The Python runner requires Python +3.11 or later and wraps the same Go CLI; it is not a Python SDK. Linux wheel installation requires pip 20.3 or later for the manylinux tag. Use an installer compatible with the selected Python. The bounded compatibility -smoke exercises the initial Python 3.11 release with a checksum-pinned compatible -installer. This is compatibility evidence, not a recommendation to use an -obsolete interpreter. +smoke exercises the initial Python 3.11 release on Linux arm64 and x64 with a +checksum-pinned compatible installer. This is compatibility evidence, not a +recommendation to use an obsolete interpreter. After an exact Python package version is available from an admitted channel, use one complete package-manager chain: diff --git a/docs/specs/proofkit-supply-chain-quality/overview.md b/docs/specs/proofkit-supply-chain-quality/overview.md index 831b31f3..9b39c2a3 100644 --- a/docs/specs/proofkit-supply-chain-quality/overview.md +++ b/docs/specs/proofkit-supply-chain-quality/overview.md @@ -73,18 +73,21 @@ vulnerability absence, or consumer rollout safety by itself. tests does not prove assertion-branch execution, mutation adequacy, or exhaustive command semantics. - `REQ-PROOFKIT-QUALITY-011`: CI separates the OS-independent full - source/package gate from macOS platform smoke, executes the complete Go - package set through its owner command, uses explicit hosted runner labels - instead of floating latest labels, and exposes one fail-closed aggregate gate - whose exact expression and script reject neutralization and require every - required leaf check to finish with success, while the aggregate job omits - job defaults and `continue-on-error` entirely. Exact workflow-level bash - defaults plus a closed inventory require exact provider-check names and - scalar runners, bind each macOS job's (required arm64 `macos-15` and x86_64 - `macos-15-intel`) presence-aware exact ordered step inventory and exact - local setup-action bytes to the fail-closed package-script owner command, - require every required leaf job and step to - omit conditions except for exact named owner conditions, and reject + source/package gate from macOS and Linux arm64 platform smoke, executes the + complete Go package set through its owner command, uses explicit hosted + runner labels instead of floating latest labels, and exposes one fail-closed + aggregate gate whose exact expression and script reject neutralization and + require every required leaf check to finish with success, while the + aggregate job omits job defaults and `continue-on-error` entirely. Exact + workflow-level bash defaults plus a closed inventory require exact + provider-check names and scalar runners, bind each platform job's (required + arm64 `macos-15`, x86_64 `macos-15-intel` and aarch64 `ubuntu-24.04-arm`) + presence-aware exact ordered step inventory, whose first step checks the + native architecture, and exact local setup-action bytes to the fail-closed + package-script owner command, require the minimum-version CPython installed-wheel + smoke in both the Linux x86_64 source job and the Linux arm64 job, require + every required leaf job and step to omit conditions except for exact named + owner conditions, and reject dynamically false or explicit-null conditions, dual `run`/`uses` keys, non-string runner labels, nested or top-level semantic shadow steps, reusable-job substitution, inherited or local environment entries, shell, diff --git a/docs/specs/proofkit-supply-chain-quality/requirements.v2.json b/docs/specs/proofkit-supply-chain-quality/requirements.v2.json index a49c6300..5a455c75 100644 --- a/docs/specs/proofkit-supply-chain-quality/requirements.v2.json +++ b/docs/specs/proofkit-supply-chain-quality/requirements.v2.json @@ -21,7 +21,7 @@ , {"requirementId":"REQ-PROOFKIT-QUALITY-008","statementCompletion":"GitHub Actions workflow syntax and expression semantics are checked by actionlint in the local package gate.","fields":{"nonClaims":["This requirement does not claim workflow runtime success, external action safety, branch protection, or release approval."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-008"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"medium","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} , {"requirementId":"REQ-PROOFKIT-QUALITY-009","statementCompletion":"Performance-sensitive parser and serializer paths expose benchmark entrypoints without making wall-clock budgets a required PR gate before stable baselines exist.","fields":{"nonClaims":["This requirement does not claim current performance maximum, CI wall-clock budget enforcement, or regression absence without benchmark evidence."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-009"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"medium","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} , {"requirementId":"REQ-PROOFKIT-QUALITY-010","statementCompletion":"Coverage metrics report requirement, binding, witness, CLI inventory linkage, descriptor-owned command proof-route candidates, and a separate execution-backed command-oracle ledger. The ledger must bind every candidate commandRef, selector, concrete negative case, falsification event, assertion oracle, expected public outcome, and owner invariant to one immutable exact-file materialized source snapshot; execute exactly the selected active Go tests through canonically ordered package-scoped argv vectors that cannot select same-named non-candidate tests in another package; require unique package start and pass events, unique test run and pass events, owner-reserved cooperative attributes for every candidate, bounded cancellable subprocesses, and immediate process-group termination on output-limit violation; reject missing, unknown, duplicate, skipped, failed, reordered, context-invalid, spoofed-output, cross-test, source-drift, producer-unreachable candidate projections, and correlated identity substitutions; publish and invalidate local diagnostic and metrics artifacts atomically through repository-confined non-symlink destinations; on supported release platforms, open retained input artifacts descriptor-first through a nonblocking no-follow boundary before type, identity, and exact byte-limit checks so named pipes, symlinks, path substitution, and post-read mutation cannot block or escape, and fail closed instead of using a weaker file-open path elsewhere; revalidate current candidate, corpus, runtime, and source identities before and after artifact publication; and fail closed when any public command lacks execution-backed evidence. An independently authored versioned counterfeit corpus must bijectively cover every shipped evidence class, required policy axis, record coordinate, and declared substitution axis with positive controls and checked-in exact expected decisions evaluated through the production admission and lifecycle owners rather than a generated expectation registry. Static route metadata, prose, legacy source markers, test existence, and failure-capable syntax remain proof-route candidates and cannot satisfy the execution-backed closure.","fields":{"nonClaims":["Successful selected tests do not prove assertion-branch execution, mutation adequacy, line coverage, dead-code absence, exhaustive command semantics, malicious-test resistance, producer authentication, receipt freshness, merge satisfaction, or production readiness."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-010"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"medium","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} - , {"requirementId":"REQ-PROOFKIT-QUALITY-011","statementCompletion":"CI separates the full source/package gate with an explicit native Linux Python 3.11.0 installed-wheel subcheck from the platform-specific macOS execution smoke, runs the complete processgroup package on each macOS lane without selector-based narrowing alongside the existing filesystem checks, executes the complete Go package set through the owner command rather than a partial package list, requires the local composed Go gate and CI source job to run non-mutating go mod tidy -diff followed by go mod verify before Go tests with either failure stopping the chain, requires exact owner-reviewed root check and go:check AND-only command chains, forbids pre/post npm lifecycle script keys by presence for check and every direct npm-run target in those chains or CI source steps without redefining unrelated npm scripts, uses exact scalar hosted runners and provider-check names, requires each macOS job's presence-aware exact ordered step inventory, with distinct required macos-15 arm64 and macos-15-intel x86_64 jobs, and exact repository-local setup-action bytes to execute the exact fail-closed platform-smoke package-script owner command, and exposes one exact fail-closed aggregate quality gate under exact bash workflow run defaults with no inherited working-directory or environment entries; a closed inventory requires every required leaf job and step to omit conditions except for exact owner-reviewed named conditions, every required leaf job and the aggregate job omit job defaults, reusable-workflow substitution, and continue-on-error and have no job environment entries, all of their steps omit shell, working-directory, and continue-on-error and have no step environment entries, and the aggregate's reviewed expression and script require every exact required leaf check to finish with success; workflow mutation fixtures preserve nil versus empty values, field-presence bits and detached mutable data before applying negative changes; neutralizing wrappers, dynamically false or explicit-null conditions, execution overrides, dual run/uses keys, extra disjunctions, non-string runner labels, nested or top-level semantic shadow steps, no-op or reusable-job substitution, skipped, failed, or cancelled leaf jobs cannot satisfy branch-protection evidence. The unconditional minimum-Python owner step immediately follows package artifact verification and precedes receipts and closeout; omission, neutralization or failure cannot yield source-quality success, and the same four required leaf jobs remain in the aggregate.","fields":{"nonClaims":["This requirement does not claim provider-run success, branch-protection settings, Windows support, performance budget enforcement, or release approval."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-011"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"medium","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} + , {"requirementId":"REQ-PROOFKIT-QUALITY-011","statementCompletion":"CI separates the full source/package gate with an explicit native Linux x86_64 Python 3.11.0 installed-wheel subcheck from the platform-specific macOS and Linux arm64 execution smoke, runs the complete processgroup package on each macOS and Linux arm64 platform lane without selector-based narrowing alongside the existing filesystem checks, executes the complete Go package set through the owner command rather than a partial package list, requires the local composed Go gate and CI source job to run non-mutating go mod tidy -diff followed by go mod verify before Go tests with either failure stopping the chain, requires exact owner-reviewed root check and go:check AND-only command chains, forbids pre/post npm lifecycle script keys by presence for check and every direct npm-run target in those chains or CI source steps without redefining unrelated npm scripts, uses exact scalar hosted runners and provider-check names, requires each platform job's presence-aware exact ordered step inventory, with distinct required macos-15 arm64, macos-15-intel x86_64 and ubuntu-24.04-arm aarch64 jobs whose first step fails unless uname -m reports the job's native architecture, and exact repository-local setup-action bytes to execute the exact fail-closed platform-smoke package-script owner command, and exposes one exact fail-closed aggregate quality gate under exact bash workflow run defaults with no inherited working-directory or environment entries; a closed inventory requires every required leaf job and step to omit conditions except for exact owner-reviewed named conditions, every required leaf job and the aggregate job omit job defaults, reusable-workflow substitution, and continue-on-error and have no job environment entries, all of their steps omit shell, working-directory, and continue-on-error and have no step environment entries, and the aggregate's reviewed expression and script require every exact required leaf check to finish with success; workflow mutation fixtures preserve nil versus empty values, field-presence bits and detached mutable data before applying negative changes; neutralizing wrappers, dynamically false or explicit-null conditions, execution overrides, dual run/uses keys, extra disjunctions, non-string runner labels, nested or top-level semantic shadow steps, no-op or reusable-job substitution, skipped, failed, or cancelled leaf jobs cannot satisfy branch-protection evidence. The unconditional minimum-Python owner step immediately follows package artifact verification in the source job and precedes receipts and closeout, and in the Linux arm64 job it immediately follows a build of the complete current release wheel set; omission or neutralization of either step cannot yield source-quality success, failure of either step cannot yield success of the job that runs it, and none of them can yield required-aggregate success; exactly five required leaf jobs remain in the aggregate.","fields":{"nonClaims":["This requirement does not claim provider-run success, branch-protection settings, Windows support, performance budget enforcement, or release approval."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-011"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"medium","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} , {"requirementId":"REQ-PROOFKIT-QUALITY-012","statementCompletion":"Release and adoption channel identifiers are admitted by one canonical authority vocabulary that separates authorityChannel values from display labels, publisher environments, statuses, registry sub-vocabulary, and candidate evidence classes.","fields":{"nonClaims":["This requirement does not claim provider-run success, registry publication, GitHub Release creation, consumer adoption, rollout approval, or release approval."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-012"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"medium","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} , {"requirementId":"REQ-PROOFKIT-QUALITY-013","statementCompletion":"Self-hosting workflow package-gate evidence is checked by a typed workflow oracle that proves the package gate is reachable, exact, advisory, and guarded only by an absent condition or the exact owner-reviewed condition for every trust-significant job and step; the oracle checks the complete required CI and release job inventories, rejects neutralizing or dynamically false expression variants, requires any workflow run defaults to retain exact bash without a working-directory override, admits only the exact owner-reviewed workflow and step environment entries, requires the gate job to omit job defaults and continue-on-error and have no job environment entries, requires every step in that job to omit shell, working-directory, and continue-on-error, and orders the gate before evidence publication instead of relying on workflow text search. The CI source job includes the exact minimum-Python installed-wheel invocation after package artifacts and before self-hosting receipts, coverage and closeout; its execution requirement does not promote local advisory receipts into authenticated provider evidence.","fields":{"nonClaims":["This requirement does not claim full GitHub Actions interpretation, provider-run success, branch-protection enforcement, required-check enforcement, external action safety, registry publication, release approval, or rollout approval."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-013"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"medium","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} , {"requirementId":"REQ-PROOFKIT-QUALITY-014","statementCompletion":"Release authority consumers compare downstream policy against the admitted typed release-authority projection and admitted report digest instead of rereading caller-owned raw releaseAuthorityInput fields after validation.","fields":{"nonClaims":["This requirement does not claim public SDK stability, registry publication, package installation, consumer dependency policy, rollout approval, merge approval, provider provenance, proof freshness, or native witness execution."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-014"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"high","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} diff --git a/internal/tools/pythonpackage/metadata_test.go b/internal/tools/pythonpackage/metadata_test.go index 5a3f6ddb..bd36e638 100644 --- a/internal/tools/pythonpackage/metadata_test.go +++ b/internal/tools/pythonpackage/metadata_test.go @@ -181,20 +181,29 @@ func TestREADMEPlatformAndPythonProjection(t *testing.T) { if ownerMinimums["darwin"] != "13" { t.Fatalf("Darwin wheel-tag minimum %s no longer matches the README executed/metadata-only split", ownerMinimums["darwin"]) } + // Each OS names its required CI runners in the owner architecture order; + // a changed architecture set requires re-authoring the runner lists. + for _, goos := range []string{"darwin", "linux"} { + if strings.Join(ownerArchitectures[goos], " ") != "arm64 x64" { + t.Fatalf("%s architectures %v no longer match the README runner lists", goos, ownerArchitectures[goos]) + } + } expectedBlock := strings.Join([]string{ fmt.Sprintf("Supported binary targets are macOS 15 or later on %s, executed by", strings.Join(ownerArchitectures["darwin"], " or ")), "required CI on the GitHub-hosted `macos-15` and `macos-15-intel` runners. Darwin", "binaries declare a macOS 13.0 minimum in their Mach-O build metadata; macOS 13", "and 14 are binary-compatible by that metadata only, not executed or verified.", - fmt.Sprintf("Linux manylinux %s or later is supported on %s. Windows is unsupported. The Python", ownerMinimums["linux"], strings.Join(ownerArchitectures["linux"], " or ")), - "runner requires Python 3.11 or later and wraps the same Go CLI; it is not a", - "Python SDK.", + fmt.Sprintf("Linux manylinux %s or later is supported on %s, executed by", ownerMinimums["linux"], strings.Join(ownerArchitectures["linux"], " or ")), + "required CI on the GitHub-hosted `ubuntu-24.04-arm` and `ubuntu-24.04` runners.", + fmt.Sprintf("The manylinux %s floor is the wheel-tag declaration; CI does not execute a", ownerMinimums["linux"]), + "host at that floor. Windows is unsupported. The Python runner requires Python", + "3.11 or later and wraps the same Go CLI; it is not a Python SDK.", "", "Linux wheel installation requires pip 20.3 or later for the manylinux tag.", "Use an installer compatible with the selected Python. The bounded compatibility", - "smoke exercises the initial Python 3.11 release with a checksum-pinned compatible", - "installer. This is compatibility evidence, not a recommendation to use an", - "obsolete interpreter.", + "smoke exercises the initial Python 3.11 release on Linux arm64 and x64 with a", + "checksum-pinned compatible installer. This is compatibility evidence, not a", + "recommendation to use an obsolete interpreter.", "", "After an exact Python package version is available from an admitted channel,", "use one complete package-manager chain:", diff --git a/scripts/workflow_package_gate_oracle_test.go b/scripts/workflow_package_gate_oracle_test.go index 8f7b5331..a1da8a7e 100644 --- a/scripts/workflow_package_gate_oracle_test.go +++ b/scripts/workflow_package_gate_oracle_test.go @@ -669,6 +669,25 @@ func needsList(raw any) []string { } } +// withoutNeed removes exactly one owner need so a mutant omits only that job. +func withoutNeed(t *testing.T, raw any, job string) []any { + t.Helper() + values, ok := raw.([]any) + if !ok { + t.Fatalf("owner needs=%#v, want a list", raw) + } + result := make([]any, 0, len(values)) + for _, value := range values { + if value != job { + result = append(result, value) + } + } + if len(result) != len(values)-1 { + t.Fatalf("owner needs=%#v must contain %q exactly once", raw, job) + } + return result +} + func nestedValue(raw any, path []string) any { value := raw for _, key := range path { @@ -734,7 +753,7 @@ func TestCIWorkflowDeclaresFailClosedRequiredAggregate(t *testing.T) { if !usesAlwaysStatusCheck(gate.If) { t.Fatalf("ci-required-gate if=%q, want always() so failed or skipped needs are inspected", gate.If) } - wantNeeds := []string{"browser-runtime", "platform-smoke", "platform-smoke-intel", "source-quality"} + wantNeeds := []string{"browser-runtime", "platform-smoke", "platform-smoke-intel", "platform-smoke-linux-arm64", "source-quality"} if got := needsList(gate.Needs); !reflect.DeepEqual(got, wantNeeds) { t.Fatalf("ci-required-gate needs=%#v, want %#v", got, wantNeeds) } @@ -906,6 +925,34 @@ func TestCIRequiredAggregateRejectsExecutionOverrides(t *testing.T) { workflow.Jobs["platform-smoke-intel"] = job }, }, + { + name: "required Linux arm64 leaf job continue on error", + mutate: func(workflow *githubWorkflow) { + job := workflow.Jobs["platform-smoke-linux-arm64"] + job.ContinueOnError = "${{ 1 == 1 }}" + workflow.Jobs["platform-smoke-linux-arm64"] = job + }, + }, + { + name: "required Linux arm64 leaf job environment", + mutate: func(workflow *githubWorkflow) { + job := workflow.Jobs["platform-smoke-linux-arm64"] + job.Env = map[string]any{"GOARCH": "amd64"} + workflow.Jobs["platform-smoke-linux-arm64"] = job + }, + }, + { + name: "required Linux arm64 leaf step environment", + mutate: func(workflow *githubWorkflow) { + job := workflow.Jobs["platform-smoke-linux-arm64"] + for index := range job.Steps { + if job.Steps[index].Name == "Verify Python 3.11.0 installed wheel" { + job.Steps[index].Env = map[string]any{"DOCKER_DEFAULT_PLATFORM": "linux/amd64"} + } + } + workflow.Jobs["platform-smoke-linux-arm64"] = job + }, + }, { name: "required leaf job null continue on error", mutate: func(workflow *githubWorkflow) { @@ -1348,7 +1395,7 @@ func TestCIRequiredAggregateRejectsPlatformSmokeSubstitution(t *testing.T) { }, "x86_64 job dropped from aggregate needs": func(workflow *githubWorkflow) { gate := workflow.Jobs["ci-required-gate"] - gate.Needs = []any{"browser-runtime", "source-quality", "platform-smoke"} + gate.Needs = withoutNeed(t, gate.Needs, "platform-smoke-intel") workflow.Jobs["ci-required-gate"] = gate }, "x86_64 result unchecked by aggregate": func(workflow *githubWorkflow) { @@ -1359,7 +1406,7 @@ func TestCIRequiredAggregateRejectsPlatformSmokeSubstitution(t *testing.T) { "x86_64 job removed consistently": func(workflow *githubWorkflow) { delete(workflow.Jobs, "platform-smoke-intel") gate := workflow.Jobs["ci-required-gate"] - gate.Needs = []any{"browser-runtime", "source-quality", "platform-smoke"} + gate.Needs = withoutNeed(t, gate.Needs, "platform-smoke-intel") gate.Steps[0].Run = strings.Replace(gate.Steps[0].Run, intelAggregateLine, "", 1) workflow.Jobs["ci-required-gate"] = gate }, @@ -1375,6 +1422,198 @@ func TestCIRequiredAggregateRejectsPlatformSmokeSubstitution(t *testing.T) { } }) } + // The Linux arm64 claim depends on its own required native ARM runner: the + // job must prove aarch64 in-job before checkout, run the owner invariant, + // smoke and CPython 3.11.0 steps exactly, and stay in the aggregate. + linuxJob := linuxArm64PlatformSmokeJob + linuxStepIndex := func(job githubJob, name string) int { + index, err := uniqueStepIndex(job.Steps, name) + if err != nil || index < 0 { + t.Fatalf("locate %s step %q: index=%d err=%v", linuxJob, name, index, err) + } + return index + } + linuxArchitectureLine := "test \"$machine\" = \"aarch64\"\n" + linuxAggregateLine := "test \"${{ needs.platform-smoke-linux-arm64.result }}\" = \"success\"\n" + if job := base.Jobs[linuxJob]; !strings.Contains(job.Steps[linuxStepIndex(job, "Verify native Linux architecture")].Run, linuxArchitectureLine) { + t.Fatal("owner Linux arm64 job lacks the aarch64 assertion line") + } + if !strings.Contains(base.Jobs["ci-required-gate"].Steps[0].Run, linuxAggregateLine) { + t.Fatal("owner aggregate lacks the Linux arm64 result line") + } + linuxCases := []struct { + name string + mutate func(*githubWorkflow) + }{ + {"x64 runner", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + job.RunsOn = "ubuntu-24.04" + workflow.Jobs[linuxJob] = job + }}, + {"Darwin arm64 runner", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + job.RunsOn = "macos-15" + workflow.Jobs[linuxJob] = job + }}, + {"older ARM runner image", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + job.RunsOn = "ubuntu-22.04-arm" + workflow.Jobs[linuxJob] = job + }}, + {"non-string runner label", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + job.RunsOn = []any{"ubuntu-24.04-arm", false} + workflow.Jobs[linuxJob] = job + }}, + {"timeout lowered", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + job.TimeoutMinutes = 1 + workflow.Jobs[linuxJob] = job + }}, + {"provider check name drift", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + job.Name = "quality / platform smoke" + workflow.Jobs[linuxJob] = job + }}, + {"architecture check omitted", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + index := linuxStepIndex(job, "Verify native Linux architecture") + job.Steps = append(job.Steps[:index], job.Steps[index+1:]...) + workflow.Jobs[linuxJob] = job + }}, + {"architecture assertion removed", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + index := linuxStepIndex(job, "Verify native Linux architecture") + job.Steps[index].Run = strings.Replace(job.Steps[index].Run, linuxArchitectureLine, "", 1) + workflow.Jobs[linuxJob] = job + }}, + {"architecture assertion expects x86_64", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + index := linuxStepIndex(job, "Verify native Linux architecture") + job.Steps[index].Run = strings.Replace(job.Steps[index].Run, linuxArchitectureLine, "test \"$machine\" = \"x86_64\"\n", 1) + workflow.Jobs[linuxJob] = job + }}, + {"architecture assertion masked", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + index := linuxStepIndex(job, "Verify native Linux architecture") + job.Steps[index].Run = strings.Replace(job.Steps[index].Run, linuxArchitectureLine, "test \"$machine\" = \"aarch64\" || true\n", 1) + workflow.Jobs[linuxJob] = job + }}, + {"architecture check moved after checkout", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + index := linuxStepIndex(job, "Verify native Linux architecture") + checkout := linuxStepIndex(job, "Checkout") + job.Steps[index], job.Steps[checkout] = job.Steps[checkout], job.Steps[index] + workflow.Jobs[linuxJob] = job + }}, + {"Darwin architecture check substituted", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + index := linuxStepIndex(job, "Verify native Linux architecture") + job.Steps[index] = workflow.Jobs["platform-smoke"].Steps[0] + workflow.Jobs[linuxJob] = job + }}, + {"process package omitted", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + for i := range job.Steps { + job.Steps[i].Run = strings.ReplaceAll(job.Steps[i].Run, " ./internal/kernel/processgroup", "") + } + workflow.Jobs[linuxJob] = job + }}, + {"process suite narrowed", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + index := linuxStepIndex(job, "Run Linux filesystem and process invariants") + job.Steps[index].Run += " -run '^TestLinuxObservationClassification$'" + workflow.Jobs[linuxJob] = job + }}, + {"invariants omitted", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + index := linuxStepIndex(job, "Run Linux filesystem and process invariants") + job.Steps = append(job.Steps[:index], job.Steps[index+1:]...) + workflow.Jobs[linuxJob] = job + }}, + {"platform smoke neutralized", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + job.Steps[linuxStepIndex(job, "Run platform smoke")].Run = "true" + workflow.Jobs[linuxJob] = job + }}, + {"release wheel set narrowed to current", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + index := linuxStepIndex(job, "Build release wheel set") + job.Steps[index].Run = strings.Replace(job.Steps[index].Run, "packagebuild all", "packagebuild current", 1) + workflow.Jobs[linuxJob] = job + }}, + {"CPython 3.11.0 smoke omitted", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + index := linuxStepIndex(job, "Verify Python 3.11.0 installed wheel") + job.Steps = append(job.Steps[:index], job.Steps[index+1:]...) + workflow.Jobs[linuxJob] = job + }}, + {"no-op replaces job steps", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + job.Steps = []githubStep{{Name: "No-op", Run: "true", runPresent: true}} + workflow.Jobs[linuxJob] = job + }}, + {"Darwin step list substituted", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + job.Steps = workflow.Jobs["platform-smoke"].Steps + workflow.Jobs[linuxJob] = job + }}, + {"reusable job substitution", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + job.Steps = nil + job.Uses = "example/ci/.github/workflows/no-op.yml@0123456789012345678901234567890123456789" + workflow.Jobs[linuxJob] = job + }}, + {"inserted semantic shadow step", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + job.Steps = append([]githubStep{{Name: "Shadow platform smoke", Run: "npm pkg set scripts.platform:smoke=true", runPresent: true}}, job.Steps...) + workflow.Jobs[linuxJob] = job + }}, + {"job continue on error", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + job.ContinueOnError = true + workflow.Jobs[linuxJob] = job + }}, + {"job condition", func(workflow *githubWorkflow) { + job := workflow.Jobs[linuxJob] + job.If = "${{ github.event_name == 'never' }}" + workflow.Jobs[linuxJob] = job + }}, + {"dropped from aggregate needs", func(workflow *githubWorkflow) { + gate := workflow.Jobs["ci-required-gate"] + gate.Needs = withoutNeed(t, gate.Needs, linuxJob) + workflow.Jobs["ci-required-gate"] = gate + }}, + {"result unchecked by aggregate", func(workflow *githubWorkflow) { + gate := workflow.Jobs["ci-required-gate"] + gate.Steps[0].Run = strings.Replace(gate.Steps[0].Run, linuxAggregateLine, "", 1) + workflow.Jobs["ci-required-gate"] = gate + }}, + {"result masked by aggregate", func(workflow *githubWorkflow) { + gate := workflow.Jobs["ci-required-gate"] + gate.Steps[0].Run = strings.Replace(gate.Steps[0].Run, linuxAggregateLine, strings.TrimSuffix(linuxAggregateLine, "\n")+" || true\n", 1) + workflow.Jobs["ci-required-gate"] = gate + }}, + {"removed consistently", func(workflow *githubWorkflow) { + delete(workflow.Jobs, linuxJob) + gate := workflow.Jobs["ci-required-gate"] + gate.Needs = withoutNeed(t, gate.Needs, linuxJob) + gate.Steps[0].Run = strings.Replace(gate.Steps[0].Run, linuxAggregateLine, "", 1) + workflow.Jobs["ci-required-gate"] = gate + }}, + } + for _, test := range linuxCases { + t.Run(linuxJob+"/"+test.name, func(t *testing.T) { + workflow := cloneWorkflow(t, base) + test.mutate(&workflow) + if reflect.DeepEqual(workflow, base) { + t.Fatal("Linux arm64 mutation did not change the owner workflow") + } + if err := validateCIRequiredAggregate(workflow); err == nil { + t.Fatal("Linux arm64 platform-smoke weakening was admitted") + } + }) + } t.Run("package script owner neutralized", func(t *testing.T) { var manifest map[string]any if err := json.Unmarshal(packageManifest, &manifest); err != nil { @@ -1614,6 +1853,7 @@ test "${{ needs.browser-runtime.result }}" = "success" test "${{ needs.source-quality.result }}" = "success" test "${{ needs.platform-smoke.result }}" = "success" test "${{ needs.platform-smoke-intel.result }}" = "success" +test "${{ needs.platform-smoke-linux-arm64.result }}" = "success" printf 'OK: all required quality checks passed\n'` const requiredPlatformSmokeShell = `set -euo pipefail @@ -1621,6 +1861,25 @@ npm run npm:version npm run platform:smoke ` +// The exact required leaf jobs of the CI aggregate, in needsList order. +var ciRequiredLeafJobs = []string{"browser-runtime", "platform-smoke", "platform-smoke-intel", "platform-smoke-linux-arm64", "source-quality"} + +const linuxArm64PlatformSmokeJob = "platform-smoke-linux-arm64" + +// The Linux arm64 job builds the complete current release wheel set so the +// CPython 3.11.0 installed smoke can select and rebuild its native wheel. +const requiredReleaseWheelSetShell = `set -euo pipefail +go run ./internal/tools/packagebuild all +go run ./internal/tools/pythonpackage build +` + +const requiredMinimumPythonCommand = "go run ./internal/tools/pythonpackage verify-minimum" + +// Every platform job runs the same filesystem and process packages natively; +// their Darwin or Linux build-tagged tests qualify that host for the source +// tools' process lifecycle. +const requiredPlatformInvariantCommand = "go test ./internal/kernel/rootpath ./internal/kernel/repositorytransaction ./internal/kernel/processgroup ./internal/command/projectstatus -count=1" + func readPackageManifestForTest(t *testing.T) []byte { t.Helper() raw, err := os.ReadFile(filepath.Join("..", "package.json")) @@ -1677,6 +1936,15 @@ var platformSmokeArchitectures = map[string]string{ "platform-smoke-intel": "x86_64", } +// The Linux arm64 job first proves in-job that the runner is natively +// aarch64, so the runner label is not the only evidence. The job declares no +// container, so its run steps execute on the runner host. +const requiredLinuxArm64ArchitectureShell = `set -euo pipefail +machine="$(uname -m)" +printf 'uname -m=%s\n' "$machine" +test "$machine" = "aarch64" +` + func requiredDarwinArchitectureShell(architecture string) string { return `set -euo pipefail machine="$(uname -m)" @@ -1689,17 +1957,70 @@ test "${translated:-0}" = "0" // Both Darwin architectures run the same owner step list. Only the first // step's expected architecture and the job topology (name, runner label, -// timeout) distinguish arm64 from x86_64. +// timeout) distinguish arm64 from x86_64. The Linux arm64 job runs the same +// setup, invariant packages and platform smoke after its own architecture +// check, then the CPython 3.11.0 installed smoke over a fresh release wheel set. func validateExactPlatformSmokeSteps(jobID string, job githubJob) error { - architecture, ok := platformSmokeArchitectures[jobID] - if !ok { - return fmt.Errorf("ci workflow %s has no owner Darwin architecture", jobID) - } - expected := []githubStep{ - { + var expected []githubStep + if jobID == linuxArm64PlatformSmokeJob { + expected = append(expected, githubStep{ + Name: "Verify native Linux architecture", + Run: requiredLinuxArm64ArchitectureShell, + }) + expected = append(expected, platformSmokeSetupSteps()...) + expected = append(expected, + githubStep{Name: "Run Linux filesystem and process invariants", Run: requiredPlatformInvariantCommand}, + githubStep{Name: "Run platform smoke", Run: requiredPlatformSmokeShell}, + githubStep{Name: "Build release wheel set", Run: requiredReleaseWheelSetShell}, + githubStep{Name: "Verify Python 3.11.0 installed wheel", Run: requiredMinimumPythonCommand}, + ) + } else { + architecture, ok := platformSmokeArchitectures[jobID] + if !ok { + return fmt.Errorf("ci workflow %s has no owner platform architecture", jobID) + } + expected = append(expected, githubStep{ Name: "Verify native Darwin architecture", Run: requiredDarwinArchitectureShell(architecture), - }, + }) + expected = append(expected, platformSmokeSetupSteps()...) + expected = append(expected, + githubStep{Name: "Run Darwin filesystem and process invariants", Run: requiredPlatformInvariantCommand}, + githubStep{Name: "Run platform smoke", Run: requiredPlatformSmokeShell}, + ) + } + if len(job.Steps) != len(expected) { + return fmt.Errorf( + "ci workflow %s step count=%d, want exact %d", + jobID, + len(job.Steps), + len(expected), + ) + } + for index, want := range expected { + actual := job.Steps[index] + if actual.Name != want.Name || + actual.Uses != want.Uses || + actual.Run != want.Run || + !reflect.DeepEqual(actual.With, want.With) || + actual.runPresent != (want.Run != "") || + actual.usesPresent != (want.Uses != "") || + actual.withPresent != (want.With != nil) { + return fmt.Errorf( + "ci workflow %s step %d differs from exact owner step %q", + jobID, + index, + want.Name, + ) + } + } + return nil +} + +// The pinned setup steps every platform job runs between its architecture +// check and its first repository command. +func platformSmokeSetupSteps() []githubStep { + return []githubStep{ { Name: "Checkout", Uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", @@ -1730,41 +2051,7 @@ func validateExactPlatformSmokeSteps(jobID string, job githubJob) error { Uses: "actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97", With: map[string]any{"python-version": "3.15.0"}, }, - { - Name: "Run Darwin filesystem and process invariants", - Run: "go test ./internal/kernel/rootpath ./internal/kernel/repositorytransaction ./internal/kernel/processgroup ./internal/command/projectstatus -count=1", - }, - { - Name: "Run platform smoke", - Run: requiredPlatformSmokeShell, - }, } - if len(job.Steps) != len(expected) { - return fmt.Errorf( - "ci workflow %s step count=%d, want exact %d", - jobID, - len(job.Steps), - len(expected), - ) - } - for index, want := range expected { - actual := job.Steps[index] - if actual.Name != want.Name || - actual.Uses != want.Uses || - actual.Run != want.Run || - !reflect.DeepEqual(actual.With, want.With) || - actual.runPresent != (want.Run != "") || - actual.usesPresent != (want.Uses != "") || - actual.withPresent != (want.With != nil) { - return fmt.Errorf( - "ci workflow %s step %d differs from exact owner step %q", - jobID, - index, - want.Name, - ) - } - } - return nil } func validateCIRequiredAggregate(workflow githubWorkflow) error { @@ -1790,7 +2077,7 @@ func validateCIRequiredAggregate(workflow githubWorkflow) error { if err := validateOptionalWorkflowRunDefaults(workflow.Defaults); err != nil { return fmt.Errorf("ci workflow run defaults: %w", err) } - wantNeeds := []string{"browser-runtime", "platform-smoke", "platform-smoke-intel", "source-quality"} + wantNeeds := ciRequiredLeafJobs for _, jobID := range append(append([]string{}, wantNeeds...), "ci-required-gate") { job, ok := workflow.Jobs[jobID] if !ok { @@ -1808,11 +2095,12 @@ func validateCIRequiredAggregate(workflow githubWorkflow) error { "ci workflow", workflow, map[string]string{ - "browser-runtime": "", - "platform-smoke": "", - "platform-smoke-intel": "", - "source-quality": "", - "ci-required-gate": "always()", + "browser-runtime": "", + "platform-smoke": "", + "platform-smoke-intel": "", + "platform-smoke-linux-arm64": "", + "source-quality": "", + "ci-required-gate": "always()", }, map[string]map[string]string{ "browser-runtime": { @@ -1827,11 +2115,12 @@ func validateCIRequiredAggregate(workflow githubWorkflow) error { runner string timeoutMinutes int }{ - "source-quality": {name: "quality / source", runner: "ubuntu-24.04", timeoutMinutes: 20}, - "platform-smoke": {name: "quality / platform smoke / macos-15", runner: "macos-15", timeoutMinutes: 12}, - "platform-smoke-intel": {name: "quality / platform smoke / macos-15-intel", runner: "macos-15-intel", timeoutMinutes: 20}, - "browser-runtime": {name: "quality / browser runtime", runner: "ubuntu-24.04", timeoutMinutes: 20}, - "ci-required-gate": {name: "quality / required aggregate", runner: "ubuntu-24.04", timeoutMinutes: 5}, + "source-quality": {name: "quality / source", runner: "ubuntu-24.04", timeoutMinutes: 20}, + "platform-smoke": {name: "quality / platform smoke / macos-15", runner: "macos-15", timeoutMinutes: 12}, + "platform-smoke-intel": {name: "quality / platform smoke / macos-15-intel", runner: "macos-15-intel", timeoutMinutes: 20}, + "platform-smoke-linux-arm64": {name: "quality / platform smoke / linux-arm64", runner: "ubuntu-24.04-arm", timeoutMinutes: 15}, + "browser-runtime": {name: "quality / browser runtime", runner: "ubuntu-24.04", timeoutMinutes: 20}, + "ci-required-gate": {name: "quality / required aggregate", runner: "ubuntu-24.04", timeoutMinutes: 5}, } for jobID, expected := range exactJobTopology { job := workflow.Jobs[jobID] @@ -1851,13 +2140,15 @@ func validateCIRequiredAggregate(workflow githubWorkflow) error { ) } } - for _, jobID := range []string{"platform-smoke", "platform-smoke-intel"} { + for _, jobID := range []string{"platform-smoke", "platform-smoke-intel", linuxArm64PlatformSmokeJob} { if err := validateExactPlatformSmokeSteps(jobID, workflow.Jobs[jobID]); err != nil { return err } } - if err := validateCIMinimumPythonStep(workflow.Jobs["source-quality"]); err != nil { - return err + for _, jobID := range ciMinimumPythonJobIDs() { + if err := validateCIMinimumPythonStep(jobID, workflow.Jobs[jobID]); err != nil { + return err + } } if err := validateCILegacyUnicodeStep(workflow.Jobs["source-quality"]); err != nil { return err diff --git a/scripts/workflow_python_minimum_test.go b/scripts/workflow_python_minimum_test.go index 4605d38d..017d8065 100644 --- a/scripts/workflow_python_minimum_test.go +++ b/scripts/workflow_python_minimum_test.go @@ -4,6 +4,7 @@ import ( "fmt" "os/exec" "path/filepath" + "slices" "strings" "testing" ) @@ -13,49 +14,85 @@ func TestCIMinimumFailureCannotSatisfyAggregate(t *testing.T) { if err := validateCIRequiredAggregate(workflow); err != nil { t.Fatal(err) } - for _, status := range []string{"success", "failure", "skipped", "cancelled"} { - t.Run(status, func(t *testing.T) { - script := workflow.Jobs["ci-required-gate"].Steps[0].Run - for _, job := range []string{"source-quality", "browser-runtime", "platform-smoke", "platform-smoke-intel"} { - value := "success" - if job == "source-quality" { - value = status + // Both jobs that run the CPython 3.11.0 installed smoke are required + // leaves: a minimum failure in either job is a job result the aggregate + // must refuse, whatever the other leaves report. + leaves := []string{"source-quality", "browser-runtime", "platform-smoke", "platform-smoke-intel", "platform-smoke-linux-arm64"} + for _, failing := range ciMinimumPythonJobIDs() { + for _, status := range []string{"success", "failure", "skipped", "cancelled"} { + t.Run(failing+"/"+status, func(t *testing.T) { + script := workflow.Jobs["ci-required-gate"].Steps[0].Run + for _, job := range leaves { + value := "success" + if job == failing { + value = status + } + script = strings.ReplaceAll(script, "${{ needs."+job+".result }}", value) } - script = strings.ReplaceAll(script, "${{ needs."+job+".result }}", value) - } - command := exec.CommandContext(t.Context(), "bash", "-c", script) - output, err := command.CombinedOutput() - if (err == nil) != (status == "success") { - t.Fatalf("source-quality %s aggregate error=%v", status, err) - } - if status != "success" && strings.Contains(string(output), "OK:") { - t.Fatal("failed minimum lane emitted aggregate success") - } - }) + if strings.Contains(script, "${{") { + t.Fatalf("aggregate script references an unsubstituted job result: %q", script) + } + command := exec.CommandContext(t.Context(), "bash", "-c", script) + output, err := command.CombinedOutput() + if (err == nil) != (status == "success") { + t.Fatalf("%s %s aggregate error=%v", failing, status, err) + } + if status != "success" && strings.Contains(string(output), "OK:") { + t.Fatal("failed minimum lane emitted aggregate success") + } + }) + } + } +} + +// Each required job that runs the CPython 3.11.0 installed smoke names the +// step that produces its current release wheel set and the steps the smoke +// must precede. +var ciMinimumPythonJobs = map[string]struct { + producer string + successors []string +}{ + "source-quality": { + producer: "Build and verify package artifacts", + successors: []string{"Verify self-hosting receipts", "Verify self-hosting coverage", "Verify release closeout"}, + }, + linuxArm64PlatformSmokeJob: {producer: "Build release wheel set"}, +} + +func ciMinimumPythonJobIDs() []string { + ids := make([]string, 0, len(ciMinimumPythonJobs)) + for id := range ciMinimumPythonJobs { + ids = append(ids, id) } + slices.Sort(ids) + return ids } -func validateCIMinimumPythonStep(job githubJob) error { +func validateCIMinimumPythonStep(jobID string, job githubJob) error { + owner, ok := ciMinimumPythonJobs[jobID] + if !ok { + return fmt.Errorf("CI job %s has no owner Python minimum step", jobID) + } index, err := uniqueStepIndex(job.Steps, "Verify Python 3.11.0 installed wheel") if err != nil || index < 0 { - return fmt.Errorf("CI must contain exactly one Python minimum installed-wheel step") + return fmt.Errorf("CI job %s must contain exactly one Python minimum installed-wheel step", jobID) } step := job.Steps[index] - if step.Run != "go run ./internal/tools/pythonpackage verify-minimum" || !step.runPresent || + if step.Run != requiredMinimumPythonCommand || !step.runPresent || step.If != "" || step.ifPresent || step.ContinueOnError != nil || step.continueOnErrorPresent || step.Uses != "" || step.usesPresent || step.With != nil || step.withPresent || step.Env != nil || step.Shell != nil || step.shellPresent || step.WorkingDirectory != nil || step.workingDirectoryPresent || step.TimeoutMinutes != nil || step.timeoutMinutesPresent || step.ID != "" || step.idPresent { - return fmt.Errorf("CI Python minimum step must be the exact unconditional owner invocation") + return fmt.Errorf("CI job %s Python minimum step must be the exact unconditional owner invocation", jobID) } - buildIndex, err := uniqueStepIndex(job.Steps, "Build and verify package artifacts") + buildIndex, err := uniqueStepIndex(job.Steps, owner.producer) if err != nil || buildIndex < 0 || index != buildIndex+1 { - return fmt.Errorf("CI Python minimum step must immediately follow current package artifacts") + return fmt.Errorf("CI job %s Python minimum step must immediately follow %s", jobID, owner.producer) } - for _, successor := range []string{"Verify self-hosting receipts", "Verify self-hosting coverage", "Verify release closeout"} { + for _, successor := range owner.successors { next, err := uniqueStepIndex(job.Steps, successor) if err != nil || next <= index { - return fmt.Errorf("CI Python minimum step must precede %s", successor) + return fmt.Errorf("CI job %s Python minimum step must precede %s", jobID, successor) } } return nil @@ -63,48 +100,68 @@ func validateCIMinimumPythonStep(job githubJob) error { func TestCIMinimumPythonStepRejectsOmissionAndNeutralization(t *testing.T) { base := readWorkflowForTest(t, filepath.Join("..", ".github", "workflows", "ci.yml")) - if err := validateCIMinimumPythonStep(base.Jobs["source-quality"]); err != nil { - t.Fatal(err) + for _, jobID := range ciMinimumPythonJobIDs() { + if err := validateCIMinimumPythonStep(jobID, base.Jobs[jobID]); err != nil { + t.Fatal(err) + } } mutations := []struct { name string + jobs []string change func(*githubJob, int) }{ - {"omitted", func(job *githubJob, index int) { job.Steps = append(job.Steps[:index], job.Steps[index+1:]...) }}, - {"duplicate", func(job *githubJob, index int) { job.Steps = append(job.Steps, job.Steps[index]) }}, - {"before artifact", func(job *githubJob, index int) { + {"omitted", nil, func(job *githubJob, index int) { job.Steps = append(job.Steps[:index], job.Steps[index+1:]...) }}, + {"duplicate", nil, func(job *githubJob, index int) { job.Steps = append(job.Steps, job.Steps[index]) }}, + {"before artifact", nil, func(job *githubJob, index int) { job.Steps[index], job.Steps[index-1] = job.Steps[index-1], job.Steps[index] }}, - {"after receipts", func(job *githubJob, index int) { + {"after receipts", []string{"source-quality"}, func(job *githubJob, index int) { job.Steps[index], job.Steps[index+1] = job.Steps[index+1], job.Steps[index] }}, - {"no-op", func(job *githubJob, index int) { job.Steps[index].Run = "true" }}, - {"masked exit", func(job *githubJob, index int) { job.Steps[index].Run += " || true" }}, - {"background", func(job *githubJob, index int) { job.Steps[index].Run += " &" }}, - {"conditional", func(job *githubJob, index int) { job.Steps[index].If = "false" }}, - {"null condition", func(job *githubJob, index int) { job.Steps[index].ifPresent = true }}, - {"continue on error", func(job *githubJob, index int) { job.Steps[index].ContinueOnError = true }}, - {"null continue", func(job *githubJob, index int) { job.Steps[index].continueOnErrorPresent = true }}, - {"shadow environment", func(job *githubJob, index int) { job.Steps[index].Env = map[string]any{"PATH": "/other"} }}, - {"uses", func(job *githubJob, index int) { job.Steps[index].Uses = "other/action@main" }}, - {"wrong tool", func(job *githubJob, index int) { job.Steps[index].Run = "go run ./internal/tools/pythonpackage verify" }}, - } - for _, mutation := range mutations { - t.Run(mutation.name, func(t *testing.T) { - workflow := cloneWorkflow(t, base) - job := workflow.Jobs["source-quality"] - index, err := uniqueStepIndex(job.Steps, "Verify Python 3.11.0 installed wheel") - if err != nil { - t.Fatal(err) - } - mutation.change(&job, index) - workflow.Jobs["source-quality"] = job - if err := validateCIMinimumPythonStep(job); err == nil { - t.Fatal("minimum step mutation admitted by independent owner predicate") + {"before platform smoke", []string{linuxArm64PlatformSmokeJob}, func(job *githubJob, index int) { + step := job.Steps[index] + job.Steps = append(job.Steps[:index], job.Steps[index+1:]...) + smoke, err := uniqueStepIndex(job.Steps, "Run platform smoke") + if err != nil || smoke < 0 { + panic("owner Linux arm64 workflow lost Run platform smoke") } - if err := validateCIRequiredAggregate(workflow); err == nil { - t.Fatal("minimum step mutation admitted by aggregate oracle") + job.Steps = slices.Insert(job.Steps, smoke, step) + }}, + {"wheel set omitted", []string{linuxArm64PlatformSmokeJob}, func(job *githubJob, index int) { + job.Steps = append(job.Steps[:index-1], job.Steps[index:]...) + }}, + {"no-op", nil, func(job *githubJob, index int) { job.Steps[index].Run = "true" }}, + {"masked exit", nil, func(job *githubJob, index int) { job.Steps[index].Run += " || true" }}, + {"background", nil, func(job *githubJob, index int) { job.Steps[index].Run += " &" }}, + {"conditional", nil, func(job *githubJob, index int) { job.Steps[index].If = "false" }}, + {"null condition", nil, func(job *githubJob, index int) { job.Steps[index].ifPresent = true }}, + {"continue on error", nil, func(job *githubJob, index int) { job.Steps[index].ContinueOnError = true }}, + {"null continue", nil, func(job *githubJob, index int) { job.Steps[index].continueOnErrorPresent = true }}, + {"shadow environment", nil, func(job *githubJob, index int) { job.Steps[index].Env = map[string]any{"PATH": "/other"} }}, + {"uses", nil, func(job *githubJob, index int) { job.Steps[index].Uses = "other/action@main" }}, + {"wrong tool", nil, func(job *githubJob, index int) { job.Steps[index].Run = "go run ./internal/tools/pythonpackage verify" }}, + } + for _, jobID := range ciMinimumPythonJobIDs() { + for _, mutation := range mutations { + if mutation.jobs != nil && !slices.Contains(mutation.jobs, jobID) { + continue } - }) + t.Run(jobID+"/"+mutation.name, func(t *testing.T) { + workflow := cloneWorkflow(t, base) + job := workflow.Jobs[jobID] + index, err := uniqueStepIndex(job.Steps, "Verify Python 3.11.0 installed wheel") + if err != nil { + t.Fatal(err) + } + mutation.change(&job, index) + workflow.Jobs[jobID] = job + if err := validateCIMinimumPythonStep(jobID, job); err == nil { + t.Fatal("minimum step mutation admitted by independent owner predicate") + } + if err := validateCIRequiredAggregate(workflow); err == nil { + t.Fatal("minimum step mutation admitted by aggregate oracle") + } + }) + } } } diff --git a/scripts/workflow_python_runtime_test.go b/scripts/workflow_python_runtime_test.go index d09e4f4e..6f2aa03f 100644 --- a/scripts/workflow_python_runtime_test.go +++ b/scripts/workflow_python_runtime_test.go @@ -21,7 +21,7 @@ var hostPythonConsumer = regexp.MustCompile(`\bpython3\b|npm run (check|go:test| // Jobs whose run steps consume the host Python. Every other job must not. var expectedPythonJobs = map[string][]string{ - "ci.yml": {"platform-smoke", "platform-smoke-intel", "source-quality"}, + "ci.yml": {"platform-smoke", "platform-smoke-intel", "platform-smoke-linux-arm64", "source-quality"}, "release.yml": {"candidate", "publish-pypi", "publish-readiness"}, } @@ -175,6 +175,16 @@ func TestWorkflowPythonRuntimeRejectsUnpinnedConsumers(t *testing.T) { {"x86_64 platform runtime drift", "ci.yml", "platform-smoke-intel", func(job *githubJob) { job.Steps[setupStep(*job)].With["python-version"] = "3.14.7" }}, + {"Linux arm64 platform setup omitted", "ci.yml", "platform-smoke-linux-arm64", func(job *githubJob) { + index := setupStep(*job) + job.Steps = append(job.Steps[:index], job.Steps[index+1:]...) + }}, + {"Linux arm64 platform setup after smoke", "ci.yml", "platform-smoke-linux-arm64", func(job *githubJob) { + index := setupStep(*job) + step := job.Steps[index] + job.Steps = append(job.Steps[:index], job.Steps[index+1:]...) + job.Steps = append(job.Steps, step) + }}, {"new unpinned consumer", "release.yml", "release-metadata", func(job *githubJob) { job.Steps = append(job.Steps, githubStep{Name: "Read version", Run: "python3 -c 'print(1)'", runPresent: true}) }}, From c659893ad64d59f433f52d0a2888386c49fbcf33 Mon Sep 17 00:00:00 2001 From: iperev Date: Fri, 23 Oct 2026 22:25:57 +0200 Subject: [PATCH 3/6] test(browser): gate rendered views on owner-admitted reference digests Add a visual reference gate for the rendered requirement views. npm run browser:visual, which npm run check does not run, compares fourteen views (the source, spec-tree, structured proof, compact proof and coverage static renderings in Cards and Table modes, and the Specifications, Coverage, Diff and Traceability workspace tabs) in Chromium, Firefox and Playwright WebKit, light and dark, at 320 and 1440 CSS pixels, plus one Chromium README scene: 169 references. A reference is a digest, not an image. tests/visual/references.json binds, for each reference ID /.., the width, the height and the SHA-256 of the canonical pixel encoding that REQ-PROOFKIT-QUALITY-028 specifies. No reference image is stored in the repository. tests/visual/environment.json records the reference environment: the mcr.microsoft.com/playwright:v1.63.0-noble image with its index and platform digests on linux/arm64, and the observed OS identity, machine architecture, image marker, fontconfig, FreeType, HarfBuzz and font package versions, the SHA-256 of the committed tests/visual/fonts.conf, the fc-match results for monospace, sans-serif, serif, system-ui and ui-monospace, the installed engine revisions, the absence of emulation and the launched browser versions. fc-match reads the text after an unescaped hyphen as a point size, so each family is queried by its whole name with every hyphen escaped. fonts.conf includes the system configuration and prefers Liberation Sans for sans-serif and system-ui, Liberation Serif for serif and Liberation Mono for monospace and ui-monospace, which have real bold faces. The system configuration appends sans-serif to every pattern that names no generic family, so the strong sans-serif preference makes each family fontconfig does not know a strong alias of Liberation Sans, which WebKit accepts for the first family that WebKit looks up in fontconfig; without the ui-monospace mapping WebKit drew the product's monospace stacks in Liberation Sans. The gate copies fonts.conf into its run directory and sets the copy as FONTCONFIG_FILE before observing, so fc-match and every browser read the hashed bytes. The engine fixture requires the copy to equal the test snapshot's fonts.conf and requires that no project sets its own browser environment, so every browser inherits that FONTCONFIG_FILE. references.json also binds the SHA-256 of the environment record's canonical JSON. The gate observes the environment, writes it with the CPU vendor, model and features to observed-environment.json in its run directory and prints the CPU before launching any browser. On another environment it records the state unverified with the divergent fields and exits 0; --require-reference-environment makes that state exit 1. Firefox drops color-scheme emulation at a navigation into or out of a document served with Cross-Origin-Opener-Policy: same-origin unless both documents have the same origin and send that header, including the first navigation from about:blank; emulation applied after the last such navigation holds across reload and across navigation within one fixture. The gate therefore applies the color scheme and reduced motion after the first navigation, reloads, applies them again after the reload, the last navigation of every captured view, and asserts both through the matching media queries before each capture. It fits the window to the document height, cut at 2400 CSS pixels, captures each view twice, requires byte-equal PNG captures, decodes them with node:zlib, refusing tRNS outside palette images and a second IHDR, and requires the dimensions and the canonical digest to equal the inventory entry; a failure names the ID and both digests and sizes. Every reference-environment run, passing or failing, keeps its captures, a candidate inventory and the observed environment below its run directory, and a run after which the materialized tests/visual tree gained, lost or changed a file fails. Six controls per engine serve a copied workspace stylesheet: a routed unchanged copy must keep the admitted digest, so routing alone cannot change the pixels, and a one-unit text color change, a one-pixel shift, a font-weight change from 400 to 700, a change of the sans-serif family to serif and a change of the monospace family to sans-serif must each change it. A font-face check per engine reads the font stacks of the workspace and static roots and of the workspace and static monospace elements from their rendered views, draws each beside the named Liberation faces in regular and bold, and requires exact pixel equality with Liberation Sans for the roots and with Liberation Mono for the monospace stacks, which must also be monospaced. The visual Chromium project adds --disable-partial-raster, without which repeated launches rasterized rounded corners differently, and the visual Firefox project adds browser.chrome.site_icons=false; the runtime harness is unchanged. One runtime-harness site with the same Firefox pattern remains: tests/browser/workspace-evidence-layout.spec.mjs emulates the color scheme before its first navigation, so its two dark Firefox rows render light; the runtime harness site is fixed by the next change in this chain, test(browser): assert media emulation in the document under test. browserproofverify selects its input manifest, witness command, record path, admissible states and run-directory retention from a closed table. The runtime manifest, witness command, record path, record format and run cleanup are unchanged. The visual row keeps its run directory, writes artifacts/proofkit/browser-visual-proof.json with the state passed or unverified, recomputes the environment divergence itself, requires each engine to pass exactly its expected comparisons, controls and font-face check with zero retries, and refuses an inventory with a missing, extra, duplicate or malformed entry, dimensions out of bounds or a binding to another environment. Any failure exits non-zero without writing a record. References are admitted by the owner action --admit-references --from , which the gate never calls. It admits only a run whose observed environment equals the recorded one and that holds exactly one regular capture file per ID and no unstable directory, recomputes every digest from the PNG captures through the same canonical encoding, refuses a run whose candidate inventory disagrees or whose observed environment claims a divergence, and only then writes references.json. It cannot authenticate the run directory as unmodified gate output; only a passing gate on the admitting tree proves the admitted digests. browsertestserver gains a compact proof selector and a README project selector, which materializes a committed synthetic example.dispatch project, serves it through the same project inspection path as the view command and removes the temporary project on every exit path. A test admits that project through the project plan. CI gains the required job "quality / browser visual references" on ubuntu-24.04-arm in the digest-pinned image with --user 1001 --ipc=host --init. It sets up Node 26.11.1, the verified npm, Go and the locked tooling, runs npm run browser:visual -- --require-reference-environment, uploads the captures, the candidate inventory and the observed environment after every outcome as browser-visual-captures-- for 30 days, and uploads the record only after success. The required aggregate needs the job, and REQ-PROOFKIT-QUALITY-011 now counts six required leaf jobs in the aggregate. REQ-PROOFKIT-QUALITY-025 admits exactly this job container and requires every workflow container image reference to be pinned by a sha256 digest; the workflow oracles pin the job's container, steps, uploads, runner, timeout and aggregate membership and refuse an unpinned image. REQ-PROOFKIT-QUALITY-028 states the reference gate, the canonical pixel encoding, the font-face check and the admission with its bindings, coverage inventory entries, overview line and non-claims: weights 400 and 500, and weights 600 to 900, each render identically because only regular and bold faces are installed; a family name that WebKit looks up and fontconfig does not know, other than ui-monospace, resolves to Liberation Sans in WebKit; pixel identity is claimed only on the calibrated Apple M1 and Arm Neoverse N2 CPU models; admission does not authenticate its run directory; admitted images are reproducible only while the pinned image and the pinned toolchain and dependency downloads remain available. The binding record carries the requirement's source non-claims. The witness plan adds the browser-visual command and policy, the container-linux-arm64-browser environment class and the browser-visual parallel group, and the go-test policy lists the visual records and the digest script its tests read. CONTRIBUTING.md explains how to view an admitted image from a clone in the pinned image with checksum-verified toolchains and how references are re-admitted, and requires, before merge, a passing gate on each CPU model named in the non-claims or a change of those non-claims. The UI-DESIGN-01 backlog row now says that reference screenshot digests are committed and the screenshots are reproducible in the recorded environment, as the owner decided. Proof on this tree: A local npm run check sequence on darwin/arm64 passed (exit 0) on tree f67bb17cbdf4 with Node 26.11.1, npm 12.2.0, CPython 3.15.0 and Go 1.27.3: 561/561 runtime browser cases in Chromium, Firefox and WebKit with zero retries. One recorded deviation: the host firewall blocks Go-built binaries from the network, so go:vulncheck ran against a vulnerability-database snapshot fetched with curl (modified 2026-10-21T00:27:29Z): no vulnerabilities. npm run check does not run the visual gate. Its proof is two fresh-container runs in the pinned arm64 image on Apple M1 (190/190 each, candidates equal to references.json) and three hosted CI runs of this tree on Arm Neoverse N2 (190/190 each, all jobs green). --- .github/workflows/ci.yml | 56 + BACKLOG.md | 2 +- CONTRIBUTING.md | 81 ++ .../proofkit-supply-chain-quality/overview.md | 19 +- .../requirements.v2.json | 5 +- internal/testsupport/browserfixture/readme.go | 54 + .../testsupport/browserfixture/readme_test.go | 23 + .../docs/specs/dispatch/requirements.v2.json | 21 + .../readme-project/proofkit/bindings.json | 141 +++ .../readme-project/proofkit/project.v1.json | 29 + .../readme-project/proofkit/tests.json | 146 +++ .../browserproofverify/artifact_paths.go | 7 +- .../browserproofverify/artifact_paths_test.go | 20 +- .../browserproofverify/input_manifest.go | 13 +- .../browserproofverify/input_resolution.go | 12 +- internal/tools/browserproofverify/main.go | 183 ++- .../tools/browserproofverify/main_test.go | 10 +- .../operation_guard_test.go | 8 +- .../browserproofverify/playwright_report.go | 31 +- .../playwright_report_test.go | 8 +- .../tools/browserproofverify/proof_kinds.go | 76 ++ .../browserproofverify/visual_inventory.go | 654 +++++++++++ .../visual_inventory_test.go | 568 +++++++++ .../tools/browserproofverify/visual_record.go | 181 +++ .../browserproofverify/visual_record_test.go | 309 +++++ internal/tools/browsertestserver/main.go | 40 +- .../coveragemetrics/required_inventory.go | 54 + package.json | 3 +- playwright.visual.config.mjs | 43 + proofkit/requirement-bindings.json | 160 +++ proofkit/witness-plan.json | 149 ++- scripts/browser-proof-execution.mjs | 8 +- scripts/browser-visual-capture-digests.mjs | 16 + scripts/browser-visual-proof-inputs.v1.json | 20 + scripts/browser-visual-references.test.mjs | 232 ++++ scripts/diagnostic-entrypoints.test.mjs | 9 +- .../workflow_browser_visual_oracle_test.go | 327 ++++++ scripts/workflow_oracle_support_test.go | 65 ++ scripts/workflow_package_gate_oracle_test.go | 52 +- scripts/workflow_python_minimum_test.go | 2 +- scripts/write-browser-visual-proof.mjs | 149 +++ tests/visual/capture-inventory.mjs | 27 + tests/visual/controls.spec.mjs | 71 ++ tests/visual/environment-probe.mjs | 180 +++ tests/visual/environment.json | 56 + tests/visual/exact-png.mjs | 149 +++ tests/visual/font-faces.spec.mjs | 103 ++ tests/visual/fonts.conf | 24 + tests/visual/readme-scene.spec.mjs | 28 + tests/visual/references.json | 1022 +++++++++++++++++ tests/visual/references.spec.mjs | 14 + tests/visual/visual-harness.mjs | 286 +++++ 52 files changed, 5816 insertions(+), 130 deletions(-) create mode 100644 internal/testsupport/browserfixture/readme.go create mode 100644 internal/testsupport/browserfixture/readme_test.go create mode 100644 internal/testsupport/browserfixture/testdata/readme-project/docs/specs/dispatch/requirements.v2.json create mode 100644 internal/testsupport/browserfixture/testdata/readme-project/proofkit/bindings.json create mode 100644 internal/testsupport/browserfixture/testdata/readme-project/proofkit/project.v1.json create mode 100644 internal/testsupport/browserfixture/testdata/readme-project/proofkit/tests.json create mode 100644 internal/tools/browserproofverify/proof_kinds.go create mode 100644 internal/tools/browserproofverify/visual_inventory.go create mode 100644 internal/tools/browserproofverify/visual_inventory_test.go create mode 100644 internal/tools/browserproofverify/visual_record.go create mode 100644 internal/tools/browserproofverify/visual_record_test.go create mode 100644 playwright.visual.config.mjs create mode 100644 scripts/browser-visual-capture-digests.mjs create mode 100644 scripts/browser-visual-proof-inputs.v1.json create mode 100644 scripts/browser-visual-references.test.mjs create mode 100644 scripts/workflow_browser_visual_oracle_test.go create mode 100644 scripts/write-browser-visual-proof.mjs create mode 100644 tests/visual/capture-inventory.mjs create mode 100644 tests/visual/controls.spec.mjs create mode 100644 tests/visual/environment-probe.mjs create mode 100644 tests/visual/environment.json create mode 100644 tests/visual/exact-png.mjs create mode 100644 tests/visual/font-faces.spec.mjs create mode 100644 tests/visual/fonts.conf create mode 100644 tests/visual/readme-scene.spec.mjs create mode 100644 tests/visual/references.json create mode 100644 tests/visual/references.spec.mjs create mode 100644 tests/visual/visual-harness.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 345a9a6a..8be00deb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -369,6 +369,60 @@ jobs: if-no-files-found: error retention-days: 14 + browser-visual: + name: quality / browser visual references + runs-on: ubuntu-24.04-arm + timeout-minutes: 20 + container: + image: mcr.microsoft.com/playwright:v1.63.0-noble@sha256:eff16c30e6f3f4af0a03fa4b706120d5e9b0891c344a27d64559aff5900a4a27 + options: --user 1001 --ipc=host --init + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 26.11.1 + package-manager-cache: false + + - name: Install verified npm + uses: ./.github/actions/setup-verified-npm + + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + + - name: Install locked browser tooling + run: npm ci --ignore-scripts + + - name: Run visual reference proof + run: npm run browser:visual -- --require-reference-environment + + - name: Upload visual reference captures + if: ${{ always() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: browser-visual-captures-${{ github.sha }}-${{ github.run_attempt }} + path: | + artifacts/browser-run-*/captures + artifacts/browser-run-*/candidate-references.json + artifacts/browser-run-*/observed-environment.json + if-no-files-found: error + retention-days: 30 + + - name: Upload visual reference proof + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: browser-visual-proof-${{ github.sha }} + path: artifacts/proofkit/browser-visual-proof.json + if-no-files-found: error + retention-days: 14 + ci-required-gate: name: quality / required aggregate runs-on: ubuntu-24.04 @@ -376,6 +430,7 @@ jobs: if: ${{ always() }} needs: - browser-runtime + - browser-visual - source-quality - platform-smoke - platform-smoke-intel @@ -385,6 +440,7 @@ jobs: run: | set -euo pipefail test "${{ needs.browser-runtime.result }}" = "success" + test "${{ needs.browser-visual.result }}" = "success" test "${{ needs.source-quality.result }}" = "success" test "${{ needs.platform-smoke.result }}" = "success" test "${{ needs.platform-smoke-intel.result }}" = "success" diff --git a/BACKLOG.md b/BACKLOG.md index d8152f75..50e05a42 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -49,7 +49,7 @@ records, generated release manifests, or the owning docs named above. | Status | ID | Scope | Completion condition | |---|---|---|---| | BLOCKED | BROWSER-TOOLCHAIN-01 | Qualify a stable browser-test successor to the pinned 1.63.0 SDK that contains the Firefox channel-identity repair ([microsoft/playwright@e5d880b](https://github.com/microsoft/playwright/commit/e5d880b05f9a1d7fb5d17dd031b53739e5e39c0e), Firefox r1551, first released in 1.64.0; 1.63.0 ships r1543 and lacks it) and lacks the native WebKit select/CSP regression first shipped in 1.64.0: WebKit commit `ef67848e5875` (bug 322676) sets a style attribute on a select element's internal slot before that slot is in the user-agent shadow tree, so the page's `style-src` refuses it and logs a console error (open as WebKit bug 327212). | Unblocked only by a stable Playwright release whose Firefox is r1551 or later and whose WebKit no longer reproduces that select/CSP regression (a native `select` under the workspace CSP logs no console or security-policy error); as of 2026-10-13, 1.64.0, the only stable successor, still has the regression. The complete Chromium, Firefox and WebKit gate must then pass with native navigation, exact engine evidence, strict CSP, console and security-policy checks and zero retries. Keep Firefox site-origin process isolation: on 1.63.0, Firefox runs without it reproduced the navigation completion failure under host load. Remove it only when at least five complete Firefox runs of the successor without it, interleaved with control runs that keep it and including runs under host load at least as high as the 1.63.0 complete run that reproduced the failure (1-minute load average median 19 on 8 CPUs), all pass with zero retries and no navigation completion failure; update its fixture and contributor explanation together with the policy change. Do not substitute a custom engine or ignore console/security-policy failures. | -| NEXT | UI-DESIGN-01 | Make every rendered requirement page functional and visually polished under one design system: the static source, proof, coverage and specification-tree views (styles owned by `internal/kernel/browserdoc`) and the `requirement-browser-server` workspace (styles owned by `internal/command/requirementbrowser/assets/workspace.css`) currently keep separate palettes, type and spacing. Renderings stay derived, self-contained and offline: no external fonts, scripts or assets, and no weaker CSP. | One shipped set of design tokens (type scale, spacing, light and dark palettes, radii, border widths, layout widths, target sizes and focus indicator) is defined once and used by both style owners; a test over those two style owners refuses a literal color or size outside the token set, except media-query breakpoints, which CSS cannot read from tokens and which the test requires to equal their tokens, as it also requires of every `matchMedia` query in the workspace scripts, and graph geometry, which stays with its geometry owner. In Chromium, Firefox and WebKit, light and dark, at 320 and 1440 CSS px, and at 200% text size produced both by root font-size scaling at those widths and by full-page zoom, emulated as a 720 CSS px viewport with device scale factor 2 (a 1440 px window at 200%): every view passes axe with zero violations, color-contrast (WCAG 2.2 AA) and target-size each apply to at least one node, target-size has no incomplete result, and every node with an incomplete color-contrast result passes the computed-contrast oracle; the document reflows without horizontal scrolling or lost content, and only labelled graph and table regions may scroll horizontally, as REQ-PROOFKIT-QUALITY-022 admits; keyboard operation, visible focus, Unicode selection and the existing CSP, console and security-policy checks hold with zero retries. Task tests prove that a reader can find a requirement, read its scenarios, follow its proof binding, see coverage gaps and read a diff in every engine. Reference screenshots for each view, engine, color scheme and width are committed for one recorded reference environment (the CI ubuntu-24.04 runner image version, its installed font, fontconfig, FreeType and HarfBuzz package versions and the pinned browser engines, or a digest-pinned container image that fixes them), and the gate compares them with a fixed pixel threshold, never writes missing references and fails on a missing or changed one; a host whose recorded environment differs reports the comparison as unverified, and any change of that environment, including an engine upgrade, needs re-admitted references. The owner reviews the references before and after and admits the visual result; the README screenshot is refreshed from the admitted references. Who reviewed a reference change, a human usability study and WCAG conformance certification remain non-claims. | +| NEXT | UI-DESIGN-01 | Make every rendered requirement page functional and visually polished under one design system: the static source, proof, coverage and specification-tree views (styles owned by `internal/kernel/browserdoc`) and the `requirement-browser-server` workspace (styles owned by `internal/command/requirementbrowser/assets/workspace.css`) currently keep separate palettes, type and spacing. Renderings stay derived, self-contained and offline: no external fonts, scripts or assets, and no weaker CSP. | One shipped set of design tokens (type scale, spacing, light and dark palettes, radii, border widths, layout widths, target sizes and focus indicator) is defined once and used by both style owners; a test over those two style owners refuses a literal color or size outside the token set, except media-query breakpoints, which CSS cannot read from tokens and which the test requires to equal their tokens, as it also requires of every `matchMedia` query in the workspace scripts, and graph geometry, which stays with its geometry owner. In Chromium, Firefox and WebKit, light and dark, at 320 and 1440 CSS px, and at 200% text size produced both by root font-size scaling at those widths and by full-page zoom, emulated as a 720 CSS px viewport with device scale factor 2 (a 1440 px window at 200%): every view passes axe with zero violations, color-contrast (WCAG 2.2 AA) and target-size each apply to at least one node, target-size has no incomplete result, and every node with an incomplete color-contrast result passes the computed-contrast oracle; the document reflows without horizontal scrolling or lost content, and only labelled graph and table regions may scroll horizontally, as REQ-PROOFKIT-QUALITY-022 admits; keyboard operation, visible focus, Unicode selection and the existing CSP, console and security-policy checks hold with zero retries. Task tests prove that a reader can find a requirement, read its scenarios, follow its proof binding, see coverage gaps and read a diff in every engine. Reference screenshot digests for each view, engine, color scheme and width are committed for one recorded reference environment (the CI ubuntu-24.04 runner image version, its installed font, fontconfig, FreeType and HarfBuzz package versions and the pinned browser engines, or a digest-pinned container image that fixes them), in which the screenshots are reproducible, and the gate compares them with a fixed pixel threshold, never writes missing references and fails on a missing or changed one; a host whose recorded environment differs reports the comparison as unverified, and any change of that environment, including an engine upgrade, needs re-admitted references. The owner reviews the references before and after and admits the visual result; the README screenshot is refreshed from the admitted references. Who reviewed a reference change, a human usability study and WCAG conformance certification remain non-claims. | | NEXT | GATE-DIFF-01 | Let a closeout gate inherit the verdicts of unchanged steps from a passing parent tree instead of rerunning every step. Inheritance is unsound today: `proofkit/witness-plan.json` input selectors do not declare every path some gate steps read (`go:test` and `self:coverage` read every tracked file, and `browser:check` and the Go format, vet and staticcheck steps read paths outside their selectors), and `go:vulncheck` is not deterministic because it consults a network vulnerability database. | Every gate, full or differential, traces each step it runs and fails when a file read, directory enumeration or failed path lookup is not covered by that step's declared selectors (an enumerated directory only by a selector over its whole subtree, a failed lookup only by a selector that matches the probed path), and every step declares in its `proofkit/witness-plan.json` policy, validated by `witness-scheduler-plan`, whether its verdict is a deterministic function of its inputs, pinned toolchain and platform; the witness plan's cache policies, environment-class vocabulary and reuse non-claims are reconciled with that reuse in the same change. Only then does `AGENTS.md` admit differential closeout: a step's verdict is inherited from the parent's passing gate, run on a clean materialization of the parent's exact tree, only when the run that produced it passed that trace check, the step is deterministic, none of its selectors matches a changed path (a rename counts both paths), and the toolchain (Go, Node, npm and the browser engines), operating system, architecture and the step's environment are identical; nondeterministic steps and steps without selectors always run, and the full gate runs for release objects, for changes to the gate definition (`package.json`, `package-lock.json`, `go.mod`, `go.sum`, `go.work`, `proofkit/witness-plan.json`, `playwright.config.mjs`, `tsconfig.browser.json`, `.github` and the sources of the trace check and the inheritance decision) and for changes that match a selector of `self:receipt` or `release:closeout`, which run only in a full gate. Mutation tests show that an undeclared read, a new file in an enumerated directory, a previously absent probed path (for example `vendor/modules.txt`), a read added by a step rerun in a differential gate, a rename, a nondeterministic step, a step without selectors, a toolchain or platform change, a gate-definition change, a change to a `self:receipt` input and a release object each force a rerun or the full gate. Until then every publishable object runs the full gate. | | BLOCKED | GOVERNANCE-01 | Evaluate a generic explicit-inventory governance-observation command without promoting one consumer's policy into Proofkit; detailed candidate contract is retained in [issue #64](https://github.com/research-engineering/agentic-proofkit/issues/64). | A sanitized reproducible fixture detects one named failure without classifying its false-positive counterexample, existing owners are proven insufficient, and either a second independent consumer reproduces the predicate or the owner admits recurring first-consumer cost; otherwise retire the candidate. | | DEFERRED | VALUE-01 | Admit exact value-evidence comparisons only after a real producer and downstream consumer establish the public record boundary; detailed candidate contract is retained in [issue #65](https://github.com/research-engineering/agentic-proofkit/issues/65). | A real execution-receipt projection, baseline producer, and downstream consumer prove an exact producer-output-to-admission round trip plus compact/full-graph inclusion or an intentional omission non-claim; otherwise no public command is added. | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 059450b1..669f3c07 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,6 +82,87 @@ reproduced the failure (1-minute load average median 19 on 8 CPUs), all pass with zero retries and no navigation completion failure, and update the fixture and this explanation together with it. +The visual reference gate, `npm run browser:visual`, is separate from +`npm run check`. It compares a canonical pixel digest of every rendered view +with the owner-admitted entry in `tests/visual/references.json`, but only in +the environment recorded in `tests/visual/environment.json`: the digest-pinned +linux/arm64 `mcr.microsoft.com/playwright` image that the +`quality / browser visual references` job runs on `ubuntu-24.04-arm`, with the +font mapping of `tests/visual/fonts.conf`, which the gate copies into its run +directory before it observes the environment. On any other host it detects the +difference before launching a browser, records `unverified` and exits 0; CI +adds `--require-reference-environment`, which makes that a failure. No +reference image is stored in the repository, and the gate never writes +references. Every reference-environment run keeps its captures in +`artifacts/browser-run-*/captures//...png`, with +`candidate-references.json` and `observed-environment.json` beside them; CI +uploads these as the `browser-visual-captures--` artifact after +every run and keeps it for 30 days. A local reference-environment run keeps +about 30 MB below its `artifacts/browser-run-*` directory, which you may +delete once you no longer need its captures. + +To view an admitted image, run the gate at the admitting commit in the pinned +image and look at its captures; when it passes, every capture has the admitted +digest. Use a separate clone rather than a worktree, because the container +sees only the mounted directory and installs Linux `node_modules` into it. The +container user, uid 1001, must be able to write the clone: Docker Desktop +allows that, and on a Linux host the clone must belong to uid 1001. The +container installs the same checksum-verified Node, npm and Go versions as the +CI job: + +```bash +git clone https://github.com/research-engineering/agentic-proofkit.git proofkit-visual +cd proofkit-visual +git checkout +docker run --rm --platform linux/arm64 --user 1001 --ipc=host --init \ + -v "$PWD":/work -w /work \ + mcr.microsoft.com/playwright:v1.63.0-noble@sha256:eff16c30e6f3f4af0a03fa4b706120d5e9b0891c344a27d64559aff5900a4a27 \ + bash -euo pipefail -c ' + git config --global --add safe.directory /work + tools="$(mktemp -d)" + cd "$tools" + curl -fsSLO https://nodejs.org/dist/v26.11.1/node-v26.11.1-linux-arm64.tar.gz + curl -fsSLO https://go.dev/dl/go1.27.3.linux-arm64.tar.gz + curl -fsSLO https://registry.npmjs.org/npm/-/npm-12.2.0.tgz + sha256sum -c - < bin/npm + chmod +x bin/npm + export PATH="$tools/bin:$tools/node-v26.11.1-linux-arm64/bin:$tools/go/bin:$PATH" + cd /work + npm ci --ignore-scripts + npm run browser:visual -- --require-reference-environment' +``` + +To re-admit references after an intended visual change, let the job fail, +download its `browser-visual-captures-*` artifact, review the captures, and +regenerate `tests/visual/references.json` from that run with +`go run ./internal/tools/browserproofverify --manifest scripts/browser-visual-proof-inputs.v1.json --admit-references --from /browser-run-`; +the artifact holds the run directory `browser-run-` below its root. +Admission accepts only a run whose observed environment equals the recorded +one, with exactly one regular capture file per reference ID and no unstable +directory; it recomputes every digest from the PNG files. It cannot tell +whether the run directory changed after the run, so only a passing job proves +the admitted digests. Push until the job passes on the exact final tree. +Before merge, pass the gate on each CPU model named in the non-claims, or +change those non-claims. Merging is the owner's admission. An environment +change, including a Playwright upgrade, takes two steps. First, update +`observed` in `tests/visual/environment.json` from the +`observed-environment.json` of a run in the new environment; that run reports +`unverified` and launches no browser. +Second, run again: the browsers now launch, and if a launched version differs +the engine fixture fails with that version, which then goes into +`browserVersions`. Because `environmentId` changed, that run cannot pass until +its captures are admitted as above. The same image runs natively on Apple +Silicon; an emulated platform is always `unverified`. + The composed `go:check` and CI source job both run `npm run go:deps`: `go mod tidy -diff` rejects manifest drift without rewriting `go.mod` or `go.sum`, then `go mod verify` checks the downloaded module cache. These are diff --git a/docs/specs/proofkit-supply-chain-quality/overview.md b/docs/specs/proofkit-supply-chain-quality/overview.md index 9b39c2a3..b6caf0dd 100644 --- a/docs/specs/proofkit-supply-chain-quality/overview.md +++ b/docs/specs/proofkit-supply-chain-quality/overview.md @@ -189,10 +189,11 @@ vulnerability absence, or consumer rollout safety by itself. downloadable artifact-relative paths without inferring change completeness from the source diff. - `REQ-PROOFKIT-QUALITY-025`: exact workflow inventory, closed raw-YAML keys - before typed decoding, exact release deployment environments, pinned action - commit identities, admitted status expressions, the required aggregate shell - program, and read-only treatment of an existing release are source-oracle - invariants. + before typed decoding, exact release deployment environments, one exact + job container for the visual reference job, sha256-digest pins for every + workflow container image reference, pinned action commit identities, + admitted status expressions, the required aggregate shell program, and + read-only treatment of an existing release are source-oracle invariants. - `REQ-PROOFKIT-QUALITY-026`: every public or retained-evidence stable JSON implementation accepts only the repository-pinned Unicode scalar set and matches one preimplementation byte corpus for the pinned control, format, and separator @@ -208,6 +209,16 @@ vulnerability absence, or consumer rollout safety by itself. Shared unsupported-key diagnostics preserve their context argument and report an unknown-key count instead of unknown names or values. Context and ordinary report fields retain their separate admission owners. +- `REQ-PROOFKIT-QUALITY-028`: every rendered view in each engine, color scheme + and width, plus the README scene, has an owner-admitted canonical pixel + digest bound to one recorded, digest-pinned linux/arm64 container environment + with a committed font configuration; no reference image is stored. The + separate visual gate observes and records the environment before any browser + launch, reports another environment as unverified, captures each view twice, + requires equal dimensions and digest, keeps its captures for review, never + writes into the test directory and fails on a missing, changed or extra + reference. Admission recomputes the digests from a reference-environment + run's captures. ## Non-Claims diff --git a/docs/specs/proofkit-supply-chain-quality/requirements.v2.json b/docs/specs/proofkit-supply-chain-quality/requirements.v2.json index 5a455c75..e77b0c1e 100644 --- a/docs/specs/proofkit-supply-chain-quality/requirements.v2.json +++ b/docs/specs/proofkit-supply-chain-quality/requirements.v2.json @@ -21,7 +21,7 @@ , {"requirementId":"REQ-PROOFKIT-QUALITY-008","statementCompletion":"GitHub Actions workflow syntax and expression semantics are checked by actionlint in the local package gate.","fields":{"nonClaims":["This requirement does not claim workflow runtime success, external action safety, branch protection, or release approval."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-008"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"medium","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} , {"requirementId":"REQ-PROOFKIT-QUALITY-009","statementCompletion":"Performance-sensitive parser and serializer paths expose benchmark entrypoints without making wall-clock budgets a required PR gate before stable baselines exist.","fields":{"nonClaims":["This requirement does not claim current performance maximum, CI wall-clock budget enforcement, or regression absence without benchmark evidence."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-009"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"medium","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} , {"requirementId":"REQ-PROOFKIT-QUALITY-010","statementCompletion":"Coverage metrics report requirement, binding, witness, CLI inventory linkage, descriptor-owned command proof-route candidates, and a separate execution-backed command-oracle ledger. The ledger must bind every candidate commandRef, selector, concrete negative case, falsification event, assertion oracle, expected public outcome, and owner invariant to one immutable exact-file materialized source snapshot; execute exactly the selected active Go tests through canonically ordered package-scoped argv vectors that cannot select same-named non-candidate tests in another package; require unique package start and pass events, unique test run and pass events, owner-reserved cooperative attributes for every candidate, bounded cancellable subprocesses, and immediate process-group termination on output-limit violation; reject missing, unknown, duplicate, skipped, failed, reordered, context-invalid, spoofed-output, cross-test, source-drift, producer-unreachable candidate projections, and correlated identity substitutions; publish and invalidate local diagnostic and metrics artifacts atomically through repository-confined non-symlink destinations; on supported release platforms, open retained input artifacts descriptor-first through a nonblocking no-follow boundary before type, identity, and exact byte-limit checks so named pipes, symlinks, path substitution, and post-read mutation cannot block or escape, and fail closed instead of using a weaker file-open path elsewhere; revalidate current candidate, corpus, runtime, and source identities before and after artifact publication; and fail closed when any public command lacks execution-backed evidence. An independently authored versioned counterfeit corpus must bijectively cover every shipped evidence class, required policy axis, record coordinate, and declared substitution axis with positive controls and checked-in exact expected decisions evaluated through the production admission and lifecycle owners rather than a generated expectation registry. Static route metadata, prose, legacy source markers, test existence, and failure-capable syntax remain proof-route candidates and cannot satisfy the execution-backed closure.","fields":{"nonClaims":["Successful selected tests do not prove assertion-branch execution, mutation adequacy, line coverage, dead-code absence, exhaustive command semantics, malicious-test resistance, producer authentication, receipt freshness, merge satisfaction, or production readiness."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-010"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"medium","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} - , {"requirementId":"REQ-PROOFKIT-QUALITY-011","statementCompletion":"CI separates the full source/package gate with an explicit native Linux x86_64 Python 3.11.0 installed-wheel subcheck from the platform-specific macOS and Linux arm64 execution smoke, runs the complete processgroup package on each macOS and Linux arm64 platform lane without selector-based narrowing alongside the existing filesystem checks, executes the complete Go package set through the owner command rather than a partial package list, requires the local composed Go gate and CI source job to run non-mutating go mod tidy -diff followed by go mod verify before Go tests with either failure stopping the chain, requires exact owner-reviewed root check and go:check AND-only command chains, forbids pre/post npm lifecycle script keys by presence for check and every direct npm-run target in those chains or CI source steps without redefining unrelated npm scripts, uses exact scalar hosted runners and provider-check names, requires each platform job's presence-aware exact ordered step inventory, with distinct required macos-15 arm64, macos-15-intel x86_64 and ubuntu-24.04-arm aarch64 jobs whose first step fails unless uname -m reports the job's native architecture, and exact repository-local setup-action bytes to execute the exact fail-closed platform-smoke package-script owner command, and exposes one exact fail-closed aggregate quality gate under exact bash workflow run defaults with no inherited working-directory or environment entries; a closed inventory requires every required leaf job and step to omit conditions except for exact owner-reviewed named conditions, every required leaf job and the aggregate job omit job defaults, reusable-workflow substitution, and continue-on-error and have no job environment entries, all of their steps omit shell, working-directory, and continue-on-error and have no step environment entries, and the aggregate's reviewed expression and script require every exact required leaf check to finish with success; workflow mutation fixtures preserve nil versus empty values, field-presence bits and detached mutable data before applying negative changes; neutralizing wrappers, dynamically false or explicit-null conditions, execution overrides, dual run/uses keys, extra disjunctions, non-string runner labels, nested or top-level semantic shadow steps, no-op or reusable-job substitution, skipped, failed, or cancelled leaf jobs cannot satisfy branch-protection evidence. The unconditional minimum-Python owner step immediately follows package artifact verification in the source job and precedes receipts and closeout, and in the Linux arm64 job it immediately follows a build of the complete current release wheel set; omission or neutralization of either step cannot yield source-quality success, failure of either step cannot yield success of the job that runs it, and none of them can yield required-aggregate success; exactly five required leaf jobs remain in the aggregate.","fields":{"nonClaims":["This requirement does not claim provider-run success, branch-protection settings, Windows support, performance budget enforcement, or release approval."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-011"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"medium","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} + , {"requirementId":"REQ-PROOFKIT-QUALITY-011","statementCompletion":"CI separates the full source/package gate with an explicit native Linux x86_64 Python 3.11.0 installed-wheel subcheck from the platform-specific macOS and Linux arm64 execution smoke, runs the complete processgroup package on each macOS and Linux arm64 platform lane without selector-based narrowing alongside the existing filesystem checks, executes the complete Go package set through the owner command rather than a partial package list, requires the local composed Go gate and CI source job to run non-mutating go mod tidy -diff followed by go mod verify before Go tests with either failure stopping the chain, requires exact owner-reviewed root check and go:check AND-only command chains, forbids pre/post npm lifecycle script keys by presence for check and every direct npm-run target in those chains or CI source steps without redefining unrelated npm scripts, uses exact scalar hosted runners and provider-check names, requires each platform job's presence-aware exact ordered step inventory, with distinct required macos-15 arm64, macos-15-intel x86_64 and ubuntu-24.04-arm aarch64 jobs whose first step fails unless uname -m reports the job's native architecture, and exact repository-local setup-action bytes to execute the exact fail-closed platform-smoke package-script owner command, and exposes one exact fail-closed aggregate quality gate under exact bash workflow run defaults with no inherited working-directory or environment entries; a closed inventory requires every required leaf job and step to omit conditions except for exact owner-reviewed named conditions, every required leaf job and the aggregate job omit job defaults, reusable-workflow substitution, and continue-on-error and have no job environment entries, all of their steps omit shell, working-directory, and continue-on-error and have no step environment entries, and the aggregate's reviewed expression and script require every exact required leaf check to finish with success; workflow mutation fixtures preserve nil versus empty values, field-presence bits and detached mutable data before applying negative changes; neutralizing wrappers, dynamically false or explicit-null conditions, execution overrides, dual run/uses keys, extra disjunctions, non-string runner labels, nested or top-level semantic shadow steps, no-op or reusable-job substitution, skipped, failed, or cancelled leaf jobs cannot satisfy branch-protection evidence. The unconditional minimum-Python owner step immediately follows package artifact verification in the source job and precedes receipts and closeout, and in the Linux arm64 job it immediately follows a build of the complete current release wheel set; omission or neutralization of either step cannot yield source-quality success, failure of either step cannot yield success of the job that runs it, and none of them can yield required-aggregate success; exactly six required leaf jobs remain in the aggregate.","fields":{"nonClaims":["This requirement does not claim provider-run success, branch-protection settings, Windows support, performance budget enforcement, or release approval."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-011"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"medium","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} , {"requirementId":"REQ-PROOFKIT-QUALITY-012","statementCompletion":"Release and adoption channel identifiers are admitted by one canonical authority vocabulary that separates authorityChannel values from display labels, publisher environments, statuses, registry sub-vocabulary, and candidate evidence classes.","fields":{"nonClaims":["This requirement does not claim provider-run success, registry publication, GitHub Release creation, consumer adoption, rollout approval, or release approval."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-012"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"medium","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} , {"requirementId":"REQ-PROOFKIT-QUALITY-013","statementCompletion":"Self-hosting workflow package-gate evidence is checked by a typed workflow oracle that proves the package gate is reachable, exact, advisory, and guarded only by an absent condition or the exact owner-reviewed condition for every trust-significant job and step; the oracle checks the complete required CI and release job inventories, rejects neutralizing or dynamically false expression variants, requires any workflow run defaults to retain exact bash without a working-directory override, admits only the exact owner-reviewed workflow and step environment entries, requires the gate job to omit job defaults and continue-on-error and have no job environment entries, requires every step in that job to omit shell, working-directory, and continue-on-error, and orders the gate before evidence publication instead of relying on workflow text search. The CI source job includes the exact minimum-Python installed-wheel invocation after package artifacts and before self-hosting receipts, coverage and closeout; its execution requirement does not promote local advisory receipts into authenticated provider evidence.","fields":{"nonClaims":["This requirement does not claim full GitHub Actions interpretation, provider-run success, branch-protection enforcement, required-check enforcement, external action safety, registry publication, release approval, or rollout approval."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-013"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"medium","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} , {"requirementId":"REQ-PROOFKIT-QUALITY-014","statementCompletion":"Release authority consumers compare downstream policy against the admitted typed release-authority projection and admitted report digest instead of rereading caller-owned raw releaseAuthorityInput fields after validation.","fields":{"nonClaims":["This requirement does not claim public SDK stability, registry publication, package installation, consumer dependency policy, rollout approval, merge approval, provider provenance, proof freshness, or native witness execution."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-014"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"high","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} @@ -35,9 +35,10 @@ , {"requirementId":"REQ-PROOFKIT-QUALITY-022","statementCompletion":"Authored requirement workspace JavaScript and CSS are embedded through an explicit compile-time asset set, served only by exact routes with restrictive response headers, and strictly type-checked without a production bundler; Go lifecycle, concurrent terminal-state, HTTP security, and repository-confined non-symlink artifact falsifiers plus one machine-admitted state-matrix execution report prove the same non-empty passed test identities in pinned Chromium, Firefox, and Playwright WebKit across independently held bootstrap, specifications, diff, and graph loading states, bootstrap failure, terminal specifications, diff, graph, unavailable views, view failure, handoff result, and handoff failure, with one direct audit initializing the pinned axe distribution through the browser-context script channel, admitting only the main frame and exact engine identity, preserving same-origin and Playwright branding, and running default rules plus explicitly enabled target-size after state identity, followed by 320 CSS-pixel document reflow, labelled internal graph and table overflow, and rendered light/dark text, boundary, and focus contrast oracles, a combined negative fixture for undersized named and normal-sized unnamed controls, bounded browser-server cleanup, attempt-scoped failed-run action, DOM, network, and source traces without continuous screenshot capture plus one bounded best-effort failure screenshot retained separately from authoritative passed proof, and no added runtime package dependencies.","fields":{"nonClaims":["This requirement does not claim full WCAG 2.2 conformance, branded Safari behavior, complete screen-reader interoperability, every operating-system theme, 400-percent zoom behavior, continuous screenshot evidence, guaranteed capture of the best-effort failure screenshot, provider CI ingestion, registry publication, release approval, rollout approval, or production readiness."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-022"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"high","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} , {"requirementId":"REQ-PROOFKIT-QUALITY-023","statementCompletion":"Every npm platform-binary tar entry and every Python wheel independently binds its embedded executable bytes to the same release-platform binary, while an independently asserted complete OS/architecture-to-suffix bijection proves that every launcher target selects exactly one owned platform payload. Release metadata generation admits every final npm tarball, wheel, and SBOM exactly once into one bounded regular non-symlink immutable snapshot, derives npm pack SHA-1 and SRI claims, final carrier decoding, direct chunked binary equality, wheel and embedded-binary digests, release checksums, and SBOM-subject checksums only from that epoch, requires every npm pack and Python package-set digest claim to equal those decoded bytes, and revalidates every source identity before metadata persistence without relying on a mutable intermediate dist directory or claimed digests; release closeout independently reopens every declared repository-confined npm tarball and wheel through bounded identity-checked non-symlink handles, recomputes npm SHA-1 and SRI plus both wheel and embedded-binary digests, and rejects unsafe or duplicate archive topology before accepting package evidence; both tar and wheel decoders reject duplicate entry names, preflight bounded archive and entry cardinality plus compressed and uncompressed sizes before decompression, cap every decoded payload, and require exact release-platform closure; the wheel-set verifier independently rejects version drift, duplicate identities, missing files, and SHA drift; each wheel independently decodes Darwin Mach-O bytes so an advertised macOS platform minimum is never lower than the executable minimum, uses Core Metadata 2.4 License-Expression and License-File fields, carries an exact repository LICENSE copy under the wheel licenses directory, and closes every payload entry through RECORD.","fields":{"nonClaims":["This requirement does not claim execution on every accepted operating-system version, legal approval, registry publication, release approval, rollout approval, or production readiness."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-023"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"critical","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} , {"requirementId":"REQ-PROOFKIT-QUALITY-024","statementCompletion":"Release metadata generation admits one closed schema-versioned machine-readable declaration of the reviewed public-contract change set, requires exact ordered equality for the complete current breaking-change, addition, and migration inventories, including channel-specific changes to public generated continuation bytes, binds its exact previous and current canonical SemVer values to a compatible or breaking change class, rejects missing, substituted, reordered, or surplus current entries plus non-monotonic or patch-range breaking releases, and renders one byte-exact independently authored complete current release-note projection with no relocated, duplicate, appended, surplus, or second owned section; release candidate preflight binds the admitted npm package identity and candidate state to registry latest so an unpublished candidate requires latest to equal previousVersion and an exact existing-byte-match candidate requires latest to equal version; release metadata promotes the npm channel to published only from a strictly admitted registry-authority record whose unique filename-keyed package set exactly matches both retained registry records and local candidate identity and bytes; one repository-owned retained-evidence builder and verifier checksum the exact final downloadable artifact topology with artifact-relative paths, reject unbound evidence files and symlink substitution, and fail release closeout on record, note, path, or digest drift. After failed npm view, both release legs admit absence only from a strict JSON error report bounded to 64 KiB with exact string error.code E404, a closed error envelope, and optional opaque summary/detail fields; all other reports fail closed without emitting caller diagnostics or arbitrary keys.","fields":{"nonClaims":["This requirement does not infer change-record completeness from source changes, reconstruct release-history migration data before the declared support baseline, claim a cumulative release-history planner, make release notes approval authority, or prove provider publication, attestation authenticity, consumer adoption, rollout approval, or production readiness."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-024"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"high","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} - , {"requirementId":"REQ-PROOFKIT-QUALITY-025","statementCompletion":"Every tracked GitHub Actions workflow is admitted by one exact sorted inventory and a closed raw-YAML key grammar at workflow, job, and step boundaries before typed decoding, with only exact owner-reviewed release deployment environments; every external action reference is pinned to a full lowercase 40-hex commit identity; local action references remain repository-confined; status expressions and the required CI aggregate match exact owner-reviewed forms; and an already-existing GitHub Release is a read-only object whose metadata, exact asset set, and asset bytes must match the candidate before the workflow exits successfully.","fields":{"nonClaims":["This requirement does not claim external action safety, tag equivalence, provider execution, branch-protection configuration, release publication, or provider-side asset immutability."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-025"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"critical","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} + , {"requirementId":"REQ-PROOFKIT-QUALITY-025","statementCompletion":"Every tracked GitHub Actions workflow is admitted by one exact sorted inventory and a closed raw-YAML key grammar at workflow, job, and step boundaries before typed decoding, with only exact owner-reviewed release deployment environments and exactly one job container, admitted only for the ci workflow's browser-visual job with the exact owner-reviewed image reference and the exact options `--user 1001 --ipc=host --init`; every workflow container image reference is a name:tag reference pinned by a lowercase sha256 digest; every external action reference is pinned to a full lowercase 40-hex commit identity; local action references remain repository-confined; status expressions and the required CI aggregate match exact owner-reviewed forms; and an already-existing GitHub Release is a read-only object whose metadata, exact asset set, and asset bytes must match the candidate before the workflow exits successfully.","fields":{"nonClaims":["This requirement does not claim external action safety, tag equivalence, provider execution, branch-protection configuration, release publication, or provider-side asset immutability."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-025"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"critical","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} , {"requirementId":"REQ-PROOFKIT-QUALITY-026","statementCompletion":"Every repository-owned stable JSON implementation used by a public or retained-evidence path accepts only Unicode 17.0.0 scalar strings, rejecting malformed UTF-8 byte strings and unpaired UTF-16 surrogates as applicable, emits the same deterministic semantic-preserving JSON escapes for the pinned Cc, Cf, Zl, and Zp range table, preserves ordinary decoded values, and remains byte-compatible with one versioned owner-authored literal range corpus whose Go projection is exhaustively checked against the Unicode 17.0.0 category oracle and whose JavaScript and generated TypeScript projections are exhaustively checked over the scalar domain.","fields":{"nonClaims":["This bounded encoding policy does not prove terminal-rendering equivalence or safety of arbitrary downstream decoders."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-026"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"high","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} , {"requirementId":"REQ-PROOFKIT-QUALITY-027","statementCompletion":"Every report-visible diagnostic and structural-text sanitizer accepts only Unicode 17.0.0 scalar strings, recognizes one explicit Unicode whitespace alphabet plus the pinned Cc, Cf, Zl, and Zp range table and contiguous or control-split secret-shaped values through one shared taxonomy, replaces any rejected caller-owned value as a whole with one fixed command-owned label without emitting its bytes, code points, prefix, suffix, or caller-controlled replacement text, and bounds each admitted diagnostic projection to at most 512 caller-derived Unicode scalars followed, only when truncated, by the fixed 25-scalar command-owned marker `...`, for a total of at most 537 Unicode scalars; every Go and JavaScript repository tool entrypoint routes dynamic errors through that owner, failed child-process stderr that enters this diagnostic boundary is captured under a fixed byte limit before whole-value sanitization while successful child stderr is not projected, direct child stderr wiring is rejected, and the installed-carrier smoke adapter bounds stdout, stderr, time, and cleanup; on supported Unix hosts, a successful installed-carrier result requires confirmed process-group absence after the command parent is reaped, while cleanup timeout or probe failure is an explicit failed operation rather than a successful terminal result. Shared unsupported-key admission preserves its context argument and emits only the unknown-key count; for fixed context and count its diagnostic is independent of unknown names and values, without changing exact key membership or caller inputs.","fields":{"nonClaims":["This bounded diagnostic taxonomy does not prove confidentiality of values outside its admitted secret and control classes or claim that every repository-internal tool invocation uses the installed-carrier timeout policy."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-027"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"high","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} + , {"requirementId":"REQ-PROOFKIT-QUALITY-028","statementCompletion":"Rendered requirement views are compared with owner-admitted reference digests in one recorded reference environment. tests/visual/environment.json records the digest-pinned Playwright container image (name, tag, index digest, platform digests and platform linux/arm64) and its observed tuple: the OS identity, the machine architecture, the image marker, the installed fontconfig, FreeType, HarfBuzz and font package versions, the SHA-256 of the committed tests/visual/fonts.conf, which prefers Liberation Sans for sans-serif and system-ui, Liberation Serif for serif and Liberation Mono for monospace and ui-monospace and which the gate copies into its run directory and sets as FONTCONFIG_FILE before observing, so that every browser reads the hashed bytes, the fc-match results for monospace, sans-serif, serif, system-ui and ui-monospace, each queried by its whole family name with every hyphen escaped, the installed engine revisions, no emulation, and the launched browser versions; its image tag equals the locked @playwright/test version, its engine revisions and browser versions equal playwright-core browsers.json, and the ci workflow's browser-visual container reference equals its name, tag and index digest. tests/visual/references.json binds environmentId, the SHA-256 of the canonical environment record, and, for each reference ID /.., the width, the height and rgbaSHA256, the SHA-256 of the canonical pixel encoding: the 14 ASCII bytes PROOFKIT-RGBA8, the width and the height as big-endian unsigned 32-bit integers, then the 8-bit non-premultiplied RGBA rows from top to bottom with no padding. The IDs are exactly the closed set of fourteen views (five static renderings, source, spec-tree, structured proof, compact proof and coverage, in Cards and Table modes, and four workspace tabs, Specifications, Coverage, Diff and Traceability), three engines, light and dark color schemes and widths of 320 and 1440 CSS pixels, plus one Chromium light 1440x960 README scene served from a committed synthetic project through the project view path, with no missing, extra or duplicate ID; every digest is 64 lowercase hexadecimal digits, a view is exactly its width wide and 1 to 2400 pixels tall, and the README scene is exactly 1440x960. No reference image is stored in the repository. `npm run browser:visual`, which `npm run check` does not run, copies fonts.conf into its run directory and sets FONTCONFIG_FILE to the copy, which every browser inherits because no project sets its own browser environment, observes the environment and writes it with the CPU vendor, model and features to its run directory before any browser launches; when a compared field differs it records the state unverified with the divergent fields and launches no browser, and `--require-reference-environment` makes that state a failure. Otherwise it renders every view, applies the color scheme and reduced-motion emulation after the last navigation of the view and asserts both in effect through the matching media queries before capture, captures it twice as the full page cut at 2400 CSS pixels in a window fitted to the document height, requires both PNG captures to be byte-equal, decodes the capture with node:zlib and requires its dimensions and canonical digest to equal the admitted entry, a fixed threshold of zero differing pixels; a mismatch names the ID and both digests and sizes. Every reference-environment run, passing or failing, keeps its captures, a candidate inventory and the observed environment below its run directory, never writes into the test directory, and fails when the materialized test directory gained, lost or changed a file; it records passed only when every engine passes exactly its expected comparisons plus six controls and one font-face check with zero retries. Each control serves a copied workspace stylesheet: an unchanged copy must keep the admitted digest, and five mutated copies must change it: a one-unit text color change, a one-pixel shift, a font-weight change from 400 to 700, a change of the sans-serif family to serif and a change of the monospace family to sans-serif. The font-face check reads the computed font-family of the workspace and static roots and of the workspace and static monospace elements from their rendered views, draws each beside the named Liberation faces in regular and bold weight, and requires exact pixel equality with Liberation Sans for the roots and with Liberation Mono for the monospace stacks, and equal advances of i and M for exactly the monospace stacks. Beyond the runtime harness projects, the visual Firefox project adds only the preference browser.chrome.site_icons=false and the visual Chromium project adds only the argument --disable-partial-raster, which makes repeated launches rasterize identically. Any failure, including a missing, malformed or extra inventory entry or an inventory whose environmentId differs from the recorded environment, exits non-zero and writes no record, so any change of the environment requires re-admitted references. Reference admission is a separate owner action, `browserproofverify --admit-references --from `, which the gate never calls: it admits only a run whose recorded observed environment equals the recorded environment and which holds exactly one regular capture file per ID and no unstable directory, recomputes every digest from the PNG captures, refuses a run whose candidate inventory disagrees with them, and only then writes references.json.","fields":{"nonClaims":["This requirement does not claim pixel identity outside the recorded reference environment or on CPU models other than the calibrated Apple M1 (implementer 0x61) and Arm Neoverse N2 (implementer 0x41, part 0xd49), coverage of content below the first 2400 CSS pixels of a view, that weights 400 and 500, or weights 600 to 900, render differently (only regular and bold faces are installed, so each group renders identically), that a family name unknown to fontconfig, other than ui-monospace, falls through to the next family of its stack in WebKit (a name WebKit looks up resolves to Liberation Sans), that an admitted run directory is unmodified gate output (only a passing gate on the admitting tree proves the admitted digests), that the release candidate or the package gate reruns the visual gate, who reviewed or admitted a reference change, a human usability study, or WCAG conformance certification. Admitted images are not stored in the repository; they are reproducible from the admitting commit in the recorded environment on a calibrated CPU model while the pinned image and the pinned toolchain and dependency downloads remain available, and the uploaded gallery is derived, retention-limited evidence."],"externalNonClaimRefs":["NC-PROOFKIT-QUALITY-028"],"proofBindingRefs":["proofkit/requirement-bindings.json"],"ownerId":"proofkit.supply-chain-quality","claimLevel":"blocking","riskClass":"high","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"proofkit.supply-chain-quality","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} ] } ] diff --git a/internal/testsupport/browserfixture/readme.go b/internal/testsupport/browserfixture/readme.go new file mode 100644 index 00000000..d4a121c0 --- /dev/null +++ b/internal/testsupport/browserfixture/readme.go @@ -0,0 +1,54 @@ +package browserfixture + +import ( + "embed" + "fmt" + "io/fs" + "os" + "path/filepath" +) + +// readmeProject is a committed, materialized synthetic project: four delivery +// requirements of example.dispatch with their bindings, test inventory and +// routing manifest. The README scene serves it through the project view path. +// +//go:embed testdata/readme-project +var readmeProject embed.FS + +const readmeProjectRoot = "testdata/readme-project" + +// MaterializeReadmeProject writes the committed project files into an empty +// directory so the project view path inspects real repository files. +func MaterializeReadmeProject(directory string) error { + return fs.WalkDir(readmeProject, readmeProjectRoot, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + relative, err := filepath.Rel(readmeProjectRoot, filepath.FromSlash(path)) + if err != nil { + return err + } + target := filepath.Join(directory, relative) + if entry.IsDir() { + return os.MkdirAll(target, 0o755) + } + if !entry.Type().IsRegular() { + return fmt.Errorf("README project fixture must contain only regular files") + } + content, err := readmeProject.ReadFile(path) + if err != nil { + return err + } + return os.WriteFile(target, content, 0o644) + }) +} + +// StaticCompactProof exercises the compact proof contract through the static +// proof renderer, alongside the structured binding of StaticProof. +func StaticCompactProof() (map[string]any, error) { + input, err := CoverageInput("compact") + if err != nil { + return nil, err + } + return input["compactProofContract"].(map[string]any), nil +} diff --git a/internal/testsupport/browserfixture/readme_test.go b/internal/testsupport/browserfixture/readme_test.go new file mode 100644 index 00000000..95ab1c3a --- /dev/null +++ b/internal/testsupport/browserfixture/readme_test.go @@ -0,0 +1,23 @@ +package browserfixture + +import ( + "testing" + + "github.com/research-engineering/agentic-proofkit/internal/command/requirementbrowser" +) + +// The committed README project is a complete admitted project for the same +// project path the view command uses. +func TestReadmeProjectMaterializesAnAdmittedViewProject(t *testing.T) { + directory := t.TempDir() + if err := MaterializeReadmeProject(directory); err != nil { + t.Fatal(err) + } + plan, exit, err := requirementbrowser.BuildProjectPlan(t.Context(), directory, requirementbrowser.Options{}) + if err != nil || exit != 0 { + t.Fatalf("README project plan exit=%d err=%v", exit, err) + } + if plan["view"] != "workspace" || plan["renderedViewKind"] != "proofkit.requirement-workspace" || plan["renderedAuthority"] != "presentation_adapter" { + t.Fatalf("README project plan lost its project workspace: view=%v kind=%v authority=%v", plan["view"], plan["renderedViewKind"], plan["renderedAuthority"]) + } +} diff --git a/internal/testsupport/browserfixture/testdata/readme-project/docs/specs/dispatch/requirements.v2.json b/internal/testsupport/browserfixture/testdata/readme-project/docs/specs/dispatch/requirements.v2.json new file mode 100644 index 00000000..2f251d6e --- /dev/null +++ b/internal/testsupport/browserfixture/testdata/readme-project/docs/specs/dispatch/requirements.v2.json @@ -0,0 +1,21 @@ +{ + "schemaVersion": 2, + "kind": "proofkit.requirement-source", + "sourceId": "example.dispatch", + "specPackagePath": "docs/specs/dispatch", + "sourceNonClaims": ["This synthetic delivery source does not describe a real service."], + "groups": [ + { + "groupId": "RGRP-DISPATCH", + "profileId": "", + "statementStem": "", + "sharedPremises": [], + "members": [ + {"requirementId":"REQ-DISPATCH-001","statementCompletion":"A delivery request is accepted only when its destination and idempotency key are present.","fields":{"nonClaims":["This synthetic requirement does not establish execution evidence."],"externalNonClaimRefs":[],"proofBindingRefs":["proofkit/bindings.json"],"ownerId":"example.dispatch","claimLevel":"blocking","riskClass":"high","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"example.dispatch","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} + , {"requirementId":"REQ-DISPATCH-002","statementCompletion":"Repeating the same idempotency key returns the original delivery identifier without scheduling another delivery.","fields":{"nonClaims":["This synthetic requirement does not establish execution evidence."],"externalNonClaimRefs":[],"proofBindingRefs":["proofkit/bindings.json"],"ownerId":"example.dispatch","claimLevel":"blocking","riskClass":"high","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"example.dispatch","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} + , {"requirementId":"REQ-DISPATCH-003","statementCompletion":"A temporary provider failure may be retried at most three times; the final failure remains visible to the caller.","fields":{"nonClaims":["This synthetic requirement does not establish execution evidence."],"externalNonClaimRefs":[],"proofBindingRefs":["proofkit/bindings.json"],"ownerId":"example.dispatch","claimLevel":"blocking","riskClass":"high","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"example.dispatch","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} + , {"requirementId":"REQ-DISPATCH-004","statementCompletion":"Delivery diagnostics omit credentials and the recipient address.","fields":{"nonClaims":["This synthetic requirement does not establish execution evidence."],"externalNonClaimRefs":[],"proofBindingRefs":["proofkit/bindings.json"],"ownerId":"example.dispatch","claimLevel":"blocking","riskClass":"high","nonClaimRefs":[],"lifecycle":{"state":"active"},"deferral":null,"updatePolicy":{"reviewOwnerId":"example.dispatch","requiresImpactDeclaration":true,"requiresProofBindingReview":true}}} + ] + } + ] +} diff --git a/internal/testsupport/browserfixture/testdata/readme-project/proofkit/bindings.json b/internal/testsupport/browserfixture/testdata/readme-project/proofkit/bindings.json new file mode 100644 index 00000000..14fa8450 --- /dev/null +++ b/internal/testsupport/browserfixture/testdata/readme-project/proofkit/bindings.json @@ -0,0 +1,141 @@ +{ + "bindingId": "example.dispatch", + "bindings": [ + { + "commandIds": [ + "dispatch.check" + ], + "environmentClasses": [ + "local-go" + ], + "requirementId": "REQ-DISPATCH-001", + "scenarioId": "dispatch.scenario.001", + "witnessId": "dispatch.witness.001", + "witnessKind": "contract", + "witnessPath": "tests/dispatch_test.go", + "witnessSelectors": [ + { + "command": "go test ./tests -run TestDispatch001", + "selector": "TestDispatch001" + } + ] + }, + { + "commandIds": [ + "dispatch.check" + ], + "environmentClasses": [ + "local-go" + ], + "requirementId": "REQ-DISPATCH-002", + "scenarioId": "dispatch.scenario.002", + "witnessId": "dispatch.witness.002", + "witnessKind": "contract", + "witnessPath": "tests/dispatch_test.go", + "witnessSelectors": [ + { + "command": "go test ./tests -run TestDispatch002", + "selector": "TestDispatch002" + } + ] + }, + { + "commandIds": [ + "dispatch.check" + ], + "environmentClasses": [ + "local-go" + ], + "requirementId": "REQ-DISPATCH-003", + "scenarioId": "dispatch.scenario.003", + "witnessId": "dispatch.witness.003", + "witnessKind": "contract", + "witnessPath": "tests/dispatch_test.go", + "witnessSelectors": [ + { + "command": "go test ./tests -run TestDispatch003", + "selector": "TestDispatch003" + } + ] + }, + { + "commandIds": [ + "dispatch.check" + ], + "environmentClasses": [ + "local-go" + ], + "requirementId": "REQ-DISPATCH-004", + "scenarioId": "dispatch.scenario.004", + "witnessId": "dispatch.witness.004", + "witnessKind": "contract", + "witnessPath": "tests/dispatch_test.go", + "witnessSelectors": [ + { + "command": "go test ./tests -run TestDispatch004", + "selector": "TestDispatch004" + } + ] + } + ], + "nonClaims": [ + "Synthetic delivery bindings do not execute witnesses." + ], + "requirements": [ + { + "claimLevel": "blocking", + "nonClaims": [ + "This synthetic requirement does not establish execution evidence." + ], + "ownerId": "example.dispatch", + "proofState": "witness_backed", + "requirementId": "REQ-DISPATCH-001", + "specPath": "docs/specs/dispatch/requirements.v2.json" + }, + { + "claimLevel": "blocking", + "nonClaims": [ + "This synthetic requirement does not establish execution evidence." + ], + "ownerId": "example.dispatch", + "proofState": "witness_backed", + "requirementId": "REQ-DISPATCH-002", + "specPath": "docs/specs/dispatch/requirements.v2.json" + }, + { + "claimLevel": "blocking", + "nonClaims": [ + "This synthetic requirement does not establish execution evidence." + ], + "ownerId": "example.dispatch", + "proofState": "witness_backed", + "requirementId": "REQ-DISPATCH-003", + "specPath": "docs/specs/dispatch/requirements.v2.json" + }, + { + "claimLevel": "blocking", + "nonClaims": [ + "This synthetic requirement does not establish execution evidence." + ], + "ownerId": "example.dispatch", + "proofState": "witness_backed", + "requirementId": "REQ-DISPATCH-004", + "specPath": "docs/specs/dispatch/requirements.v2.json" + } + ], + "schemaVersion": 1, + "selection": { + "changedPaths": [], + "ownerIds": [], + "requirementIds": [] + }, + "witnessCommands": [ + { + "command": "go test ./tests", + "commandId": "dispatch.check", + "environmentClasses": [ + "local-go" + ] + } + ] +} diff --git a/internal/testsupport/browserfixture/testdata/readme-project/proofkit/project.v1.json b/internal/testsupport/browserfixture/testdata/readme-project/proofkit/project.v1.json new file mode 100644 index 00000000..a623855e --- /dev/null +++ b/internal/testsupport/browserfixture/testdata/readme-project/proofkit/project.v1.json @@ -0,0 +1,29 @@ +{ + "authority": "routing_only", + "manifestId": "sha256:7f176c3093004fb75b7c50da2833fb062481d314774482b86107ce54cc7eee36", + "manifestKind": "proofkit.project-routing-manifest", + "materializationRequestId": "dispatch.request", + "nonClaims": [ + "Project routing manifests do not duplicate child semantics or prove child admission, freshness, execution, merge, release, rollout, or production readiness." + ], + "projectId": "example.dispatch", + "routes": [ + { + "artifactId": "sha256:fc42c57de63f3520b567f5b1f1a87acff666adae7e6b49f9f21b53ece823f738", + "artifactKind": "requirement_source", + "path": "docs/specs/dispatch/requirements.v2.json" + }, + { + "artifactId": "sha256:5af29397368c993818b0f7520b1be553e2c2c122c06a2e00708f32aa52ea40f8", + "artifactKind": "requirement_proof_binding", + "path": "proofkit/bindings.json" + }, + { + "artifactId": "sha256:9e2ae062461d1e919413de9feb9d7c433f9e73ad072bf7c0eb77099e7a1f1a93", + "artifactKind": "test_evidence_inventory", + "path": "proofkit/tests.json" + } + ], + "schemaVersion": 1, + "sourcePlanId": "sha256:8ef780506e8b5c91fed0d0e76e78061be04053e67c53c9419d4b27d60f31ca50" +} diff --git a/internal/testsupport/browserfixture/testdata/readme-project/proofkit/tests.json b/internal/testsupport/browserfixture/testdata/readme-project/proofkit/tests.json new file mode 100644 index 00000000..cca4b68e --- /dev/null +++ b/internal/testsupport/browserfixture/testdata/readme-project/proofkit/tests.json @@ -0,0 +1,146 @@ +{ + "authority": "caller_owned_inventory", + "entries": [ + { + "commandRefs": [ + "dispatch.check" + ], + "evidenceClass": "declared_semantic_falsifier_route", + "falsifier": { + "dominanceGroup": "dispatch.group.001", + "falsifierId": "dispatch.falsifier.001", + "negativeCaseId": "dispatch.case.001", + "supersedes": [], + "wrongImplementationClassId": "dispatch.wrong.001" + }, + "nonClaims": [], + "oracle": { + "assertionSummary": "The negative case is rejected.", + "expectedPublicOutcome": "a contradicting delivery is rejected", + "oracleId": "dispatch.oracle.001", + "oracleKind": "negative_exit_and_diagnostic" + }, + "ownerId": "example.dispatch", + "ownerInvariantRefs": [], + "qualityFindings": [], + "requirementRefs": [ + "REQ-DISPATCH-001" + ], + "selector": "go test ./tests -run TestDispatch001", + "sourcePath": "tests/dispatch_test.go", + "testId": "dispatch.test.001", + "witnessRefs": [ + "dispatch.witness.001" + ] + }, + { + "commandRefs": [ + "dispatch.check" + ], + "evidenceClass": "declared_semantic_falsifier_route", + "falsifier": { + "dominanceGroup": "dispatch.group.002", + "falsifierId": "dispatch.falsifier.002", + "negativeCaseId": "dispatch.case.002", + "supersedes": [], + "wrongImplementationClassId": "dispatch.wrong.002" + }, + "nonClaims": [], + "oracle": { + "assertionSummary": "The negative case is rejected.", + "expectedPublicOutcome": "a contradicting delivery is rejected", + "oracleId": "dispatch.oracle.002", + "oracleKind": "negative_exit_and_diagnostic" + }, + "ownerId": "example.dispatch", + "ownerInvariantRefs": [], + "qualityFindings": [], + "requirementRefs": [ + "REQ-DISPATCH-002" + ], + "selector": "go test ./tests -run TestDispatch002", + "sourcePath": "tests/dispatch_test.go", + "testId": "dispatch.test.002", + "witnessRefs": [ + "dispatch.witness.002" + ] + }, + { + "commandRefs": [ + "dispatch.check" + ], + "evidenceClass": "declared_semantic_falsifier_route", + "falsifier": { + "dominanceGroup": "dispatch.group.003", + "falsifierId": "dispatch.falsifier.003", + "negativeCaseId": "dispatch.case.003", + "supersedes": [], + "wrongImplementationClassId": "dispatch.wrong.003" + }, + "nonClaims": [], + "oracle": { + "assertionSummary": "The negative case is rejected.", + "expectedPublicOutcome": "a contradicting delivery is rejected", + "oracleId": "dispatch.oracle.003", + "oracleKind": "negative_exit_and_diagnostic" + }, + "ownerId": "example.dispatch", + "ownerInvariantRefs": [], + "qualityFindings": [], + "requirementRefs": [ + "REQ-DISPATCH-003" + ], + "selector": "go test ./tests -run TestDispatch003", + "sourcePath": "tests/dispatch_test.go", + "testId": "dispatch.test.003", + "witnessRefs": [ + "dispatch.witness.003" + ] + }, + { + "commandRefs": [ + "dispatch.check" + ], + "evidenceClass": "declared_semantic_falsifier_route", + "falsifier": { + "dominanceGroup": "dispatch.group.004", + "falsifierId": "dispatch.falsifier.004", + "negativeCaseId": "dispatch.case.004", + "supersedes": [], + "wrongImplementationClassId": "dispatch.wrong.004" + }, + "nonClaims": [], + "oracle": { + "assertionSummary": "The negative case is rejected.", + "expectedPublicOutcome": "a contradicting delivery is rejected", + "oracleId": "dispatch.oracle.004", + "oracleKind": "negative_exit_and_diagnostic" + }, + "ownerId": "example.dispatch", + "ownerInvariantRefs": [], + "qualityFindings": [], + "requirementRefs": [ + "REQ-DISPATCH-004" + ], + "selector": "go test ./tests -run TestDispatch004", + "sourcePath": "tests/dispatch_test.go", + "testId": "dispatch.test.004", + "witnessRefs": [ + "dispatch.witness.004" + ] + } + ], + "inventoryId": "example.dispatch", + "nonClaims": [ + "Synthetic delivery inventory does not establish execution.", + "Test evidence inventory reports do not approve merge, release, rollout, or repository policy.", + "Test evidence inventory reports do not authenticate runner output or receipt producers.", + "Test evidence inventory reports do not execute native tests.", + "Test evidence inventory reports do not prove repository inventory completeness.", + "Test evidence inventory reports do not resolve selectors or review caller-authored oracle quality.", + "Test evidence inventory reports do not verify that a caller-declared falsifier supersession dominates the superseded falsifier." + ], + "ownerId": "example.dispatch", + "schemaVersion": 1, + "sourceId": "dispatch.inventory" +} diff --git a/internal/tools/browserproofverify/artifact_paths.go b/internal/tools/browserproofverify/artifact_paths.go index a7e09701..c223aabe 100644 --- a/internal/tools/browserproofverify/artifact_paths.go +++ b/internal/tools/browserproofverify/artifact_paths.go @@ -17,7 +17,6 @@ import ( const ( browserArtifactsDirectory = "artifacts" browserProofDirectory = "artifacts/proofkit" - browserProofCandidateName = "browser-runtime-proof.candidate.json" browserRunDirectoryEnvironment = "PROOFKIT_BROWSER_RUN_DIRECTORY" browserProofCandidateEnvironment = "PROOFKIT_BROWSER_PROOF_CANDIDATE_PATH" ) @@ -27,7 +26,7 @@ type browserProofRunPaths struct { RunDirectory string } -func prepareBrowserProofRun(root string) (browserProofRunPaths, error) { +func prepareBrowserProofRun(root string, kind proofKind) (browserProofRunPaths, error) { rootFS, err := os.OpenRoot(root) if err != nil { return browserProofRunPaths{}, fmt.Errorf("open browser proof repository root: %w", err) @@ -39,7 +38,7 @@ func prepareBrowserProofRun(root string) (browserProofRunPaths, error) { if err := ensureRootedDirectory(rootFS, browserProofDirectory, 0o755); err != nil { return browserProofRunPaths{}, err } - if err := admitRootedDestination(rootFS, proofPath); err != nil { + if err := admitRootedDestination(rootFS, filepath.FromSlash(kind.RecordPath)); err != nil { return browserProofRunPaths{}, err } for attempt := 0; attempt < 16; attempt++ { @@ -54,7 +53,7 @@ func prepareBrowserProofRun(root string) (browserProofRunPaths, error) { return browserProofRunPaths{}, fmt.Errorf("create confined browser proof run directory: %w", err) } return browserProofRunPaths{ - CandidatePath: filepath.ToSlash(filepath.Join(runDirectory, browserProofCandidateName)), + CandidatePath: filepath.ToSlash(filepath.Join(runDirectory, kind.CandidateName)), RunDirectory: filepath.ToSlash(runDirectory), }, nil } diff --git a/internal/tools/browserproofverify/artifact_paths_test.go b/internal/tools/browserproofverify/artifact_paths_test.go index b1dd97c2..1b7bb7b8 100644 --- a/internal/tools/browserproofverify/artifact_paths_test.go +++ b/internal/tools/browserproofverify/artifact_paths_test.go @@ -10,11 +10,11 @@ import ( func TestPrepareBrowserProofRunCreatesAConfinedDisposableDirectory(t *testing.T) { root := t.TempDir() - paths, err := prepareBrowserProofRun(root) + paths, err := prepareBrowserProofRun(root, runtimeProofKind()) if err != nil { t.Fatal(err) } - if !strings.HasPrefix(paths.RunDirectory, "artifacts/browser-run-") || paths.CandidatePath != paths.RunDirectory+"/"+browserProofCandidateName { + if !strings.HasPrefix(paths.RunDirectory, "artifacts/browser-run-") || paths.CandidatePath != paths.RunDirectory+"/"+runtimeProofKind().CandidateName { t.Fatalf("unexpected confined run paths: %#v", paths) } info, err := os.Lstat(filepath.Join(root, filepath.FromSlash(paths.RunDirectory))) @@ -31,7 +31,7 @@ func TestPrepareBrowserProofRunCreatesAConfinedDisposableDirectory(t *testing.T) func TestFinalizeBrowserProofRunCleansSuccessAndRetainsFailureDiagnostics(t *testing.T) { root := t.TempDir() - successPaths, err := prepareBrowserProofRun(root) + successPaths, err := prepareBrowserProofRun(root, runtimeProofKind()) if err != nil { t.Fatal(err) } @@ -42,7 +42,7 @@ func TestFinalizeBrowserProofRunCleansSuccessAndRetainsFailureDiagnostics(t *tes t.Fatalf("successful run directory survived cleanup: %v", err) } - failurePaths, err := prepareBrowserProofRun(root) + failurePaths, err := prepareBrowserProofRun(root, runtimeProofKind()) if err != nil { t.Fatal(err) } @@ -69,7 +69,7 @@ func TestPrepareBrowserProofRunRejectsArtifactRootSymlink(t *testing.T) { if err := os.Symlink(external, filepath.Join(root, browserArtifactsDirectory)); err != nil { t.Skipf("symlink unavailable: %v", err) } - if _, err := prepareBrowserProofRun(root); err == nil { + if _, err := prepareBrowserProofRun(root, runtimeProofKind()); err == nil { t.Fatal("prepareBrowserProofRun accepted a symlinked artifact root") } content, err := os.ReadFile(sentinel) @@ -91,10 +91,10 @@ func TestWriteRootedJSONRejectsSymlinkDestination(t *testing.T) { if err := os.MkdirAll(filepath.Join(root, browserProofDirectory), 0o755); err != nil { t.Fatal(err) } - if err := os.Symlink(external, filepath.Join(root, filepath.FromSlash(proofPath))); err != nil { + if err := os.Symlink(external, filepath.Join(root, filepath.FromSlash(runtimeProofKind().RecordPath))); err != nil { t.Skipf("symlink unavailable: %v", err) } - if err := writeRootedJSON(root, proofPath, map[string]any{"state": "passed"}); err == nil { + if err := writeRootedJSON(root, runtimeProofKind().RecordPath, map[string]any{"state": "passed"}); err == nil { t.Fatal("writeRootedJSON accepted a symlink destination") } content, err := os.ReadFile(external) @@ -105,10 +105,10 @@ func TestWriteRootedJSONRejectsSymlinkDestination(t *testing.T) { func TestWriteRootedJSONPreservesExactPrettyEncodingAndMode(t *testing.T) { root := t.TempDir() - if err := writeRootedJSON(root, proofPath, map[string]any{"state": "passed"}); err != nil { + if err := writeRootedJSON(root, runtimeProofKind().RecordPath, map[string]any{"state": "passed"}); err != nil { t.Fatal(err) } - path := filepath.Join(root, filepath.FromSlash(proofPath)) + path := filepath.Join(root, filepath.FromSlash(runtimeProofKind().RecordPath)) content, err := os.ReadFile(path) if err != nil || string(content) != "{\n \"state\": \"passed\"\n}\n" { t.Fatalf("unexpected pretty bytes: %q error=%v", content, err) @@ -120,7 +120,7 @@ func TestWriteRootedJSONPreservesExactPrettyEncodingAndMode(t *testing.T) { if info.Mode().Perm() != 0o644 { t.Fatalf("mode=%v, want 0644", info.Mode()) } - value, err := readRootedJSON(root, proofPath, 1024) + value, err := readRootedJSON(root, runtimeProofKind().RecordPath, 1024) if err != nil { t.Fatal(err) } diff --git a/internal/tools/browserproofverify/input_manifest.go b/internal/tools/browserproofverify/input_manifest.go index b7679a73..21c14afe 100644 --- a/internal/tools/browserproofverify/input_manifest.go +++ b/internal/tools/browserproofverify/input_manifest.go @@ -11,10 +11,7 @@ import ( "github.com/research-engineering/agentic-proofkit/internal/kernel/admit" ) -const ( - proofInputManifestPath = "scripts/browser-runtime-proof-inputs.v1.json" - proofVerifierTarget = "./internal/tools/browserproofverify" -) +const proofVerifierTarget = "./internal/tools/browserproofverify" type proofInputManifest struct { ServerTarget string @@ -23,8 +20,8 @@ type proofInputManifest struct { Paths []string } -func loadProofInputManifest(root string) (proofInputManifest, error) { - raw, err := os.ReadFile(filepath.Join(root, proofInputManifestPath)) +func loadProofInputManifest(root string, kind proofKind) (proofInputManifest, error) { + raw, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(kind.ManifestPath))) if err != nil { return proofInputManifest{}, err } @@ -39,7 +36,7 @@ func loadProofInputManifest(root string) (proofInputManifest, error) { if err := admit.KnownKeys(record, []string{"paths", "proofKind", "schemaVersion", "serverTarget", "testRoot", "writerPath"}, "browser proof input manifest"); err != nil { return proofInputManifest{}, err } - if !admit.JSONNumberEquals(record["schemaVersion"], 1) || record["proofKind"] != "proofkit.browser-runtime-proof-inputs" { + if !admit.JSONNumberEquals(record["schemaVersion"], 1) || record["proofKind"] != kind.ManifestKind { return proofInputManifest{}, fmt.Errorf("browser proof input manifest identity is invalid") } serverTarget, err := admitManifestGoTarget(record["serverTarget"]) @@ -58,7 +55,7 @@ func loadProofInputManifest(root string) (proofInputManifest, error) { if err != nil { return proofInputManifest{}, err } - for _, ownedPath := range []string{proofInputManifestPath, testRoot, writerPath} { + for _, ownedPath := range []string{kind.ManifestPath, testRoot, writerPath} { for _, path := range paths { if path == ownedPath || strings.HasPrefix(ownedPath, path+"/") || strings.HasPrefix(path, ownedPath+"/") { return proofInputManifest{}, fmt.Errorf("browser proof input manifest paths must not duplicate role-owned paths") diff --git a/internal/tools/browserproofverify/input_resolution.go b/internal/tools/browserproofverify/input_resolution.go index 1d496f49..9cbc6961 100644 --- a/internal/tools/browserproofverify/input_resolution.go +++ b/internal/tools/browserproofverify/input_resolution.go @@ -42,7 +42,7 @@ type goListPackage struct { EmbedFiles []string } -func resolveProofInputs(root string, manifest proofInputManifest) (proofInputResolution, error) { +func resolveProofInputs(root string, manifest proofInputManifest, kind proofKind) (proofInputResolution, error) { goDirs, goFiles, err := repoLocalGoInputs(root, []string{proofVerifierTarget, manifest.ServerTarget}) if err != nil { return proofInputResolution{}, err @@ -50,9 +50,9 @@ func resolveProofInputs(root string, manifest proofInputManifest) (proofInputRes if err := validateRegularInputPaths(root, goFiles); err != nil { return proofInputResolution{}, err } - roleOwnedPaths := []string{proofInputManifestPath, manifest.TestRoot, manifest.WriterPath} + roleOwnedPaths := []string{kind.ManifestPath, manifest.TestRoot, manifest.WriterPath} selectors := sortedUnique(append(roleOwnedPaths, append(manifest.Paths, goDirs...)...)) - if err := validateBrowserWitnessPolicy(root, selectors); err != nil { + if err := validateBrowserWitnessPolicy(root, kind.CommandID, selectors); err != nil { return proofInputResolution{}, err } explicitPaths, err := collectInputPaths(root, append(roleOwnedPaths, manifest.Paths...)) @@ -146,7 +146,7 @@ func repoRelativePath(root, absolute string) (string, error) { return admit.SafeRepoRelativePath(relative, "browser proof resolved input") } -func validateBrowserWitnessPolicy(root string, expected []string) error { +func validateBrowserWitnessPolicy(root, commandID string, expected []string) error { raw, err := os.ReadFile(filepath.Join(root, "proofkit/witness-plan.json")) if err != nil { return err @@ -169,7 +169,7 @@ func validateBrowserWitnessPolicy(root string, expected []string) error { if !ok { return fmt.Errorf("witness plan policies[%d] must be an object", index) } - if policy["commandId"] != "proofkit.browser-check" { + if policy["commandId"] != commandID { continue } matches++ @@ -182,7 +182,7 @@ func validateBrowserWitnessPolicy(root string, expected []string) error { } } if matches != 1 { - return fmt.Errorf("witness plan must contain exactly one browser-check policy") + return fmt.Errorf("witness plan must contain exactly one policy for the browser proof command") } return nil } diff --git a/internal/tools/browserproofverify/main.go b/internal/tools/browserproofverify/main.go index 2be1dca7..eb090e49 100644 --- a/internal/tools/browserproofverify/main.go +++ b/internal/tools/browserproofverify/main.go @@ -22,8 +22,6 @@ import ( "github.com/research-engineering/agentic-proofkit/internal/kernel/unicodepolicy" ) -const proofPath = "artifacts/proofkit/browser-runtime-proof.json" - var proofNonClaims = []string{ "This record proves only that the listed content-bound Playwright tests passed in each recorded project before record creation.", "Playwright WebKit is not branded Safari compatibility proof.", @@ -32,11 +30,23 @@ var proofNonClaims = []string{ "This record does not independently authenticate or sandbox the same-user proof runner, external toolchain packages, or browser binaries.", } +const usage = "usage: browserproofverify [--manifest PATH] [--resolve-inputs | --run [--require-reference-environment] | --admit-playwright-report PATH | --admit-references --from RUN_DIRECTORY]" + func main() { - if len(os.Args) == 3 && os.Args[1] == "--admit-playwright-report" { + arguments := os.Args[1:] + kind := runtimeProofKind() + if len(arguments) >= 2 && arguments[0] == "--manifest" { + selected, err := selectProofKind(arguments[1]) + if err != nil { + exit(err) + } + kind = selected + arguments = arguments[2:] + } + if len(arguments) == 2 && arguments[0] == "--admit-playwright-report" { root, err := os.Getwd() if err == nil { - err = writeAdmittedPlaywrightReport(root, os.Args[2]) + err = writeAdmittedPlaywrightReport(root, arguments[1], kind) } if err != nil { exit(err) @@ -48,26 +58,30 @@ func main() { exit(err) } switch { - case len(os.Args) == 1: - err = verifyCurrentProof(root) - case len(os.Args) == 2 && os.Args[1] == "--resolve-inputs": - err = writeCurrentInputResolution(root) - case len(os.Args) == 2 && os.Args[1] == "--run": - err = runBrowserProof(root) + case len(arguments) == 0: + err = verifyCurrentProof(root, kind) + case len(arguments) == 1 && arguments[0] == "--resolve-inputs": + err = writeCurrentInputResolution(root, kind) + case len(arguments) == 1 && arguments[0] == "--run": + err = runBrowserProof(root, kind, false) + case len(arguments) == 2 && arguments[0] == "--run" && arguments[1] == "--require-reference-environment" && kind.admitsState("unverified"): + err = runBrowserProof(root, kind, true) + case len(arguments) == 3 && arguments[0] == "--admit-references" && arguments[1] == "--from" && kind.isVisual(): + err = writeVisualReferenceInventory(root, arguments[2]) default: - err = fmt.Errorf("usage: browserproofverify [--resolve-inputs | --run | --admit-playwright-report PATH]") + err = fmt.Errorf("%s", usage) } if err != nil { exit(err) } } -func verifyCurrentProof(root string) error { - manifest, err := loadProofInputManifest(root) +func verifyCurrentProof(root string, kind proofKind) error { + manifest, err := loadProofInputManifest(root, kind) if err != nil { return err } - resolution, err := resolveProofInputs(root, manifest) + resolution, err := resolveProofInputs(root, manifest, kind) if err != nil { return err } @@ -75,19 +89,26 @@ func verifyCurrentProof(root string) error { if err != nil { return err } - value, err := readRootedJSON(root, proofPath, 8<<20) + value, err := readRootedJSON(root, kind.RecordPath, 8<<20) if err != nil { return err } + return verifyKindRecord(root, kind, value, resolution, revision, state) +} + +func verifyKindRecord(root string, kind proofKind, value any, resolution proofInputResolution, revision, state string) error { + if kind.isVisual() { + return verifyVisualRecord(root, value, resolution, revision, state) + } return verifyRecord(root, value, resolution, revision, state) } -func writeCurrentInputResolution(root string) error { - manifest, err := loadProofInputManifest(root) +func writeCurrentInputResolution(root string, kind proofKind) error { + manifest, err := loadProofInputManifest(root, kind) if err != nil { return err } - resolution, err := resolveProofInputs(root, manifest) + resolution, err := resolveProofInputs(root, manifest, kind) if err != nil { return err } @@ -99,12 +120,12 @@ func writeCurrentInputResolution(root string) error { return err } -func runBrowserProof(root string) (resultErr error) { - manifest, err := loadProofInputManifest(root) +func runBrowserProof(root string, kind proofKind, requireReferenceEnvironment bool) (resultErr error) { + manifest, err := loadProofInputManifest(root, kind) if err != nil { return err } - resolution, err := resolveProofInputs(root, manifest) + resolution, err := resolveProofInputs(root, manifest, kind) if err != nil { return err } @@ -112,11 +133,14 @@ func runBrowserProof(root string) (resultErr error) { if err != nil { return err } - runPaths, err := prepareBrowserProofRun(root) + runPaths, err := prepareBrowserProofRun(root, kind) if err != nil { return err } defer func() { + if kind.RetainRun { + return + } resultErr = finalizeBrowserProofRun(root, runPaths.RunDirectory, resultErr) }() command := exec.Command("node", manifest.WriterPath) @@ -142,13 +166,39 @@ func runBrowserProof(root string) (resultErr error) { if err != nil { return err } - if err := verifyRecord(root, value, resolution, revision, state); err != nil { + if err := verifyKindRecord(root, kind, value, resolution, revision, state); err != nil { return fmt.Errorf("admit browser proof candidate: %w", err) } - if err := writeRootedJSON(root, proofPath, value); err != nil { - return fmt.Errorf("write browser runtime proof: %w", err) + record := value.(map[string]any) + if record["state"] == "unverified" { + divergent := visualDivergentFields(record) + if requireReferenceEnvironment { + return fmt.Errorf("browser visual proof is unverified: the observed environment differs from the reference environment in %s", strings.Join(divergent, ", ")) + } + fmt.Fprintln(os.Stdout, "browser visual proof: unverified; observed environment differs from the reference environment in: "+strings.Join(divergent, ", ")) } - return verifyCurrentProof(root) + if err := writeRootedJSON(root, kind.RecordPath, value); err != nil { + return fmt.Errorf("write browser proof record: %w", err) + } + return verifyCurrentProof(root, kind) +} + +// writeVisualReferenceInventory is the owner's admission of one run's +// captures; the gate never calls it. +func writeVisualReferenceInventory(root, runDirectory string) error { + encoded, err := buildVisualReferenceInventoryFromRun(root, runDirectory) + if err != nil { + return err + } + rootFS, err := os.OpenRoot(root) + if err != nil { + return err + } + defer rootFS.Close() + if err := admitRootedDestination(rootFS, filepath.FromSlash(visualReferencesPath)); err != nil { + return err + } + return rootFS.WriteFile(filepath.FromSlash(visualReferencesPath), encoded, 0o644) } func currentSourceIdentity(root string) (string, string, error) { @@ -212,26 +262,62 @@ func verifyRecord(root string, raw any, expectedResolution proofInputResolution, if err := exactTextArray(command["argv"], []string{"node_modules/@playwright/test/cli.js", "test"}, "browser runtime proof command argv"); err != nil { return err } + if err := verifyContentBoundInputs(root, record, expectedResolution, "browser runtime proof"); err != nil { + return err + } + engines, ok := record["engines"].([]any) + if !ok || len(engines) != 3 { + return fmt.Errorf("browser runtime proof must record three engines") + } + engineVersions := make([]string, len(engines)) + for index, name := range []string{"chromium", "firefox", "webkit"} { + engine, ok := engines[index].(map[string]any) + if !ok || admit.KnownKeys(engine, []string{"name", "version"}, "browser runtime proof engine") != nil || engine["name"] != name { + return fmt.Errorf("browser runtime proof engine identity is invalid") + } + version, err := admit.NonEmptyText(engine["version"], "browser runtime proof engine version") + if err != nil { + return err + } + engineVersions[index] = version + } + projectVersions, err := verifyProjectExecutions(record["projects"]) + if err != nil { + return err + } + if !slices.Equal(engineVersions, projectVersions) { + return fmt.Errorf("browser runtime proof engine versions must come from the recorded project executions") + } + findings, err := secretjson.Scan(record, "browser_runtime_proof") + if err != nil || len(findings) > 0 { + return fmt.Errorf("browser runtime proof contains secret-shaped data") + } + return nil +} + +// verifyContentBoundInputs binds a record's input resolution, ordered assets +// and inputDigest to the resolved input set and the current file bytes. +func verifyContentBoundInputs(root string, record map[string]any, expectedResolution proofInputResolution, label string) error { inputResolution, ok := record["inputResolution"].(map[string]any) - if !ok || admit.KnownKeys(inputResolution, []string{"serverTarget", "writerPath"}, "browser runtime proof inputResolution") != nil || + if !ok || admit.KnownKeys(inputResolution, []string{"serverTarget", "writerPath"}, label+" inputResolution") != nil || inputResolution["serverTarget"] != expectedResolution.ServerTarget || inputResolution["writerPath"] != expectedResolution.WriterPath { - return fmt.Errorf("browser runtime proof input resolution is invalid") + return fmt.Errorf("%s input resolution is invalid", label) } assets, ok := record["assets"].([]any) if !ok || len(assets) != len(expectedResolution.InputPaths) { - return fmt.Errorf("browser runtime proof assets do not match the input set") + return fmt.Errorf("%s assets do not match the input set", label) } identityAssets := make([]any, 0, len(assets)) for index, rawAsset := range assets { asset, ok := rawAsset.(map[string]any) - if !ok || admit.KnownKeys(asset, []string{"path", "sha256"}, "browser runtime proof asset") != nil || asset["path"] != expectedResolution.InputPaths[index] { - return fmt.Errorf("browser runtime proof asset order or shape is invalid") + if !ok || admit.KnownKeys(asset, []string{"path", "sha256"}, label+" asset") != nil || asset["path"] != expectedResolution.InputPaths[index] { + return fmt.Errorf("%s asset order or shape is invalid", label) } - path, err := admit.SafeRepoRelativePath(expectedResolution.InputPaths[index], "browser runtime proof asset path") + path, err := admit.SafeRepoRelativePath(expectedResolution.InputPaths[index], label+" asset path") if err != nil { return err } - sha, err := admit.LowercaseSHA256(asset["sha256"], "browser runtime proof asset sha256") + sha, err := admit.LowercaseSHA256(asset["sha256"], label+" asset sha256") if err != nil { return err } @@ -241,7 +327,7 @@ func verifyRecord(root string, raw any, expectedResolution proofInputResolution, } actual := sha256.Sum256(content) if sha != hex.EncodeToString(actual[:]) { - return fmt.Errorf("browser runtime proof asset digest mismatch at %s", admit.RedactStructuralText(path)) + return fmt.Errorf("%s asset digest mismatch at %s", label, admit.RedactStructuralText(path)) } identityAssets = append(identityAssets, map[string]any{"path": path, "sha256": sha}) } @@ -257,34 +343,7 @@ func verifyRecord(root string, raw any, expectedResolution proofInputResolution, } expectedInputDigest := digest.SHA256TextRef(strings.TrimSuffix(string(encodedIdentity), "\n")) if record["inputDigest"] != expectedInputDigest { - return fmt.Errorf("browser runtime proof inputDigest mismatch: got %s want %s", admit.RedactStructuralText(fmt.Sprint(record["inputDigest"])), expectedInputDigest) - } - engines, ok := record["engines"].([]any) - if !ok || len(engines) != 3 { - return fmt.Errorf("browser runtime proof must record three engines") - } - engineVersions := make([]string, len(engines)) - for index, name := range []string{"chromium", "firefox", "webkit"} { - engine, ok := engines[index].(map[string]any) - if !ok || admit.KnownKeys(engine, []string{"name", "version"}, "browser runtime proof engine") != nil || engine["name"] != name { - return fmt.Errorf("browser runtime proof engine identity is invalid") - } - version, err := admit.NonEmptyText(engine["version"], "browser runtime proof engine version") - if err != nil { - return err - } - engineVersions[index] = version - } - projectVersions, err := verifyProjectExecutions(record["projects"]) - if err != nil { - return err - } - if !slices.Equal(engineVersions, projectVersions) { - return fmt.Errorf("browser runtime proof engine versions must come from the recorded project executions") - } - findings, err := secretjson.Scan(record, "browser_runtime_proof") - if err != nil || len(findings) > 0 { - return fmt.Errorf("browser runtime proof contains secret-shaped data") + return fmt.Errorf("%s inputDigest mismatch: got %s want %s", label, admit.RedactStructuralText(fmt.Sprint(record["inputDigest"])), expectedInputDigest) } return nil } diff --git a/internal/tools/browserproofverify/main_test.go b/internal/tools/browserproofverify/main_test.go index 9f084a17..f7374791 100644 --- a/internal/tools/browserproofverify/main_test.go +++ b/internal/tools/browserproofverify/main_test.go @@ -40,7 +40,7 @@ func TestInputManifestClosesGoDependenciesAndWitnessPolicy(t *testing.T) { if err != nil { t.Fatal(err) } - manifest, err := loadProofInputManifest(root) + manifest, err := loadProofInputManifest(root, runtimeProofKind()) if err != nil { t.Fatal(err) } @@ -52,7 +52,7 @@ func TestInputManifestClosesGoDependenciesAndWitnessPolicy(t *testing.T) { t.Fatalf("manifest omits required browser proof input selector %q", required) } } - resolution, err := resolveProofInputs(root, manifest) + resolution, err := resolveProofInputs(root, manifest, runtimeProofKind()) if err != nil { t.Fatal(err) } @@ -65,7 +65,7 @@ func TestInputManifestClosesGoDependenciesAndWitnessPolicy(t *testing.T) { weakened := slices.DeleteFunc(append([]string{}, resolution.Selectors...), func(path string) bool { return path == "internal/command/requirementcontext" }) - if err := validateBrowserWitnessPolicy(root, weakened); err == nil { + if err := validateBrowserWitnessPolicy(root, "proofkit.browser-check", weakened); err == nil { t.Fatal("witness policy accepted a resolver projection with a missing Go dependency") } } @@ -87,10 +87,10 @@ func TestInputManifestRejectsNonCanonicalAuthority(t *testing.T) { if err := os.MkdirAll(filepath.Join(root, "scripts"), 0o755); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(root, filepath.FromSlash(proofInputManifestPath)), []byte(item.raw), 0o644); err != nil { + if err := os.WriteFile(filepath.Join(root, filepath.FromSlash(runtimeProofKind().ManifestPath)), []byte(item.raw), 0o644); err != nil { t.Fatal(err) } - if _, err := loadProofInputManifest(root); err == nil { + if _, err := loadProofInputManifest(root, runtimeProofKind()); err == nil { t.Fatalf("loadProofInputManifest accepted %s", item.name) } }) diff --git a/internal/tools/browserproofverify/operation_guard_test.go b/internal/tools/browserproofverify/operation_guard_test.go index 91d38a98..7a15c3f2 100644 --- a/internal/tools/browserproofverify/operation_guard_test.go +++ b/internal/tools/browserproofverify/operation_guard_test.go @@ -85,11 +85,11 @@ func writeWitnessPlan(t *testing.T, root string, selectors []any) { // stageCandidate writes the record the writer will publish; mutate may break it. func stageCandidate(t *testing.T, root string, mutate func(map[string]any)) { t.Helper() - manifest, err := loadProofInputManifest(root) + manifest, err := loadProofInputManifest(root, runtimeProofKind()) if err != nil { t.Fatal(err) } - resolution, err := resolveProofInputs(root, manifest) + resolution, err := resolveProofInputs(root, manifest, runtimeProofKind()) if err != nil { t.Fatal(err) } @@ -117,9 +117,9 @@ func observeBrowserRun(t *testing.T, root string) (writerRan bool, proofWritten t.Helper() markers := t.TempDir() t.Setenv("PROOFKIT_OPERATION_MARKERS", markers) - err = runBrowserProof(root) + err = runBrowserProof(root, runtimeProofKind(), false) _, writerErr := os.Stat(filepath.Join(markers, "writer.log")) - _, proofErr := os.Stat(filepath.Join(root, filepath.FromSlash(proofPath))) + _, proofErr := os.Stat(filepath.Join(root, filepath.FromSlash(runtimeProofKind().RecordPath))) _, artifactsErr := os.Lstat(filepath.Join(root, browserArtifactsDirectory)) return writerErr == nil, proofErr == nil, artifactsErr == nil, err } diff --git a/internal/tools/browserproofverify/playwright_report.go b/internal/tools/browserproofverify/playwright_report.go index edb170b5..d027a7bb 100644 --- a/internal/tools/browserproofverify/playwright_report.go +++ b/internal/tools/browserproofverify/playwright_report.go @@ -25,7 +25,7 @@ type browserProjectExecution struct { TestIDs []string } -func writeAdmittedPlaywrightReport(root, reportPath string) error { +func writeAdmittedPlaywrightReport(root, reportPath string, kind proofKind) error { info, err := os.Lstat(reportPath) if err != nil { return err @@ -40,11 +40,11 @@ func writeAdmittedPlaywrightReport(root, reportPath string) error { if err != nil { return err } - manifest, err := loadProofInputManifest(root) + manifest, err := loadProofInputManifest(root, kind) if err != nil { return err } - resolution, err := resolveProofInputs(root, manifest) + resolution, err := resolveProofInputs(root, manifest, kind) if err != nil { return err } @@ -52,7 +52,11 @@ func writeAdmittedPlaywrightReport(root, reportPath string) error { for _, path := range resolution.InputPaths { admittedPaths[path] = struct{}{} } - projects, err := admitPlaywrightReport(bytes.NewReader(raw), root, manifest.TestRoot, admittedPaths) + var expectedTests map[string][]string + if kind.isVisual() { + expectedTests = expectedVisualProjectTests() + } + projects, err := admitPlaywrightReport(bytes.NewReader(raw), root, manifest.TestRoot, admittedPaths, expectedTests) if err != nil { return err } @@ -64,7 +68,10 @@ func writeAdmittedPlaywrightReport(root, reportPath string) error { return err } -func admitPlaywrightReport(reader io.Reader, root, testRoot string, admittedPaths map[string]struct{}) ([]browserProjectExecution, error) { +// admitPlaywrightReport admits one all-passing report. Without expectedTests, +// every project must pass the same test identities; with expectedTests, each +// project must pass exactly its own expected identity set. +func admitPlaywrightReport(reader io.Reader, root, testRoot string, admittedPaths map[string]struct{}, expectedTests map[string][]string) ([]browserProjectExecution, error) { raw, err := admission.DecodeJSON(reader, maxPlaywrightReportBytes) if err != nil { return nil, err @@ -114,9 +121,17 @@ func admitPlaywrightReport(reader io.Reader, root, testRoot string, admittedPath projects[index] = browserProjectExecution{BrowserName: name, BrowserVersion: browserVersion, Name: name, TestIDs: testIDs} totalExecutions += len(testIDs) } - for _, project := range projects[1:] { - if !slices.Equal(project.TestIDs, projects[0].TestIDs) { - return nil, fmt.Errorf("playwright projects did not execute the same test identities") + if expectedTests != nil { + for _, project := range projects { + if !slices.Equal(project.TestIDs, expectedTests[project.Name]) { + return nil, fmt.Errorf("playwright project did not execute exactly its expected test identities") + } + } + } else { + for _, project := range projects[1:] { + if !slices.Equal(project.TestIDs, projects[0].TestIDs) { + return nil, fmt.Errorf("playwright projects did not execute the same test identities") + } } } if err := admitPlaywrightStats(report["stats"], totalExecutions); err != nil { diff --git a/internal/tools/browserproofverify/playwright_report_test.go b/internal/tools/browserproofverify/playwright_report_test.go index 31e81857..bd3ed612 100644 --- a/internal/tools/browserproofverify/playwright_report_test.go +++ b/internal/tools/browserproofverify/playwright_report_test.go @@ -12,7 +12,7 @@ import ( func TestAdmitPlaywrightReportRequiresCleanEquivalentProjectExecutions(t *testing.T) { root := t.TempDir() report := validPlaywrightReport(root) - projects, err := admitPlaywrightReport(bytes.NewReader(encodePlaywrightReport(t, report)), root, "tests/browser", admittedBrowserTestPaths()) + projects, err := admitPlaywrightReport(bytes.NewReader(encodePlaywrightReport(t, report)), root, "tests/browser", admittedBrowserTestPaths(), nil) if err != nil { t.Fatal(err) } @@ -33,7 +33,7 @@ func TestAdmitPlaywrightReportResolvesTestDirRelativeSpecPaths(t *testing.T) { for _, rawSpec := range firstPlaywrightSuite(report)["specs"].([]any) { rawSpec.(map[string]any)["file"] = "workspace.spec.mjs" } - projects, err := admitPlaywrightReport(bytes.NewReader(encodePlaywrightReport(t, report)), root, "tests/browser", admittedBrowserTestPaths()) + projects, err := admitPlaywrightReport(bytes.NewReader(encodePlaywrightReport(t, report)), root, "tests/browser", admittedBrowserTestPaths(), nil) if err != nil { t.Fatal(err) } @@ -94,7 +94,7 @@ func TestAdmitPlaywrightReportRejectsInvalidEvidenceSemantics(t *testing.T) { root := t.TempDir() report := validPlaywrightReport(root) item.mutate(report) - if _, err := admitPlaywrightReport(bytes.NewReader(encodePlaywrightReport(t, report)), root, "tests/browser", admittedBrowserTestPaths()); err == nil { + if _, err := admitPlaywrightReport(bytes.NewReader(encodePlaywrightReport(t, report)), root, "tests/browser", admittedBrowserTestPaths(), nil); err == nil { t.Fatal("expected report admission failure") } }) @@ -105,7 +105,7 @@ func TestAdmitPlaywrightReportRejectsDuplicateJSONKeys(t *testing.T) { root := t.TempDir() encoded := string(encodePlaywrightReport(t, validPlaywrightReport(root))) duplicated := strings.Replace(encoded, `"suites":`, `"suites":[],"suites":`, 1) - if _, err := admitPlaywrightReport(strings.NewReader(duplicated), root, "tests/browser", admittedBrowserTestPaths()); err == nil || !strings.Contains(err.Error(), "duplicate object key") { + if _, err := admitPlaywrightReport(strings.NewReader(duplicated), root, "tests/browser", admittedBrowserTestPaths(), nil); err == nil || !strings.Contains(err.Error(), "duplicate object key") { t.Fatalf("expected duplicate-key rejection, got %v", err) } } diff --git a/internal/tools/browserproofverify/proof_kinds.go b/internal/tools/browserproofverify/proof_kinds.go new file mode 100644 index 00000000..62ee92d3 --- /dev/null +++ b/internal/tools/browserproofverify/proof_kinds.go @@ -0,0 +1,76 @@ +package main + +import ( + "fmt" + "slices" +) + +// proofKind is one row of the closed table that maps an input manifest to the +// witness command, record path and record states it owns. The runtime row keeps +// the original manifest, command, record, states and run cleanup exactly. The +// visual row retains its run directory, passing or failing, because the run's +// captures, candidate inventory and observed environment are its gallery and +// the only admissible source of new references. +type proofKind struct { + ManifestPath string + ManifestKind string + CommandID string + RecordPath string + RecordKind string + CandidateName string + States []string + RetainRun bool +} + +const ( + runtimeProofManifestPath = "scripts/browser-runtime-proof-inputs.v1.json" + visualProofManifestPath = "scripts/browser-visual-proof-inputs.v1.json" +) + +var proofKinds = []proofKind{ + { + ManifestPath: runtimeProofManifestPath, + ManifestKind: "proofkit.browser-runtime-proof-inputs", + CommandID: "proofkit.browser-check", + RecordPath: "artifacts/proofkit/browser-runtime-proof.json", + RecordKind: "proofkit.browser-runtime-proof", + CandidateName: "browser-runtime-proof.candidate.json", + States: []string{"passed"}, + }, + { + ManifestPath: visualProofManifestPath, + ManifestKind: "proofkit.browser-visual-proof-inputs", + CommandID: "proofkit.browser-visual", + RecordPath: "artifacts/proofkit/browser-visual-proof.json", + RecordKind: "proofkit.browser-visual-proof", + CandidateName: "browser-visual-proof.candidate.json", + States: []string{"passed", "unverified"}, + RetainRun: true, + }, +} + +func runtimeProofKind() proofKind { + return proofKinds[0] +} + +func visualProofKind() proofKind { + return proofKinds[1] +} + +func (kind proofKind) isVisual() bool { + return kind.ManifestPath == visualProofManifestPath +} + +func (kind proofKind) admitsState(state string) bool { + return slices.Contains(kind.States, state) +} + +// selectProofKind admits only a manifest path that is a row of the table. +func selectProofKind(manifestPath string) (proofKind, error) { + for _, kind := range proofKinds { + if kind.ManifestPath == manifestPath { + return kind, nil + } + } + return proofKind{}, fmt.Errorf("browser proof manifest is not an admitted proof kind") +} diff --git a/internal/tools/browserproofverify/visual_inventory.go b/internal/tools/browserproofverify/visual_inventory.go new file mode 100644 index 00000000..03f5f2ce --- /dev/null +++ b/internal/tools/browserproofverify/visual_inventory.go @@ -0,0 +1,654 @@ +package main + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "slices" + "sort" + "strconv" + "strings" + + "github.com/research-engineering/agentic-proofkit/internal/kernel/admission" + "github.com/research-engineering/agentic-proofkit/internal/kernel/admit" + "github.com/research-engineering/agentic-proofkit/internal/kernel/diagnostic" + "github.com/research-engineering/agentic-proofkit/internal/kernel/digest" + "github.com/research-engineering/agentic-proofkit/internal/kernel/stablejson" +) + +// The visual reference inventory is owned here: every rendered view in each +// engine, color scheme and width, plus one README scene. References are +// canonical RGBA digests with dimensions; no image is stored in the +// repository. The harness renders these views; the record and the committed +// inventory must equal this set. +const ( + visualTestRoot = "tests/visual" + visualEnvironmentPath = "tests/visual/environment.json" + visualReferencesPath = "tests/visual/references.json" + visualFontConfigPath = "tests/visual/fonts.conf" + visualCaptureDigestScript = "scripts/browser-visual-capture-digests.mjs" + visualObservedRunFile = "observed-environment.json" + visualCandidateRunFile = "candidate-references.json" + visualCaptureRunDirectory = "captures" + visualUnstableRunDirectory = "unstable" + visualClipHeight = 2400 + readmeSceneView = "readme-scene" + readmeSceneWidth = 1440 + readmeSceneHeight = 960 + visualMaxCaptureDigestBytes = 1 << 20 +) + +var ( + visualEngines = []string{"chromium", "firefox", "webkit"} + visualSchemes = []string{"light", "dark"} + visualWidths = []int{320, 1440} + // Five static renderings in Cards and Table modes, then four workspace tabs. + visualViews = []string{ + "static-source-cards", "static-source-table", + "static-spec-tree-cards", "static-spec-tree-table", + "static-proof-structured-cards", "static-proof-structured-table", + "static-proof-compact-cards", "static-proof-compact-table", + "static-coverage-cards", "static-coverage-table", + "workspace-specifications", "workspace-coverage", "workspace-diff", "workspace-traceability", + } + // Controls run in every engine: a routed unchanged stylesheet copy must + // keep the admitted digest, and each mutated copy must change it. + visualControlTitles = []string{ + "a change of the monospace family to sans-serif differs from its reference", + "a change of the sans-serif family to serif differs from its reference", + "a font-weight change from 400 to 700 differs from its reference", + "a one-pixel shift differs from its reference", + "a one-unit text color change differs from its reference", + "a routed unchanged stylesheet keeps its reference", + } + // The font-face check runs in every engine: each product font stack must + // draw its intended face. + visualFontFaceTitles = []string{ + "each product font stack draws its intended face", + } + // Fields of the observed environment tuple compared before any browser + // launch. The CPU is recorded in every run but is not compared. + visualComparedEnvironmentFields = []string{"architecture", "dockerImage", "emulation", "engines", "fontConfig", "fontMatches", "os", "packages"} +) + +type visualReference struct { + Engine string + View string + Scheme string + Width int +} + +// id is the reference ID /... +func (reference visualReference) id() string { + return fmt.Sprintf("%s/%s.%s.%d", reference.Engine, reference.View, reference.Scheme, reference.Width) +} + +func (reference visualReference) isReadmeScene() bool { + return reference.View == readmeSceneView +} + +func (reference visualReference) testID() string { + if reference.isReadmeScene() { + return fmt.Sprintf("%s/readme-scene.spec.mjs::readme-scene.spec.mjs > %s %s %dx%d", visualTestRoot, reference.View, reference.Scheme, readmeSceneWidth, readmeSceneHeight) + } + return fmt.Sprintf("%s/references.spec.mjs::references.spec.mjs > %s %s %d", visualTestRoot, reference.View, reference.Scheme, reference.Width) +} + +// admitDimensions bounds a reference's dimensions: a view is exactly its +// width and at most clipHeight tall; the README scene is exactly 1440x960. +func (reference visualReference) admitDimensions(width, height int) error { + if reference.isReadmeScene() { + if width != readmeSceneWidth || height != readmeSceneHeight { + return fmt.Errorf("visual reference %s must be %dx%d", reference.id(), readmeSceneWidth, readmeSceneHeight) + } + return nil + } + if width != reference.Width || height < 1 || height > visualClipHeight { + return fmt.Errorf("visual reference %s must be %d CSS pixels wide and 1 to %d tall", reference.id(), reference.Width, visualClipHeight) + } + return nil +} + +// expectedVisualReferences returns the exact Cartesian inventory sorted by ID. +func expectedVisualReferences() []visualReference { + references := make([]visualReference, 0, len(visualViews)*len(visualEngines)*len(visualSchemes)*len(visualWidths)+1) + for _, engine := range visualEngines { + for _, view := range visualViews { + for _, scheme := range visualSchemes { + for _, width := range visualWidths { + references = append(references, visualReference{Engine: engine, View: view, Scheme: scheme, Width: width}) + } + } + } + } + references = append(references, visualReference{Engine: "chromium", View: readmeSceneView, Scheme: "light", Width: readmeSceneWidth}) + sort.Slice(references, func(left, right int) bool { return references[left].id() < references[right].id() }) + return references +} + +// expectedVisualProjectTests is the exact test identity set each project must +// pass: one comparison per reference of that engine plus the controls and the +// font-face check. +func expectedVisualProjectTests() map[string][]string { + result := make(map[string][]string, len(visualEngines)) + for _, engine := range visualEngines { + ids := []string{} + for _, title := range visualControlTitles { + ids = append(ids, fmt.Sprintf("%s/controls.spec.mjs::controls.spec.mjs > %s", visualTestRoot, title)) + } + for _, title := range visualFontFaceTitles { + ids = append(ids, fmt.Sprintf("%s/font-faces.spec.mjs::font-faces.spec.mjs > %s", visualTestRoot, title)) + } + for _, reference := range expectedVisualReferences() { + if reference.Engine == engine { + ids = append(ids, reference.testID()) + } + } + slices.Sort(ids) + result[engine] = ids + } + return result +} + +type visualEnvironment struct { + EnvironmentID string + Observed map[string]any + BrowserVersions map[string]string + Raw map[string]any +} + +// loadVisualEnvironment admits the recorded reference environment and returns +// its identity: the SHA-256 of its canonical compact JSON. +func loadVisualEnvironment(root string) (visualEnvironment, error) { + value, err := readRepositoryJSON(root, visualEnvironmentPath, 1<<20) + if err != nil { + return visualEnvironment{}, fmt.Errorf("read visual reference environment: %w", err) + } + record, ok := value.(map[string]any) + if !ok || admit.KnownKeys(record, []string{"browserVersions", "image", "kind", "observed", "schemaVersion"}, "visual reference environment") != nil || + !admit.JSONNumberEquals(record["schemaVersion"], 1) || record["kind"] != "proofkit.browser-visual-environment" { + return visualEnvironment{}, fmt.Errorf("visual reference environment identity is invalid") + } + if err := admitVisualImage(record["image"]); err != nil { + return visualEnvironment{}, err + } + observed, ok := record["observed"].(map[string]any) + if !ok || admit.KnownKeys(observed, visualComparedEnvironmentFields, "visual reference environment observed tuple") != nil || len(observed) != len(visualComparedEnvironmentFields) { + return visualEnvironment{}, fmt.Errorf("visual reference environment must record exactly the compared tuple") + } + if err := admitObservedTuple(observed, true); err != nil { + return visualEnvironment{}, err + } + versions, ok := record["browserVersions"].(map[string]any) + if !ok || admit.KnownKeys(versions, visualEngines, "visual reference environment browserVersions") != nil || len(versions) != len(visualEngines) { + return visualEnvironment{}, fmt.Errorf("visual reference environment must record one browser version per engine") + } + browserVersions := make(map[string]string, len(versions)) + for _, engine := range visualEngines { + version, err := admit.NonEmptyText(versions[engine], "visual reference environment browser version") + if err != nil { + return visualEnvironment{}, err + } + browserVersions[engine] = version + } + encoded, err := stablejson.MarshalLayout(record, stablejson.LayoutCompact) + if err != nil { + return visualEnvironment{}, err + } + return visualEnvironment{ + EnvironmentID: digest.SHA256TextRef(strings.TrimSuffix(string(encoded), "\n")), + Observed: observed, + BrowserVersions: browserVersions, + Raw: record, + }, nil +} + +func admitVisualImage(raw any) error { + image, ok := raw.(map[string]any) + if !ok || admit.KnownKeys(image, []string{"indexDigest", "name", "platform", "platformDigests", "tag"}, "visual reference environment image") != nil || len(image) != 5 { + return fmt.Errorf("visual reference environment image is invalid") + } + for _, key := range []string{"name", "tag", "platform"} { + if _, err := admit.NonEmptyText(image[key], "visual reference environment image "+key); err != nil { + return err + } + } + if _, err := admit.SHA256Ref(image["indexDigest"], "visual reference environment image indexDigest"); err != nil { + return err + } + platforms, ok := image["platformDigests"].(map[string]any) + if !ok || len(platforms) == 0 { + return fmt.Errorf("visual reference environment image must record its platform digests") + } + if _, recorded := platforms[fmt.Sprint(image["platform"])]; !recorded { + return fmt.Errorf("visual reference environment image platform digest is missing") + } + for platform, value := range platforms { + if _, err := admit.NonEmptyText(platform, "visual reference environment image platform"); err != nil { + return err + } + if _, err := admit.SHA256Ref(value, "visual reference environment image platform digest"); err != nil { + return err + } + } + return nil +} + +// admitObservedTuple checks the closed shape of an observed tuple. A reference +// environment must observe every field; a run may observe absent fields. +func admitObservedTuple(observed map[string]any, reference bool) error { + for _, field := range visualComparedEnvironmentFields { + value, present := observed[field] + if !present { + return fmt.Errorf("observed environment omits a compared field") + } + if value == nil { + if reference { + return fmt.Errorf("visual reference environment field must be observed") + } + continue + } + switch field { + case "architecture", "emulation": + if _, err := admit.NonEmptyText(value, "observed environment "+field); err != nil { + return err + } + case "fontConfig": + if _, err := admit.SHA256Ref(value, "observed environment fontConfig"); err != nil { + return err + } + case "os": + os, ok := value.(map[string]any) + if !ok || admit.KnownKeys(os, []string{"id", "versionId"}, "observed environment os") != nil { + return fmt.Errorf("observed environment os is invalid") + } + case "dockerImage", "engines", "fontMatches", "packages": + fields, ok := value.(map[string]any) + if !ok { + return fmt.Errorf("observed environment %s must be an object", field) + } + for key, item := range fields { + if _, err := admit.NonEmptyText(key, "observed environment key"); err != nil { + return err + } + if item != nil { + if _, ok := item.(string); !ok { + return fmt.Errorf("observed environment %s values must be text or null", field) + } + } + } + } + } + if reference && observed["emulation"] != "none" { + return fmt.Errorf("visual reference environment must not be emulated") + } + return nil +} + +// visualEnvironmentDivergence lists the compared fields whose observed value +// differs from the reference value. +func visualEnvironmentDivergence(reference, observed map[string]any) ([]string, error) { + divergent := []string{} + for _, field := range visualComparedEnvironmentFields { + left, err := stablejson.MarshalLayout(reference[field], stablejson.LayoutCompact) + if err != nil { + return nil, err + } + right, err := stablejson.MarshalLayout(observed[field], stablejson.LayoutCompact) + if err != nil { + return nil, err + } + if !bytes.Equal(left, right) { + divergent = append(divergent, field) + } + } + return divergent, nil +} + +type visualReferenceInventory struct { + EnvironmentID string + ReferencesDigest string + Count int +} + +type visualReferenceDigest struct { + ID string + Width int + Height int + RGBASHA256 string +} + +// admitVisualReferenceEntries admits a closed digest inventory: exactly the +// expected ID set in ID order with no duplicate, a lowercase SHA-256 digest +// per entry, and dimensions within each reference's bounds. +func admitVisualReferenceEntries(raw any, context string) ([]visualReferenceDigest, error) { + entries, ok := raw.([]any) + if !ok { + return nil, fmt.Errorf("%s must list its references", context) + } + expected := expectedVisualReferences() + byID := make(map[string]visualReference, len(expected)) + for _, reference := range expected { + byID[reference.id()] = reference + } + result := make([]visualReferenceDigest, 0, len(entries)) + seen := make(map[string]struct{}, len(entries)) + for index, rawEntry := range entries { + entry, ok := rawEntry.(map[string]any) + if !ok || admit.KnownKeys(entry, []string{"height", "id", "rgbaSHA256", "width"}, context+" reference") != nil || len(entry) != 4 { + return nil, fmt.Errorf("%s entry %d must be exactly {height, id, rgbaSHA256, width}", context, index) + } + id, _ := entry["id"].(string) + reference, known := byID[id] + if !known { + return nil, fmt.Errorf("%s entry %d names an ID outside the expected inventory", context, index) + } + if _, duplicate := seen[id]; duplicate { + return nil, fmt.Errorf("%s lists %s more than once", context, id) + } + seen[id] = struct{}{} + if index > 0 && id <= result[index-1].ID { + return nil, fmt.Errorf("%s must be sorted by ID", context) + } + hash, err := admit.LowercaseSHA256(entry["rgbaSHA256"], context+" rgbaSHA256") + if err != nil { + return nil, fmt.Errorf("%s %s: %w", context, id, err) + } + width, widthErr := admit.CanonicalInteger(entry["width"], context+" width") + height, heightErr := admit.CanonicalInteger(entry["height"], context+" height") + if widthErr != nil || heightErr != nil { + return nil, fmt.Errorf("%s %s dimensions must be integers", context, id) + } + if err := reference.admitDimensions(int(width), int(height)); err != nil { + return nil, err + } + result = append(result, visualReferenceDigest{ID: id, Width: int(width), Height: int(height), RGBASHA256: hash}) + } + if len(result) != len(expected) { + missing := []string{} + for _, reference := range expected { + if _, ok := seen[reference.id()]; !ok { + missing = append(missing, reference.id()) + } + } + return nil, fmt.Errorf("%s must list all %d expected references; missing %d, first %s", context, len(expected), len(missing), firstOrNone(missing)) + } + return result, nil +} + +func firstOrNone(values []string) string { + if len(values) == 0 { + return "none" + } + return values[0] +} + +// verifyVisualReferences admits the committed digest inventory: it must bind +// the current environment identity and equal the expected closed ID set. +func verifyVisualReferences(root string) (visualReferenceInventory, error) { + environment, err := loadVisualEnvironment(root) + if err != nil { + return visualReferenceInventory{}, err + } + raw, err := readRepositoryFile(root, visualReferencesPath, 4<<20) + if err != nil { + return visualReferenceInventory{}, fmt.Errorf("read visual reference inventory: %w", err) + } + value, err := admission.DecodeJSON(bytes.NewReader(raw), int64(len(raw))) + if err != nil { + return visualReferenceInventory{}, err + } + record, ok := value.(map[string]any) + if !ok || admit.KnownKeys(record, []string{"clipHeight", "environmentId", "kind", "references", "schemaVersion"}, "visual reference inventory") != nil || + !admit.JSONNumberEquals(record["schemaVersion"], 1) || record["kind"] != "proofkit.browser-visual-references" || + !admit.JSONNumberEquals(record["clipHeight"], visualClipHeight) { + return visualReferenceInventory{}, fmt.Errorf("visual reference inventory identity is invalid") + } + if record["environmentId"] != environment.EnvironmentID { + return visualReferenceInventory{}, fmt.Errorf("visual reference inventory environmentId does not match the recorded environment; re-admit the references") + } + entries, err := admitVisualReferenceEntries(record["references"], "visual reference inventory") + if err != nil { + return visualReferenceInventory{}, err + } + return visualReferenceInventory{EnvironmentID: environment.EnvironmentID, ReferencesDigest: digest.SHA256BytesRef(raw), Count: len(entries)}, nil +} + +// buildVisualReferenceInventoryFromRun is the owner action that admits the +// captures of one gate run. It admits only a run whose recorded observed +// environment equals the recorded reference environment, with no unstable +// directory and exactly one regular capture file per expected ID. It +// recomputes every digest from the PNG files and refuses a run whose candidate +// inventory disagrees. It cannot authenticate the run directory as unmodified +// gate output; only a passing gate on the admitting tree proves the admitted +// digests. The gate never calls it. +func buildVisualReferenceInventoryFromRun(root, runDirectory string) ([]byte, error) { + environment, err := loadVisualEnvironment(root) + if err != nil { + return nil, err + } + if err := admitRunObservedEnvironment(runDirectory, environment); err != nil { + return nil, err + } + if _, err := os.Lstat(filepath.Join(runDirectory, visualUnstableRunDirectory)); err == nil { + return nil, fmt.Errorf("visual run has unstable captures; every capture must be double-captured byte-equal") + } else if !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + captures := filepath.Join(runDirectory, visualCaptureRunDirectory) + if err := admitRunCaptureFiles(captures); err != nil { + return nil, err + } + recomputed, err := recomputeCaptureDigests(root, captures) + if err != nil { + return nil, err + } + if err := crossCheckCandidateInventory(runDirectory, recomputed); err != nil { + return nil, err + } + references := make([]any, 0, len(recomputed)) + for _, entry := range recomputed { + references = append(references, map[string]any{"height": entry.Height, "id": entry.ID, "rgbaSHA256": entry.RGBASHA256, "width": entry.Width}) + } + return stablejson.MarshalLayout(map[string]any{ + "clipHeight": visualClipHeight, + "environmentId": environment.EnvironmentID, + "kind": "proofkit.browser-visual-references", + "references": references, + "schemaVersion": 1, + }, stablejson.LayoutPretty) +} + +func readRunFile(path string, maxBytes int64) ([]byte, error) { + info, err := os.Lstat(path) + if err != nil { + return nil, err + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return nil, fmt.Errorf("visual run file must be a regular non-symlink file") + } + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + content, err := io.ReadAll(io.LimitReader(file, maxBytes+1)) + if err != nil { + return nil, err + } + if int64(len(content)) > maxBytes { + return nil, fmt.Errorf("visual run file exceeds its size limit") + } + return content, nil +} + +// admitRunObservedEnvironment requires the run's observed environment to equal +// the recorded reference environment in every compared field. +func admitRunObservedEnvironment(runDirectory string, environment visualEnvironment) error { + raw, err := readRunFile(filepath.Join(runDirectory, visualObservedRunFile), 1<<20) + if err != nil { + return fmt.Errorf("read the run's observed environment: %w", err) + } + value, err := admission.DecodeJSON(bytes.NewReader(raw), int64(len(raw))) + if err != nil { + return err + } + observed, ok := value.(map[string]any) + if !ok || admit.KnownKeys(observed, append([]string{"cpu", "divergentFields"}, visualComparedEnvironmentFields...), "visual run observed environment") != nil || + len(observed) != len(visualComparedEnvironmentFields)+2 { + return fmt.Errorf("visual run observed environment must record the CPU, every compared field and divergentFields") + } + if err := admitObservedTuple(observed, false); err != nil { + return err + } + if err := admitObservedCPU(observed["cpu"]); err != nil { + return err + } + divergent, err := visualEnvironmentDivergence(environment.Observed, observed) + if err != nil { + return err + } + if len(divergent) != 0 { + return fmt.Errorf("visual run observed another environment (%s); only a reference-environment run can be admitted", strings.Join(divergent, ", ")) + } + if !emptyArray(observed["divergentFields"]) { + return fmt.Errorf("visual run observed environment lists divergent fields although every compared field equals the recorded environment; only a reference-environment run can be admitted") + } + return nil +} + +// admitRunCaptureFiles requires exactly one regular PNG capture per expected +// ID below the capture directory and nothing else. +func admitRunCaptureFiles(captures string) error { + expected := map[string]struct{}{} + for _, reference := range expectedVisualReferences() { + expected[reference.id()+".png"] = struct{}{} + } + found := map[string]struct{}{} + err := filepath.WalkDir(captures, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.Type()&fs.ModeSymlink != 0 { + return fmt.Errorf("visual run captures must not contain symlinks") + } + if entry.IsDir() { + return nil + } + relative, err := filepath.Rel(captures, path) + if err != nil { + return err + } + relative = filepath.ToSlash(relative) + if _, ok := expected[relative]; !ok || !entry.Type().IsRegular() { + return fmt.Errorf("visual run captures contain an unexpected file %s", admit.RedactStructuralText(relative)) + } + found[relative] = struct{}{} + return nil + }) + if err != nil { + return fmt.Errorf("admit visual run captures: %w", err) + } + if len(found) != len(expected) { + missing := []string{} + for name := range expected { + if _, ok := found[name]; !ok { + missing = append(missing, strings.TrimSuffix(name, ".png")) + } + } + slices.Sort(missing) + return fmt.Errorf("visual run is missing %d of %d captures, first %s", len(missing), len(expected), firstOrNone(missing)) + } + return nil +} + +// recomputeCaptureDigests decodes every capture through the canonical digest +// owner in the visual harness and admits the result as a closed inventory. +func recomputeCaptureDigests(root, captures string) ([]visualReferenceDigest, error) { + absolute, err := filepath.Abs(captures) + if err != nil { + return nil, err + } + command := exec.Command("node", filepath.FromSlash(visualCaptureDigestScript), absolute) + command.Dir = root + var stdout bytes.Buffer + command.Stdout = &stdout + stderr := diagnostic.NewStderrCapture() + command.Stderr = stderr + if err := command.Run(); err != nil { + if childErr := stderr.Failure("visual capture digest stderr"); childErr != nil { + return nil, fmt.Errorf("recompute visual capture digests: %w; %s", err, childErr) + } + return nil, fmt.Errorf("recompute visual capture digests: %w", err) + } + if stdout.Len() > visualMaxCaptureDigestBytes { + return nil, fmt.Errorf("visual capture digest output exceeds its size limit") + } + value, err := admission.DecodeJSON(bytes.NewReader(stdout.Bytes()), visualMaxCaptureDigestBytes) + if err != nil { + return nil, err + } + return admitVisualReferenceEntries(value, "recomputed capture inventory") +} + +// crossCheckCandidateInventory refuses a run whose own candidate inventory +// disagrees with the recomputed digests: a capture changed after the run. +func crossCheckCandidateInventory(runDirectory string, recomputed []visualReferenceDigest) error { + raw, err := readRunFile(filepath.Join(runDirectory, visualCandidateRunFile), 4<<20) + if err != nil { + return fmt.Errorf("read the run's candidate inventory: %w", err) + } + value, err := admission.DecodeJSON(bytes.NewReader(raw), int64(len(raw))) + if err != nil { + return err + } + record, ok := value.(map[string]any) + if !ok { + return fmt.Errorf("visual run candidate inventory must be an object") + } + candidate, err := admitVisualReferenceEntries(record["references"], "visual run candidate inventory") + if err != nil { + return err + } + for index, entry := range recomputed { + if candidate[index] != entry { + return fmt.Errorf("visual run capture %s no longer matches the run's candidate inventory", entry.ID) + } + } + return nil +} + +func readRepositoryFile(root, path string, maxBytes int64) ([]byte, error) { + rootFS, err := os.OpenRoot(root) + if err != nil { + return nil, err + } + defer rootFS.Close() + localPath := filepath.FromSlash(path) + if err := admitRootedRegularFile(rootFS, localPath); err != nil { + return nil, err + } + file, err := rootFS.Open(localPath) + if err != nil { + return nil, err + } + defer file.Close() + // One byte past the limit lets callers distinguish an oversized file. + return io.ReadAll(io.LimitReader(file, maxBytes+1)) +} + +func readRepositoryJSON(root, path string, maxBytes int64) (any, error) { + content, err := readRepositoryFile(root, path, maxBytes) + if err != nil { + return nil, err + } + if int64(len(content)) > maxBytes { + return nil, fmt.Errorf("%s exceeds the %s-byte limit", path, strconv.FormatInt(maxBytes, 10)) + } + return admission.DecodeJSON(bytes.NewReader(content), maxBytes) +} diff --git a/internal/tools/browserproofverify/visual_inventory_test.go b/internal/tools/browserproofverify/visual_inventory_test.go new file mode 100644 index 00000000..01fdb722 --- /dev/null +++ b/internal/tools/browserproofverify/visual_inventory_test.go @@ -0,0 +1,568 @@ +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "image" + "image/color" + "image/png" + "io/fs" + "os" + "os/exec" + "path" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/research-engineering/agentic-proofkit/internal/kernel/admission" +) + +func TestVisualInventoryIsTheExactViewEngineSchemeWidthProduct(t *testing.T) { + references := expectedVisualReferences() + if len(references) != 169 || len(visualViews) != 14 { + t.Fatalf("visual inventory has %d references over %d views, want 169 over 14", len(references), len(visualViews)) + } + ids := map[string]struct{}{} + perEngine := map[string]int{} + for index, reference := range references { + if index > 0 && references[index-1].id() >= reference.id() { + t.Fatal("visual inventory is not sorted by ID") + } + ids[reference.id()] = struct{}{} + perEngine[reference.Engine]++ + } + if len(ids) != 169 || perEngine["chromium"] != 57 || perEngine["firefox"] != 56 || perEngine["webkit"] != 56 { + t.Fatalf("visual inventory engine split = %v", perEngine) + } + if _, ok := ids["chromium/readme-scene.light.1440"]; !ok { + t.Fatal("visual inventory lacks the README scene ID") + } + tests := expectedVisualProjectTests() + if len(tests["chromium"]) != 64 || len(tests["firefox"]) != 63 || len(tests["webkit"]) != 63 { + t.Fatalf("visual project test sets = %d/%d/%d", len(tests["chromium"]), len(tests["firefox"]), len(tests["webkit"])) + } + for _, engine := range visualEngines { + if !slices.Contains(tests[engine], "tests/visual/font-faces.spec.mjs::font-faces.spec.mjs > each product font stack draws its intended face") { + t.Fatalf("%s must run the font-face check", engine) + } + } + if !slices.Contains(tests["chromium"], "tests/visual/readme-scene.spec.mjs::readme-scene.spec.mjs > readme-scene light 1440x960") || + slices.ContainsFunc(tests["firefox"], func(id string) bool { return strings.Contains(id, "readme-scene") }) { + t.Fatal("the README scene must be a Chromium-only comparison") + } +} + +// The committed references are digests bound to the recorded environment; no +// image file exists below the visual test directory. +func TestCommittedVisualReferencesAreDigestsBoundToTheRecordedEnvironment(t *testing.T) { + root, err := repositoryRoot() + if err != nil { + t.Fatal(err) + } + inventory, err := verifyVisualReferences(root) + if err != nil { + t.Fatal(err) + } + environment, err := loadVisualEnvironment(root) + if err != nil { + t.Fatal(err) + } + if inventory.Count != 169 || inventory.EnvironmentID != environment.EnvironmentID { + t.Fatalf("committed inventory = %+v, environment = %s", inventory, environment.EnvironmentID) + } + err = filepath.WalkDir(filepath.Join(root, "tests", "visual"), func(current string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if !entry.IsDir() && strings.EqualFold(filepath.Ext(current), ".png") { + t.Errorf("image file %s is stored below tests/visual; references are digests only", current) + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} + +// copyVisualRecords copies the committed visual records into a disposable root +// so each mutant changes only its copy. +func copyVisualRecords(t *testing.T) string { + t.Helper() + source, err := repositoryRoot() + if err != nil { + t.Fatal(err) + } + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "tests", "visual"), 0o755); err != nil { + t.Fatal(err) + } + for _, relative := range []string{visualEnvironmentPath, visualReferencesPath, visualFontConfigPath} { + content, err := os.ReadFile(filepath.Join(source, filepath.FromSlash(relative))) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, filepath.FromSlash(relative)), content, 0o644); err != nil { + t.Fatal(err) + } + } + return root +} + +func editJSON(t *testing.T, root, relative string, mutate func(map[string]any)) { + t.Helper() + target := filepath.Join(root, filepath.FromSlash(relative)) + raw, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + value, err := admission.DecodeJSON(bytes.NewReader(raw), int64(len(raw))) + if err != nil { + t.Fatal(err) + } + record := value.(map[string]any) + mutate(record) + encoded, err := json.MarshalIndent(record, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target, encoded, 0o644); err != nil { + t.Fatal(err) + } +} + +func referenceEntry(record map[string]any, id string) map[string]any { + for _, raw := range record["references"].([]any) { + entry := raw.(map[string]any) + if entry["id"] == id { + return entry + } + } + panic("reference entry " + id + " is missing") +} + +func TestVisualReferenceInventoryRefusesEveryIntegrityMutant(t *testing.T) { + view := "chromium/static-source-cards.light.320" + readme := "chromium/readme-scene.light.1440" + cases := []struct { + name string + want string + mutate func(record map[string]any) + file string + }{ + {"malformed hash", "lowercase sha256", func(record map[string]any) { + entry := referenceEntry(record, view) + entry["rgbaSHA256"] = strings.ToUpper(entry["rgbaSHA256"].(string)) + }, visualReferencesPath}, + {"short hash", "lowercase sha256", func(record map[string]any) { + entry := referenceEntry(record, view) + entry["rgbaSHA256"] = entry["rgbaSHA256"].(string)[:63] + }, visualReferencesPath}, + {"missing ID", "must list all 169 expected references", func(record map[string]any) { + entries := record["references"].([]any) + record["references"] = append(entries[:5:5], entries[6:]...) + }, visualReferencesPath}, + {"extra ID", "outside the expected inventory", func(record map[string]any) { + extra := map[string]any{"height": json.Number("900"), "id": "chromium/static-extra.light.320", "rgbaSHA256": strings.Repeat("a", 64), "width": json.Number("320")} + record["references"] = append(record["references"].([]any), extra) + }, visualReferencesPath}, + {"duplicate ID", "more than once", func(record map[string]any) { + entries := record["references"].([]any) + record["references"] = append(entries[:2:2], append([]any{entries[1]}, entries[2:]...)...) + }, visualReferencesPath}, + {"reordered inventory", "sorted by ID", func(record map[string]any) { + entries := record["references"].([]any) + entries[3], entries[4] = entries[4], entries[3] + }, visualReferencesPath}, + {"width other than the view width", "320 CSS pixels wide", func(record map[string]any) { + referenceEntry(record, view)["width"] = json.Number("321") + }, visualReferencesPath}, + {"height above the clip", "1 to 2400 tall", func(record map[string]any) { + referenceEntry(record, view)["height"] = json.Number("2401") + }, visualReferencesPath}, + {"README scene with other dimensions", "must be 1440x960", func(record map[string]any) { + referenceEntry(record, readme)["height"] = json.Number("900") + }, visualReferencesPath}, + {"unowned entry key", "exactly {height, id, rgbaSHA256, width}", func(record map[string]any) { + referenceEntry(record, view)["path"] = "tests/visual/references/x.png" + }, visualReferencesPath}, + {"environmentId mismatch", "environmentId does not match", func(record map[string]any) { + record["environmentId"] = "sha256:" + strings.Repeat("0", 64) + }, visualReferencesPath}, + {"changed environment field", "environmentId does not match", func(record map[string]any) { + record["observed"].(map[string]any)["packages"].(map[string]any)["fontconfig"] = "2.15.0-1.1ubuntu3" + }, visualEnvironmentPath}, + {"changed fontconfig digest", "environmentId does not match", func(record map[string]any) { + record["observed"].(map[string]any)["fontConfig"] = "sha256:" + strings.Repeat("1", 64) + }, visualEnvironmentPath}, + {"emulated reference environment", "must not be emulated", func(record map[string]any) { + record["observed"].(map[string]any)["emulation"] = "translator-mapped" + }, visualEnvironmentPath}, + } + for _, item := range cases { + t.Run(item.name, func(t *testing.T) { + root := copyVisualRecords(t) + if _, err := verifyVisualReferences(root); err != nil { + t.Fatalf("unmutated copy was refused: %v", err) + } + editJSON(t, root, item.file, item.mutate) + _, err := verifyVisualReferences(root) + if err == nil || !strings.Contains(err.Error(), item.want) { + t.Fatalf("mutant error = %v, want %q", err, item.want) + } + }) + } +} + +// visualAdmissionRun builds a complete reference-environment run directory +// with synthetic captures of the admitted dimensions, its observed +// environment and a candidate inventory recomputed by the digest owner. +func visualAdmissionRun(t *testing.T, root string) string { + t.Helper() + if _, err := exec.LookPath("node"); err != nil { + t.Skip("reference admission recomputes digests with node") + } + run := t.TempDir() + for index, reference := range expectedVisualReferences() { + width, height := reference.Width, 3 + if reference.isReadmeScene() { + height = readmeSceneHeight + } + picture := image.NewNRGBA(image.Rect(0, 0, width, height)) + picture.Set(index%width, 0, color.NRGBA{R: uint8(index), G: 7, B: 9, A: 255}) + target := filepath.Join(run, visualCaptureRunDirectory, filepath.FromSlash(reference.id()+".png")) + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + t.Fatal(err) + } + var encoded bytes.Buffer + if err := png.Encode(&encoded, picture); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target, encoded.Bytes(), 0o644); err != nil { + t.Fatal(err) + } + } + environment, err := loadVisualEnvironment(root) + if err != nil { + t.Fatal(err) + } + observed := map[string]any{ + "cpu": map[string]any{"avx2": nil, "avx512f": nil, "features": "fp asimd", "hypervisor": nil, "model": "part 0xd49", "vendor": "implementer 0x41"}, + "divergentFields": []any{}, + } + for key, value := range environment.Observed { + observed[key] = value + } + writeRunJSON(t, filepath.Join(run, visualObservedRunFile), observed) + recomputed, err := recomputeCaptureDigests(root, filepath.Join(run, visualCaptureRunDirectory)) + if err != nil { + t.Fatal(err) + } + references := []any{} + for _, entry := range recomputed { + references = append(references, map[string]any{"height": entry.Height, "id": entry.ID, "rgbaSHA256": entry.RGBASHA256, "width": entry.Width}) + } + writeRunJSON(t, filepath.Join(run, visualCandidateRunFile), map[string]any{"clipHeight": visualClipHeight, "environmentId": environment.EnvironmentID, "kind": "proofkit.browser-visual-references", "references": references, "schemaVersion": 1}) + return run +} + +func writeRunJSON(t *testing.T, path string, value any) { + t.Helper() + encoded, err := json.MarshalIndent(value, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, encoded, 0o644); err != nil { + t.Fatal(err) + } +} + +// Reference admission accepts only a complete, stable reference-environment +// run, recomputes every digest from the PNG files and never trusts the run's +// JSON for a digest. +func TestVisualReferenceAdmissionFromRunRefusesEveryUnadmissibleRun(t *testing.T) { + root, err := repositoryRoot() + if err != nil { + t.Fatal(err) + } + run := visualAdmissionRun(t, root) + encoded, err := buildVisualReferenceInventoryFromRun(root, run) + if err != nil { + t.Fatalf("complete reference-environment run was refused: %v", err) + } + admitted := copyVisualRecords(t) + if err := os.WriteFile(filepath.Join(admitted, filepath.FromSlash(visualReferencesPath)), encoded, 0o644); err != nil { + t.Fatal(err) + } + if inventory, err := verifyVisualReferences(admitted); err != nil || inventory.Count != 169 { + t.Fatalf("admitted inventory = %+v, %v", inventory, err) + } + captureOf := func(run, id string) string { + return filepath.Join(run, visualCaptureRunDirectory, filepath.FromSlash(id+".png")) + } + cases := []struct { + name string + want string + mutate func(t *testing.T, run string) + }{ + {"divergent observed environment", "observed another environment", func(t *testing.T, run string) { + editJSONFile(t, filepath.Join(run, visualObservedRunFile), func(record map[string]any) { + record["architecture"] = "x86_64" + record["divergentFields"] = []any{"architecture"} + }) + }}, + {"claimed divergence", "lists divergent fields although every compared field equals the recorded environment", func(t *testing.T, run string) { + editJSONFile(t, filepath.Join(run, visualObservedRunFile), func(record map[string]any) { + record["divergentFields"] = []any{"fontConfig"} + }) + }}, + {"missing observed environment", "observed environment", func(t *testing.T, run string) { + if err := os.Remove(filepath.Join(run, visualObservedRunFile)); err != nil { + t.Fatal(err) + } + }}, + {"missing capture", "missing 1 of 169 captures", func(t *testing.T, run string) { + if err := os.Remove(captureOf(run, "webkit/workspace-diff.dark.320")); err != nil { + t.Fatal(err) + } + }}, + {"extra capture", "unexpected file", func(t *testing.T, run string) { + content, err := os.ReadFile(captureOf(run, "webkit/workspace-diff.dark.320")) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(run, visualCaptureRunDirectory, "webkit", "extra.dark.320.png"), content, 0o644); err != nil { + t.Fatal(err) + } + }}, + {"unstable capture", "unstable captures", func(t *testing.T, run string) { + if err := os.MkdirAll(filepath.Join(run, visualUnstableRunDirectory), 0o755); err != nil { + t.Fatal(err) + } + }}, + {"corrupted PNG", "recompute visual capture digests", func(t *testing.T, run string) { + target := captureOf(run, "firefox/static-coverage-table.dark.1440") + content, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + content[len(content)/2] ^= 0xff + if err := os.WriteFile(target, content, 0o644); err != nil { + t.Fatal(err) + } + }}, + {"replaced PNG", "no longer matches", func(t *testing.T, run string) { + if err := copyFile(captureOf(run, "firefox/static-coverage-table.light.1440"), captureOf(run, "firefox/static-coverage-table.dark.1440")); err != nil { + t.Fatal(err) + } + }}, + {"capture with another width", "1440 CSS pixels wide", func(t *testing.T, run string) { + if err := copyFile(captureOf(run, "firefox/static-coverage-table.light.320"), captureOf(run, "firefox/static-coverage-table.dark.1440")); err != nil { + t.Fatal(err) + } + }}, + {"symlinked capture", "symlink", func(t *testing.T, run string) { + target := captureOf(run, "chromium/workspace-coverage.light.320") + if err := os.Remove(target); err != nil { + t.Fatal(err) + } + if err := os.Symlink(captureOf(run, "chromium/workspace-coverage.dark.320"), target); err != nil { + t.Fatal(err) + } + }}, + } + for _, item := range cases { + t.Run(item.name, func(t *testing.T) { + mutated := t.TempDir() + if err := os.CopyFS(mutated, os.DirFS(run)); err != nil { + t.Fatal(err) + } + item.mutate(t, mutated) + if _, err := buildVisualReferenceInventoryFromRun(root, mutated); err == nil || !strings.Contains(err.Error(), item.want) { + t.Fatalf("admission error = %v, want %q", err, item.want) + } + }) + } +} + +func editJSONFile(t *testing.T, target string, mutate func(map[string]any)) { + t.Helper() + raw, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + value, err := admission.DecodeJSON(bytes.NewReader(raw), int64(len(raw))) + if err != nil { + t.Fatal(err) + } + record := value.(map[string]any) + mutate(record) + writeRunJSON(t, target, record) +} + +func copyFile(source, target string) error { + content, err := os.ReadFile(source) + if err != nil { + return err + } + return os.WriteFile(target, content, 0o644) +} + +func TestVisualEnvironmentBindsThePinnedPlaywrightToolchain(t *testing.T) { + root, err := repositoryRoot() + if err != nil { + t.Fatal(err) + } + environment, err := loadVisualEnvironment(root) + if err != nil { + t.Fatal(err) + } + lockRaw, err := os.ReadFile(filepath.Join(root, "package-lock.json")) + if err != nil { + t.Fatal(err) + } + var lock struct { + Packages map[string]struct { + Version string `json:"version"` + } `json:"packages"` + } + if err := json.Unmarshal(lockRaw, &lock); err != nil { + t.Fatal(err) + } + version := lock.Packages["node_modules/@playwright/test"].Version + if version == "" || lock.Packages["node_modules/playwright-core"].Version != version { + t.Fatalf("package-lock.json must pin @playwright/test and playwright-core to one version") + } + image := environment.Raw["image"].(map[string]any) + if image["tag"] != "v"+version+"-noble" || image["platform"] != "linux/arm64" { + t.Fatalf("reference image tag/platform = %v/%v, want v%s-noble on linux/arm64", image["tag"], image["platform"], version) + } + docker := environment.Observed["dockerImage"].(map[string]any) + if docker["dockerImageName"] != image["name"].(string)+":"+image["tag"].(string) || docker["driverVersion"] != version { + t.Fatalf("observed image marker %v does not name the recorded image", docker) + } + // The engine revisions and launched versions equal the pinned registry. + browsersRaw, err := os.ReadFile(filepath.Join(root, "node_modules", "playwright-core", "browsers.json")) + if err != nil { + t.Fatalf("read the installed playwright-core browsers.json (run npm ci first): %v", err) + } + var browsers struct { + Browsers []struct { + Name string `json:"name"` + Revision string `json:"revision"` + BrowserVersion string `json:"browserVersion"` + } `json:"browsers"` + } + if err := json.Unmarshal(browsersRaw, &browsers); err != nil { + t.Fatal(err) + } + // The compared font environment is the committed fontconfig file. + fontConfig, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(visualFontConfigPath))) + if err != nil { + t.Fatal(err) + } + fontSum := sha256.Sum256(fontConfig) + if environment.Observed["fontConfig"] != "sha256:"+hex.EncodeToString(fontSum[:]) { + t.Fatalf("environment fontConfig=%v does not bind %s", environment.Observed["fontConfig"], visualFontConfigPath) + } + // Each recorded fc-match result is the match of its whole family name. + fontMatches, _ := environment.Observed["fontMatches"].(map[string]any) + wantMatches := map[string]string{"sans-serif": "Liberation Sans", "system-ui": "Liberation Sans", "serif": "Liberation Serif", "monospace": "Liberation Mono", "ui-monospace": "Liberation Mono"} + if len(fontMatches) != len(wantMatches) { + t.Fatalf("recorded fc-match families = %v, want %v", fontMatches, wantMatches) + } + for family, face := range wantMatches { + match, _ := fontMatches[family].(string) + if !strings.Contains(match, "\""+face+"\"") { + t.Fatalf("recorded fc-match for %s is %q, want %s", family, match, face) + } + } + engines := environment.Observed["engines"].(map[string]any) + for _, name := range visualEngines { + found := false + for _, browser := range browsers.Browsers { + if browser.Name == name { + found = true + if engines[name] != browser.Revision || environment.BrowserVersions[name] != browser.BrowserVersion { + t.Fatalf("%s revision/version = %v/%s, want browsers.json %s/%s", name, engines[name], environment.BrowserVersions[name], browser.Revision, browser.BrowserVersion) + } + } + } + if !found { + t.Fatalf("browsers.json has no %s engine", name) + } + } +} + +func TestVisualReferencesStayOutsideThePackage(t *testing.T) { + root, err := repositoryRoot() + if err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(filepath.Join(root, "package.json")) + if err != nil { + t.Fatal(err) + } + var manifest struct { + Files []string `json:"files"` + } + if err := json.Unmarshal(raw, &manifest); err != nil { + t.Fatal(err) + } + candidates := []string{} + err = filepath.WalkDir(filepath.Join(root, "tests", "visual"), func(current string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil || entry.IsDir() { + return walkErr + } + relative, err := filepath.Rel(root, current) + candidates = append(candidates, filepath.ToSlash(relative)) + return err + }) + if err != nil || !slices.Contains(candidates, visualReferencesPath) || !slices.Contains(candidates, visualFontConfigPath) { + t.Fatalf("visual test files = %v, %v", candidates, err) + } + for _, pattern := range manifest.Files { + for _, candidate := range candidates { + prefix := strings.TrimSuffix(strings.TrimSuffix(pattern, "/**/*"), "/**") + matched, err := path.Match(pattern, candidate) + if err != nil { + t.Fatal(err) + } + if matched || pattern == candidate || strings.HasPrefix(candidate, prefix+"/") { + t.Fatalf("package files pattern %q includes visual gate material %s", pattern, candidate) + } + } + } +} + +func TestBrowserVisualScriptIsSeparateFromThePackageGate(t *testing.T) { + root, err := repositoryRoot() + if err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(filepath.Join(root, "package.json")) + if err != nil { + t.Fatal(err) + } + var manifest struct { + Scripts map[string]string `json:"scripts"` + } + if err := json.Unmarshal(raw, &manifest); err != nil { + t.Fatal(err) + } + if manifest.Scripts["browser:visual"] != "go run ./internal/tools/browserproofverify --manifest scripts/browser-visual-proof-inputs.v1.json --run" { + t.Fatalf("browser:visual must delegate to the single Go orchestrator with the visual manifest, got %q", manifest.Scripts["browser:visual"]) + } + for name, script := range manifest.Scripts { + if name != "browser:visual" && strings.Contains(script, "browser:visual") || strings.Contains(script, visualProofManifestPath) && name != "browser:visual" { + t.Fatalf("script %q composes the visual gate; it must stay outside npm run check", name) + } + } + for _, hook := range []string{"prebrowser:visual", "postbrowser:visual"} { + if _, exists := manifest.Scripts[hook]; exists { + t.Fatalf("browser:visual must not have lifecycle hook %q", hook) + } + } +} diff --git a/internal/tools/browserproofverify/visual_record.go b/internal/tools/browserproofverify/visual_record.go new file mode 100644 index 00000000..f98ca464 --- /dev/null +++ b/internal/tools/browserproofverify/visual_record.go @@ -0,0 +1,181 @@ +package main + +import ( + "fmt" + "slices" + + "github.com/research-engineering/agentic-proofkit/internal/kernel/admit" + "github.com/research-engineering/agentic-proofkit/internal/kernel/secretjson" +) + +var visualProofNonClaims = []string{ + "A passed record proves only that every listed reference view produced, in two byte-equal captures, its admitted canonical RGBA digest and dimensions in the recorded reference environment before record creation.", + "Only the first 2400 CSS pixels of each view are compared; content below that height is not covered by a reference.", + "An unverified record makes no pixel claim; it lists the compared environment fields that differ from the recorded reference environment.", + "The CPU vendor, model and features are recorded but are not part of the compared environment identity; pixel identity is calibrated only on Apple M1 (implementer 0x61) and Arm Neoverse N2 (implementer 0x41, part 0xd49).", + "Admitted images are not stored in the repository; they are reproducible from the admitting commit in the recorded environment on a calibrated CPU model while the pinned image and the pinned toolchain and dependency downloads remain available, and an uploaded capture gallery is derived, retention-limited evidence.", + "This record does not identify who reviewed or admitted a reference.", + "Playwright WebKit is not branded Safari compatibility proof.", + "A dirty-tree record is content-bound only to its listed assets; sourceRevision is contextual.", + "This record does not independently authenticate or sandbox the same-user proof runner, the container image beyond its observed tuple, external toolchain packages, or browser binaries.", +} + +var visualCommandArgv = []string{"node_modules/@playwright/test/cli.js", "test", "--config", "playwright.visual.config.mjs"} + +// verifyVisualRecord admits a passed or unverified visual record. It binds the +// record to the content-bound inputs, the committed reference inventory and +// environment identity, and recomputes the environment divergence itself; a +// passed record must also carry every engine's exact expected test set. +func verifyVisualRecord(root string, raw any, expectedResolution proofInputResolution, revision, treeState string) error { + record, ok := raw.(map[string]any) + if !ok { + return fmt.Errorf("browser visual proof must be an object") + } + if err := admit.KnownKeys(record, []string{"assets", "command", "divergentFields", "engines", "inputDigest", "inputResolution", "nonClaims", "observedEnvironment", "projects", "proofKind", "referenceEnvironment", "schemaVersion", "sourceRevision", "sourceTreeState", "state"}, "browser visual proof"); err != nil { + return err + } + kind := visualProofKind() + state, _ := record["state"].(string) + if !admit.JSONNumberEquals(record["schemaVersion"], 1) || record["proofKind"] != kind.RecordKind || !kind.admitsState(state) || + record["sourceRevision"] != revision || record["sourceTreeState"] != treeState { + return fmt.Errorf("browser visual proof identity or source state is invalid") + } + if err := exactTextArray(record["nonClaims"], visualProofNonClaims, "browser visual proof nonClaims"); err != nil { + return err + } + if err := verifyContentBoundInputs(root, record, expectedResolution, "browser visual proof"); err != nil { + return err + } + inventory, err := verifyVisualReferences(root) + if err != nil { + return err + } + reference, ok := record["referenceEnvironment"].(map[string]any) + if !ok || admit.KnownKeys(reference, []string{"environmentId", "referenceCount", "referencesDigest"}, "browser visual proof referenceEnvironment") != nil || + reference["environmentId"] != inventory.EnvironmentID || reference["referencesDigest"] != inventory.ReferencesDigest || + !admit.JSONNumberEquals(reference["referenceCount"], int64(inventory.Count)) { + return fmt.Errorf("browser visual proof is not bound to the committed reference inventory") + } + environment, err := loadVisualEnvironment(root) + if err != nil { + return err + } + observed, ok := record["observedEnvironment"].(map[string]any) + if !ok || admit.KnownKeys(observed, append([]string{"cpu"}, visualComparedEnvironmentFields...), "browser visual proof observedEnvironment") != nil || + len(observed) != len(visualComparedEnvironmentFields)+1 { + return fmt.Errorf("browser visual proof must record the complete observed environment") + } + if err := admitObservedTuple(observed, false); err != nil { + return err + } + if err := admitObservedCPU(observed["cpu"]); err != nil { + return err + } + divergent, err := visualEnvironmentDivergence(environment.Observed, observed) + if err != nil { + return err + } + if err := exactTextArray(record["divergentFields"], divergent, "browser visual proof divergentFields"); err != nil { + return err + } + if (state == "passed") != (len(divergent) == 0) { + return fmt.Errorf("browser visual proof state must be passed exactly when the observed environment equals the reference environment") + } + if state == "unverified" { + if record["command"] != nil || !emptyArray(record["engines"]) || !emptyArray(record["projects"]) { + return fmt.Errorf("unverified browser visual proof must not claim a browser execution") + } + } else { + if err := verifyVisualExecution(record, environment); err != nil { + return err + } + } + findings, err := secretjson.Scan(record, "browser_visual_proof") + if err != nil || len(findings) > 0 { + return fmt.Errorf("browser visual proof contains secret-shaped data") + } + return nil +} + +func verifyVisualExecution(record map[string]any, environment visualEnvironment) error { + command, ok := record["command"].(map[string]any) + if !ok || admit.KnownKeys(command, []string{"argv", "exitCode", "inputMode", "runner"}, "browser visual proof command") != nil || + command["inputMode"] != "materialized_snapshot" || command["runner"] != "node" || !admit.JSONNumberEquals(command["exitCode"], 0) { + return fmt.Errorf("browser visual proof command is invalid") + } + if err := exactTextArray(command["argv"], visualCommandArgv, "browser visual proof command argv"); err != nil { + return err + } + engines, ok := record["engines"].([]any) + if !ok || len(engines) != len(visualEngines) { + return fmt.Errorf("browser visual proof must record three engines") + } + for index, name := range visualEngines { + engine, ok := engines[index].(map[string]any) + if !ok || admit.KnownKeys(engine, []string{"name", "version"}, "browser visual proof engine") != nil || engine["name"] != name || + engine["version"] != environment.BrowserVersions[name] { + return fmt.Errorf("browser visual proof engine versions must equal the reference environment") + } + } + projects, ok := record["projects"].([]any) + if !ok || len(projects) != len(visualEngines) { + return fmt.Errorf("browser visual proof must record three project executions") + } + expected := expectedVisualProjectTests() + for index, name := range visualEngines { + project, ok := projects[index].(map[string]any) + if !ok || admit.KnownKeys(project, []string{"browserName", "browserVersion", "executedTestCount", "name", "passedTestCount", "testIds"}, "browser visual proof project") != nil || + project["name"] != name || project["browserName"] != name || project["browserVersion"] != environment.BrowserVersions[name] { + return fmt.Errorf("browser visual proof project identity is invalid") + } + testIDs, err := sortedUniqueTextArray(project["testIds"], "browser visual proof testIds") + if err != nil { + return err + } + if !slices.Equal(testIDs, expected[name]) || !admit.JSONNumberEquals(project["executedTestCount"], int64(len(testIDs))) || + !admit.JSONNumberEquals(project["passedTestCount"], int64(len(testIDs))) { + return fmt.Errorf("browser visual proof project must pass exactly its expected reference comparisons and controls") + } + } + return nil +} + +// admitObservedCPU admits the recorded CPU: x86 instruction-set and +// hypervisor flags are booleans where observable and null otherwise; vendor, +// model and the cpuinfo features line are text or null. +func admitObservedCPU(raw any) error { + cpu, ok := raw.(map[string]any) + if !ok || admit.KnownKeys(cpu, []string{"avx2", "avx512f", "features", "hypervisor", "model", "vendor"}, "browser visual proof observed cpu") != nil || len(cpu) != 6 { + return fmt.Errorf("browser visual proof must record the observed CPU") + } + for _, flag := range []string{"avx2", "avx512f", "hypervisor"} { + if _, isBool := cpu[flag].(bool); cpu[flag] != nil && !isBool { + return fmt.Errorf("browser visual proof observed CPU flags must be booleans or null") + } + } + for _, field := range []string{"features", "model", "vendor"} { + if cpu[field] != nil { + if _, err := admit.NonEmptyText(cpu[field], "browser visual proof observed cpu "+field); err != nil { + return err + } + } + } + return nil +} + +func emptyArray(raw any) bool { + values, ok := raw.([]any) + return ok && len(values) == 0 +} + +// visualDivergentFields reads the admitted divergence list of a visual record. +func visualDivergentFields(record map[string]any) []string { + values, _ := record["divergentFields"].([]any) + fields := make([]string, 0, len(values)) + for _, value := range values { + if text, ok := value.(string); ok { + fields = append(fields, text) + } + } + return fields +} diff --git a/internal/tools/browserproofverify/visual_record_test.go b/internal/tools/browserproofverify/visual_record_test.go new file mode 100644 index 00000000..4bbf7379 --- /dev/null +++ b/internal/tools/browserproofverify/visual_record_test.go @@ -0,0 +1,309 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/research-engineering/agentic-proofkit/internal/kernel/digest" + "github.com/research-engineering/agentic-proofkit/internal/kernel/stablejson" +) + +// visualRecordFixture builds the admitted record a writer would produce for +// root in the given state, recomputing every content-bound field. +func visualRecordFixture(t *testing.T, root string, resolution proofInputResolution, state string) map[string]any { + t.Helper() + assets := make([]any, 0, len(resolution.InputPaths)) + for _, path := range resolution.InputPaths { + content, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(path))) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(content) + assets = append(assets, map[string]any{"path": path, "sha256": hex.EncodeToString(sum[:])}) + } + inputResolution := map[string]any{"serverTarget": resolution.ServerTarget, "writerPath": resolution.WriterPath} + encoded, err := stablejson.MarshalLayout(map[string]any{"assets": assets, "inputResolution": inputResolution}, stablejson.LayoutCompact) + if err != nil { + t.Fatal(err) + } + inventory, err := verifyVisualReferences(root) + if err != nil { + t.Fatal(err) + } + environment, err := loadVisualEnvironment(root) + if err != nil { + t.Fatal(err) + } + observed := map[string]any{"cpu": map[string]any{"avx2": nil, "avx512f": nil, "features": "fp asimd sve sve2", "hypervisor": nil, "model": "part 0xd49", "vendor": "implementer 0x41"}} + for key, value := range environment.Observed { + observed[key] = value + } + nonClaims := make([]any, len(visualProofNonClaims)) + for index, nonClaim := range visualProofNonClaims { + nonClaims[index] = nonClaim + } + record := map[string]any{ + "assets": assets, "command": nil, "divergentFields": []any{}, "engines": []any{}, + "inputDigest": digest.SHA256TextRef(strings.TrimSuffix(string(encoded), "\n")), "inputResolution": inputResolution, + "nonClaims": nonClaims, "observedEnvironment": observed, "projects": []any{}, + "proofKind": "proofkit.browser-visual-proof", + "referenceEnvironment": map[string]any{ + "environmentId": inventory.EnvironmentID, "referenceCount": json.Number("169"), "referencesDigest": inventory.ReferencesDigest, + }, + "schemaVersion": json.Number("1"), "sourceRevision": "abc123", "sourceTreeState": "dirty", "state": state, + } + if state == "unverified" { + observed["architecture"] = "x86_64" + observed["packages"] = nil + record["divergentFields"] = []any{"architecture", "packages"} + return record + } + argv := make([]any, len(visualCommandArgv)) + for index, value := range visualCommandArgv { + argv[index] = value + } + record["command"] = map[string]any{"argv": argv, "exitCode": json.Number("0"), "inputMode": "materialized_snapshot", "runner": "node"} + engines, projects := []any{}, []any{} + expected := expectedVisualProjectTests() + for _, name := range visualEngines { + ids := make([]any, len(expected[name])) + for index, id := range expected[name] { + ids[index] = id + } + count := json.Number(fmtInt(len(ids))) + engines = append(engines, map[string]any{"name": name, "version": environment.BrowserVersions[name]}) + projects = append(projects, map[string]any{"browserName": name, "browserVersion": environment.BrowserVersions[name], "executedTestCount": count, "name": name, "passedTestCount": count, "testIds": ids}) + } + record["engines"], record["projects"] = engines, projects + return record +} + +func fmtInt(value int) string { + encoded, _ := json.Marshal(value) + return string(encoded) +} + +func repositoryVisualResolution(t *testing.T) (string, proofInputResolution) { + t.Helper() + root, err := repositoryRoot() + if err != nil { + t.Fatal(err) + } + manifest, err := loadProofInputManifest(root, visualProofKind()) + if err != nil { + t.Fatal(err) + } + resolution, err := resolveProofInputs(root, manifest, visualProofKind()) + if err != nil { + t.Fatal(err) + } + return root, resolution +} + +func TestVisualRecordAdmitsOnlyConsistentPassedOrUnverifiedStates(t *testing.T) { + root, resolution := repositoryVisualResolution(t) + for _, state := range []string{"passed", "unverified"} { + if err := verifyVisualRecord(root, visualRecordFixture(t, root, resolution, state), resolution, "abc123", "dirty"); err != nil { + t.Fatalf("%s visual record was refused: %v", state, err) + } + } + emulated := visualRecordFixture(t, root, resolution, "unverified") + emulated["observedEnvironment"].(map[string]any)["architecture"] = "aarch64" + emulated["observedEnvironment"].(map[string]any)["packages"] = visualRecordFixture(t, root, resolution, "passed")["observedEnvironment"].(map[string]any)["packages"] + emulated["observedEnvironment"].(map[string]any)["emulation"] = "translator-mapped" + emulated["divergentFields"] = []any{"emulation"} + if err := verifyVisualRecord(root, emulated, resolution, "abc123", "dirty"); err != nil { + t.Fatalf("an emulated host was not admitted as unverified: %v", err) + } + cases := []struct { + name string + state string + mutate func(map[string]any) + }{ + {"passed with a divergent environment", "passed", func(record map[string]any) { + record["observedEnvironment"].(map[string]any)["emulation"] = "translator-mapped" + }}, + {"passed claiming a divergence", "passed", func(record map[string]any) { record["divergentFields"] = []any{"architecture"} }}, + {"unverified without a divergence", "passed", func(record map[string]any) { record["state"] = "unverified" }}, + {"unverified omitting a divergent field", "unverified", func(record map[string]any) { record["divergentFields"] = []any{"architecture"} }}, + {"unverified claiming an execution", "unverified", func(record map[string]any) { + record["engines"] = []any{map[string]any{"name": "chromium", "version": "1"}} + }}, + {"failed state", "passed", func(record map[string]any) { record["state"] = "failed" }}, + {"schema version", "passed", func(record map[string]any) { record["schemaVersion"] = json.Number("2") }}, + {"missing reference comparison", "passed", func(record map[string]any) { + project := record["projects"].([]any)[1].(map[string]any) + ids := project["testIds"].([]any) + project["testIds"] = ids[:len(ids)-1] + project["executedTestCount"], project["passedTestCount"] = json.Number(fmtInt(len(ids)-1)), json.Number(fmtInt(len(ids)-1)) + }}, + {"other engine version", "passed", func(record map[string]any) { + record["projects"].([]any)[2].(map[string]any)["browserVersion"] = "0" + }}, + {"command without the visual config", "passed", func(record map[string]any) { + record["command"].(map[string]any)["argv"] = []any{"node_modules/@playwright/test/cli.js", "test"} + }}, + {"failed command", "passed", func(record map[string]any) { + record["command"].(map[string]any)["exitCode"] = json.Number("1") + }}, + {"other environment identity", "passed", func(record map[string]any) { + record["referenceEnvironment"].(map[string]any)["environmentId"] = "sha256:" + strings.Repeat("0", 64) + }}, + {"other reference inventory", "unverified", func(record map[string]any) { + record["referenceEnvironment"].(map[string]any)["referencesDigest"] = "sha256:" + strings.Repeat("0", 64) + }}, + {"observed CPU omitted", "passed", func(record map[string]any) { delete(record["observedEnvironment"].(map[string]any), "cpu") }}, + {"weaker non-claim", "unverified", func(record map[string]any) { record["nonClaims"].([]any)[0] = "weaker" }}, + {"stale input digest", "unverified", func(record map[string]any) { + record["inputDigest"] = "sha256:" + strings.Repeat("0", 64) + }}, + } + for _, item := range cases { + t.Run(item.name, func(t *testing.T) { + record := visualRecordFixture(t, root, resolution, item.state) + item.mutate(record) + if err := verifyVisualRecord(root, record, resolution, "abc123", "dirty"); err == nil { + t.Fatalf("visual record mutant %q was admitted", item.name) + } + }) + } +} + +// visualOperationFixture is a disposable repository with the committed visual +// records and references, a stub server package and a writer that publishes a +// staged candidate where the orchestrator expects it. +func visualOperationFixture(t *testing.T) string { + t.Helper() + root := browserOperationFixture(t) + source, err := repositoryRoot() + if err != nil { + t.Fatal(err) + } + err = filepath.WalkDir(filepath.Join(source, "tests", "visual"), func(current string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + relative, err := filepath.Rel(source, current) + if err != nil { + return err + } + if entry.IsDir() { + return os.MkdirAll(filepath.Join(root, relative), 0o755) + } + content, err := os.ReadFile(current) + if err != nil { + return err + } + return os.WriteFile(filepath.Join(root, relative), content, 0o644) + }) + if err != nil { + t.Fatal(err) + } + manifest := `{"schemaVersion":1,"proofKind":"proofkit.browser-visual-proof-inputs","serverTarget":"./internal/tools/browserfixture","testRoot":"tests/visual","writerPath":"scripts/writer.mjs","paths":["go.mod"]}` + if err := os.WriteFile(filepath.Join(root, filepath.FromSlash(visualProofManifestPath)), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + plan, err := json.Marshal(map[string]any{"policies": []any{ + map[string]any{"commandId": "proofkit.browser-check", "inputSelectors": fixtureSelectors}, + map[string]any{"commandId": "proofkit.browser-visual", "inputSelectors": []any{ + "go.mod", "internal/tools/browserfixture", "internal/tools/browserproofverify", + "scripts/browser-visual-proof-inputs.v1.json", "scripts/writer.mjs", "tests/visual", + }}, + }}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "proofkit", "witness-plan.json"), plan, 0o644); err != nil { + t.Fatal(err) + } + for _, args := range [][]string{{"add", "-A"}, {"-c", "user.name=fixture", "-c", "user.email=fixture@example.test", "commit", "-q", "-m", "visual fixture"}} { + command := exec.Command("git", args...) + command.Dir = root + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, output) + } + } + // A deterministic dirty tree, independent of run-directory timing. + if err := os.WriteFile(filepath.Join(root, "untracked-visual.txt"), []byte("dirty\n"), 0o644); err != nil { + t.Fatal(err) + } + return root +} + +func stageVisualCandidate(t *testing.T, root, state string, mutate func(map[string]any)) { + t.Helper() + manifest, err := loadProofInputManifest(root, visualProofKind()) + if err != nil { + t.Fatal(err) + } + resolution, err := resolveProofInputs(root, manifest, visualProofKind()) + if err != nil { + t.Fatal(err) + } + revision, treeState, err := currentSourceIdentity(root) + if err != nil { + t.Fatal(err) + } + record := visualRecordFixture(t, root, resolution, state) + record["sourceRevision"], record["sourceTreeState"] = revision, treeState + if mutate != nil { + mutate(record) + } + encoded, err := json.Marshal(record) + if err != nil { + t.Fatal(err) + } + candidate := filepath.Join(t.TempDir(), "candidate.json") + if err := os.WriteFile(candidate, encoded, 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("PROOFKIT_OPERATION_CANDIDATE", candidate) +} + +func observeVisualRun(t *testing.T, root string, requireReference bool) (bool, error) { + t.Helper() + t.Setenv("PROOFKIT_OPERATION_MARKERS", t.TempDir()) + recordPath := filepath.Join(root, filepath.FromSlash(visualProofKind().RecordPath)) + _ = os.Remove(recordPath) + err := runBrowserProof(root, visualProofKind(), requireReference) + _, statErr := os.Stat(recordPath) + return statErr == nil, err +} + +// The visual orchestrator writes a record only for an admitted candidate; +// --require-reference-environment turns an unverified record into a failure. +func TestVisualProofRunWritesNoRecordOnFailureOrRequiredUnverified(t *testing.T) { + root := visualOperationFixture(t) + stageVisualCandidate(t, root, "unverified", nil) + if written, err := observeVisualRun(t, root, false); err != nil || !written { + t.Fatalf("local unverified run: written=%v err=%v, want an unverified record and exit 0", written, err) + } + if written, err := observeVisualRun(t, root, true); err == nil || written || !strings.Contains(err.Error(), "unverified") { + t.Fatalf("required reference environment: written=%v err=%v, want failure without a record", written, err) + } + stageVisualCandidate(t, root, "passed", nil) + if written, err := observeVisualRun(t, root, true); err != nil || !written { + t.Fatalf("passed run: written=%v err=%v", written, err) + } + stageVisualCandidate(t, root, "passed", func(record map[string]any) { + record["projects"].([]any)[0].(map[string]any)["testIds"] = []any{"tests/visual/references.spec.mjs::only"} + }) + if written, err := observeVisualRun(t, root, false); err == nil || written { + t.Fatalf("refused candidate: written=%v err=%v, want failure without a record", written, err) + } + stageVisualCandidate(t, root, "unverified", nil) + editJSON(t, root, visualReferencesPath, func(record map[string]any) { + entries := record["references"].([]any) + record["references"] = entries[1:] + }) + if written, err := observeVisualRun(t, root, false); err == nil || written { + t.Fatalf("missing reference ID: written=%v err=%v, want failure without a record", written, err) + } +} diff --git a/internal/tools/browsertestserver/main.go b/internal/tools/browsertestserver/main.go index fa5f6766..51821e36 100644 --- a/internal/tools/browsertestserver/main.go +++ b/internal/tools/browsertestserver/main.go @@ -15,6 +15,11 @@ import ( func main() { build := browserfixture.Workspace view := "workspace" + var localEnvironmentClasses []string + if len(os.Args) == 2 && os.Args[1] == "--readme-project" { + serveReadmeProject() + return + } if len(os.Args) == 2 && os.Args[1] == "--lookup" { build = browserfixture.LookupWorkspace } else if len(os.Args) == 2 && os.Args[1] == "--paging" { @@ -50,6 +55,10 @@ func main() { } else if len(os.Args) == 2 && os.Args[1] == "--static-proof" { build = browserfixture.StaticProof view = "proof" + } else if len(os.Args) == 2 && os.Args[1] == "--static-proof-compact" { + build = browserfixture.StaticCompactProof + view = "proof" + localEnvironmentClasses = []string{"local-go"} } else if len(os.Args) == 2 && os.Args[1] == "--static-coverage" { build = func() (map[string]any, error) { return browserfixture.CoverageInput("structured") } view = "coverage" @@ -62,11 +71,40 @@ func main() { } ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer cancel() - if err := requirementbrowser.Serve(ctx, workspace, requirementbrowser.Options{Host: "127.0.0.1", Port: 0, PortSet: true, SessionMode: "browse", View: view}, os.Stdout); err != nil && err != context.Canceled { + if err := requirementbrowser.Serve(ctx, workspace, requirementbrowser.Options{Host: "127.0.0.1", Port: 0, PortSet: true, SessionMode: "browse", View: view, LocalEnvironmentClasses: localEnvironmentClasses}, os.Stdout); err != nil && err != context.Canceled { fatal(err) } } +// serveReadmeProject materializes the committed README project and serves it +// through the same project inspection path as the view command. The temporary +// project directory is removed on every exit path before a failure exits. +func serveReadmeProject() { + directory, err := os.MkdirTemp("", "proofkit-readme-project-") + if err != nil { + fatal(err) + } + err = serveMaterializedReadmeProject(directory) + if removeErr := os.RemoveAll(directory); err == nil { + err = removeErr + } + if err != nil { + fatal(err) + } +} + +func serveMaterializedReadmeProject(directory string) error { + if err := browserfixture.MaterializeReadmeProject(directory); err != nil { + return err + } + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + if err := requirementbrowser.ServeProject(ctx, directory, requirementbrowser.Options{Host: "127.0.0.1", Port: 0, PortSet: true, SessionMode: "browse"}, os.Stdout); err != nil && err != context.Canceled { + return err + } + return nil +} + func fatal(err error) { diagnostic.WriteError(os.Stderr, err) os.Exit(1) diff --git a/internal/tools/coveragemetrics/required_inventory.go b/internal/tools/coveragemetrics/required_inventory.go index 9d78d1c7..46429df1 100644 --- a/internal/tools/coveragemetrics/required_inventory.go +++ b/internal/tools/coveragemetrics/required_inventory.go @@ -1254,6 +1254,60 @@ func requiredBindingWitnessInventory() map[inventoryKey]requiredInventoryEntry { "TestWorkflowExternalActionsUseFullCommitSHAs", }, }, + {"REQ-PROOFKIT-QUALITY-025", "proofkit.supply-chain-quality.workflow-container-digest-pins"}: { + witnessPath: "scripts/workflow_browser_visual_oracle_test.go", + selectors: []string{"TestWorkflowContainerImagesArePinnedByDigest"}, + }, + {"REQ-PROOFKIT-QUALITY-028", "proofkit.supply-chain-quality.visual-reference-inventory"}: { + witnessPath: "internal/tools/browserproofverify/visual_inventory_test.go", + commandIDs: []string{"proofkit.go-test"}, environmentClasses: []string{"local-go"}, + selectors: []string{ + "TestBrowserVisualScriptIsSeparateFromThePackageGate", + "TestCommittedVisualReferencesAreDigestsBoundToTheRecordedEnvironment", + "TestVisualInventoryIsTheExactViewEngineSchemeWidthProduct", + "TestVisualReferenceInventoryRefusesEveryIntegrityMutant", + "TestVisualReferencesStayOutsideThePackage", + }, + }, + {"REQ-PROOFKIT-QUALITY-028", "proofkit.supply-chain-quality.visual-environment-toolchain-binding"}: { + witnessPath: "internal/tools/browserproofverify/visual_inventory_test.go", + commandIDs: []string{"proofkit.go-test"}, environmentClasses: []string{"local-go"}, + selectors: []string{"TestVisualEnvironmentBindsThePinnedPlaywrightToolchain"}, + }, + {"REQ-PROOFKIT-QUALITY-028", "proofkit.supply-chain-quality.visual-proof-record"}: { + witnessPath: "internal/tools/browserproofverify/visual_record_test.go", + commandIDs: []string{"proofkit.go-test"}, environmentClasses: []string{"local-go"}, + selectors: []string{ + "TestVisualProofRunWritesNoRecordOnFailureOrRequiredUnverified", + "TestVisualRecordAdmitsOnlyConsistentPassedOrUnverifiedStates", + }, + }, + {"REQ-PROOFKIT-QUALITY-028", "proofkit.supply-chain-quality.visual-reference-admission"}: { + witnessPath: "internal/tools/browserproofverify/visual_inventory_test.go", + commandIDs: []string{"proofkit.go-test"}, environmentClasses: []string{"local-go"}, + selectors: []string{"TestVisualReferenceAdmissionFromRunRefusesEveryUnadmissibleRun"}, + }, + {"REQ-PROOFKIT-QUALITY-028", "proofkit.supply-chain-quality.visual-readme-project"}: { + witnessPath: "internal/testsupport/browserfixture/readme_test.go", + commandIDs: []string{"proofkit.go-test"}, environmentClasses: []string{"local-go"}, + selectors: []string{"TestReadmeProjectMaterializesAnAdmittedViewProject"}, + }, + {"REQ-PROOFKIT-QUALITY-028", "proofkit.supply-chain-quality.visual-ci-reference-environment"}: { + witnessPath: "scripts/workflow_browser_visual_oracle_test.go", + commandIDs: []string{"proofkit.go-test"}, environmentClasses: []string{"local-go"}, + selectors: []string{ + "TestCIBrowserVisualRejectsWeakenedReferenceGate", + "TestCIBrowserVisualRunsTheRecordedReferenceEnvironment", + }, + }, + {"REQ-PROOFKIT-QUALITY-028", "proofkit.supply-chain-quality.visual-exact-comparator"}: { + witnessPath: "scripts/browser-visual-references.test.mjs", + commandIDs: []string{"proofkit.browser-check"}, environmentClasses: []string{"local-node-browser"}, + }, + {"REQ-PROOFKIT-QUALITY-028", "proofkit.supply-chain-quality.visual-reference-gate"}: { + witnessPath: "tests/visual/references.spec.mjs", + commandIDs: []string{"proofkit.browser-visual"}, environmentClasses: []string{"container-linux-arm64-browser"}, + }, {"REQ-PROOFKIT-SPEC-011", "proofkit.spec-proof-core.adoption-contract-envelope-cli-abi"}: { witnessPath: "internal/app/cli_abi_test.go", selectors: []string{"TestAdoptionContractEnvelopeCLIABI"}, diff --git a/package.json b/package.json index 511169d1..060eb5f0 100644 --- a/package.json +++ b/package.json @@ -37,8 +37,9 @@ "build": "go run ./internal/tools/packagebuild", "check": "npm run npm:version && npm run source-hygiene && npm run command-contract:check && npm run command-family:check && npm run text-policy && npm run mermaid:check && npm run go:check && npm run compact-contract:check && npm run receipt-contract:check && npm run spec-tree-contract:check && npm run graph-contract:check && npm run boundary-contract:check && npm run browser:check && npm run package:artifact && npm run self:receipt && npm run self:coverage && npm run release:closeout", "browser:check": "npm run browser:static-check && npm run browser:test", - "browser:static-check": "tsc -p tsconfig.browser.json && node --test scripts/browser-evidence-models.test.mjs scripts/browser-proof-execution.test.mjs scripts/browser-proof-inputs.test.mjs scripts/browser-selection-authority.test.mjs scripts/browser-workspace-requests.test.mjs scripts/diagnostic-entrypoints.test.mjs scripts/stable-json.test.mjs", + "browser:static-check": "tsc -p tsconfig.browser.json && node --test scripts/browser-evidence-models.test.mjs scripts/browser-proof-execution.test.mjs scripts/browser-proof-inputs.test.mjs scripts/browser-selection-authority.test.mjs scripts/browser-visual-references.test.mjs scripts/browser-workspace-requests.test.mjs scripts/diagnostic-entrypoints.test.mjs scripts/stable-json.test.mjs", "browser:test": "go run ./internal/tools/browserproofverify --run", + "browser:visual": "go run ./internal/tools/browserproofverify --manifest scripts/browser-visual-proof-inputs.v1.json --run", "command-family:check": "go run ./internal/tools/commandfamilygen --check", "go:actionlint": "go tool actionlint", "go:bench": "go test ./internal/kernel/admission ./internal/kernel/stablejson -run '^$' -bench . -benchmem -count=10", diff --git a/playwright.visual.config.mjs b/playwright.visual.config.mjs new file mode 100644 index 00000000..c33cd084 --- /dev/null +++ b/playwright.visual.config.mjs @@ -0,0 +1,43 @@ +import {relative, resolve, sep} from "node:path"; +import {defineConfig, devices} from "@playwright/test"; + +const baseURL = process.env.PROOFKIT_BROWSER_TEST_URL; +if (!baseURL) throw new Error("PROOFKIT_BROWSER_TEST_URL is required"); +const outputDir = process.env.PROOFKIT_BROWSER_TEST_OUTPUT_DIR; +if (!outputDir) throw new Error("PROOFKIT_BROWSER_TEST_OUTPUT_DIR is required"); +const reportPath = process.env.PROOFKIT_BROWSER_TEST_REPORT_PATH; +if (!reportPath) throw new Error("PROOFKIT_BROWSER_TEST_REPORT_PATH is required"); +const testDir = "tests/visual"; +// Actual captures go only to the run output directory, never into the tests. +if (!relative(resolve(testDir), resolve(outputDir)).startsWith(`..${sep}`)) { + throw new Error("visual output directory must be outside the visual test directory"); +} + +// The README scene is a Chromium-only reference. +const chromiumOnly = ["readme-scene.spec.mjs"]; + +export default defineConfig({ + outputDir, + testDir, + fullyParallel: false, + forbidOnly: true, + retries: 0, + reporter: [["line"], ["json", {outputFile: reportPath}]], + timeout: 60_000, + updateSnapshots: "none", + workers: 1, + use: { + baseURL, + screenshot: "off", + trace: "off", + }, + projects: [ + // Full tile raster makes repeated Chromium launches byte-identical. + {name: "chromium", use: {...devices["Desktop Chrome"], launchOptions: {args: ["--disable-partial-raster"]}}}, + {name: "firefox", testIgnore: chromiumOnly, use: { + ...devices["Desktop Firefox"], + launchOptions: {firefoxUserPrefs: {"browser.chrome.site_icons": false, "fission.webContentIsolationStrategy": 1}}, + }}, + {name: "webkit", testIgnore: chromiumOnly, use: {...devices["Desktop Safari"]}}, + ], +}); diff --git a/proofkit/requirement-bindings.json b/proofkit/requirement-bindings.json index e2f3f910..096cba72 100644 --- a/proofkit/requirement-bindings.json +++ b/proofkit/requirement-bindings.json @@ -628,6 +628,14 @@ "proofState": "witness_backed", "nonClaims": ["This bounded diagnostic taxonomy does not prove confidentiality of values outside its admitted secret and control classes or claim that every repository-internal tool invocation uses the installed-carrier timeout policy."] }, + { + "requirementId": "REQ-PROOFKIT-QUALITY-028", + "ownerId": "proofkit.supply-chain-quality", + "specPath": "docs/specs/proofkit-supply-chain-quality/requirements.v2.json", + "claimLevel": "blocking", + "proofState": "witness_backed", + "nonClaims": ["This requirement does not claim pixel identity outside the recorded reference environment or on CPU models other than the calibrated Apple M1 (implementer 0x61) and Arm Neoverse N2 (implementer 0x41, part 0xd49), coverage of content below the first 2400 CSS pixels of a view, that weights 400 and 500, or weights 600 to 900, render differently (only regular and bold faces are installed, so each group renders identically), that a family name unknown to fontconfig, other than ui-monospace, falls through to the next family of its stack in WebKit (a name WebKit looks up resolves to Liberation Sans), that an admitted run directory is unmodified gate output (only a passing gate on the admitting tree proves the admitted digests), that the release candidate or the package gate reruns the visual gate, who reviewed or admitted a reference change, a human usability study, or WCAG conformance certification. Admitted images are not stored in the repository; they are reproducible from the admitting commit in the recorded environment on a calibrated CPU model while the pinned image and the pinned toolchain and dependency downloads remain available, and the uploaded gallery is derived, retention-limited evidence."] + }, { "requirementId": "REQ-PROOFKIT-SPEC-017", "ownerId": "proofkit.spec-proof-core", @@ -5211,6 +5219,21 @@ "commandIds": ["proofkit.actionlint", "proofkit.go-test"], "environmentClasses": ["local-go"] }, + { + "requirementId": "REQ-PROOFKIT-QUALITY-025", + "scenarioId": "proofkit.supply-chain-quality.workflow-container-digest-pins", + "witnessId": "proofkit.workflow.container-digest-pin-falsifiers", + "witnessKind": "contract", + "witnessPath": "scripts/workflow_browser_visual_oracle_test.go", + "witnessSelectors": [ + { + "selector": "TestWorkflowContainerImagesArePinnedByDigest", + "command": "go test ./scripts -run '^TestWorkflowContainerImagesArePinnedByDigest$'" + } + ], + "commandIds": ["proofkit.go-test"], + "environmentClasses": ["local-go"] + }, { "requirementId": "REQ-PROOFKIT-QUALITY-026", "scenarioId": "proofkit.supply-chain-quality.unicode-scalar-policy", @@ -5564,6 +5587,138 @@ "commandIds": ["proofkit.go-test"], "environmentClasses": ["local-go"] }, + { + "requirementId": "REQ-PROOFKIT-QUALITY-028", + "scenarioId": "proofkit.supply-chain-quality.visual-reference-inventory", + "witnessId": "proofkit.browser-visual.reference-inventory-falsifiers", + "witnessKind": "contract", + "witnessPath": "internal/tools/browserproofverify/visual_inventory_test.go", + "witnessSelectors": [ + { + "selector": "TestBrowserVisualScriptIsSeparateFromThePackageGate", + "command": "go test ./internal/tools/browserproofverify -run '^TestBrowserVisualScriptIsSeparateFromThePackageGate$'" + }, + { + "selector": "TestCommittedVisualReferencesAreDigestsBoundToTheRecordedEnvironment", + "command": "go test ./internal/tools/browserproofverify -run '^TestCommittedVisualReferencesAreDigestsBoundToTheRecordedEnvironment$'" + }, + { + "selector": "TestVisualInventoryIsTheExactViewEngineSchemeWidthProduct", + "command": "go test ./internal/tools/browserproofverify -run '^TestVisualInventoryIsTheExactViewEngineSchemeWidthProduct$'" + }, + { + "selector": "TestVisualReferenceInventoryRefusesEveryIntegrityMutant", + "command": "go test ./internal/tools/browserproofverify -run '^TestVisualReferenceInventoryRefusesEveryIntegrityMutant$'" + }, + { + "selector": "TestVisualReferencesStayOutsideThePackage", + "command": "go test ./internal/tools/browserproofverify -run '^TestVisualReferencesStayOutsideThePackage$'" + } + ], + "commandIds": ["proofkit.go-test"], + "environmentClasses": ["local-go"] + }, + { + "requirementId": "REQ-PROOFKIT-QUALITY-028", + "scenarioId": "proofkit.supply-chain-quality.visual-environment-toolchain-binding", + "witnessId": "proofkit.browser-visual.environment-toolchain-binding", + "witnessKind": "contract", + "witnessPath": "internal/tools/browserproofverify/visual_inventory_test.go", + "witnessSelectors": [ + { + "selector": "TestVisualEnvironmentBindsThePinnedPlaywrightToolchain", + "command": "go test ./internal/tools/browserproofverify -run '^TestVisualEnvironmentBindsThePinnedPlaywrightToolchain$'" + } + ], + "commandIds": ["proofkit.go-test"], + "environmentClasses": ["local-go"] + }, + { + "requirementId": "REQ-PROOFKIT-QUALITY-028", + "scenarioId": "proofkit.supply-chain-quality.visual-proof-record", + "witnessId": "proofkit.browser-visual.proof-record-falsifiers", + "witnessKind": "contract", + "witnessPath": "internal/tools/browserproofverify/visual_record_test.go", + "witnessSelectors": [ + { + "selector": "TestVisualProofRunWritesNoRecordOnFailureOrRequiredUnverified", + "command": "go test ./internal/tools/browserproofverify -run '^TestVisualProofRunWritesNoRecordOnFailureOrRequiredUnverified$'" + }, + { + "selector": "TestVisualRecordAdmitsOnlyConsistentPassedOrUnverifiedStates", + "command": "go test ./internal/tools/browserproofverify -run '^TestVisualRecordAdmitsOnlyConsistentPassedOrUnverifiedStates$'" + } + ], + "commandIds": ["proofkit.go-test"], + "environmentClasses": ["local-go"] + }, + { + "requirementId": "REQ-PROOFKIT-QUALITY-028", + "scenarioId": "proofkit.supply-chain-quality.visual-reference-admission", + "witnessId": "proofkit.browser-visual.reference-admission-falsifiers", + "witnessKind": "contract", + "witnessPath": "internal/tools/browserproofverify/visual_inventory_test.go", + "witnessSelectors": [ + { + "selector": "TestVisualReferenceAdmissionFromRunRefusesEveryUnadmissibleRun", + "command": "go test ./internal/tools/browserproofverify -run '^TestVisualReferenceAdmissionFromRunRefusesEveryUnadmissibleRun$'" + } + ], + "commandIds": ["proofkit.go-test"], + "environmentClasses": ["local-go"] + }, + { + "requirementId": "REQ-PROOFKIT-QUALITY-028", + "scenarioId": "proofkit.supply-chain-quality.visual-readme-project", + "witnessId": "proofkit.browser-visual.readme-project-admission", + "witnessKind": "contract", + "witnessPath": "internal/testsupport/browserfixture/readme_test.go", + "witnessSelectors": [ + { + "selector": "TestReadmeProjectMaterializesAnAdmittedViewProject", + "command": "go test ./internal/testsupport/browserfixture -run '^TestReadmeProjectMaterializesAnAdmittedViewProject$'" + } + ], + "commandIds": ["proofkit.go-test"], + "environmentClasses": ["local-go"] + }, + { + "requirementId": "REQ-PROOFKIT-QUALITY-028", + "scenarioId": "proofkit.supply-chain-quality.visual-ci-reference-environment", + "witnessId": "proofkit.browser-visual.ci-reference-environment-falsifiers", + "witnessKind": "contract", + "witnessPath": "scripts/workflow_browser_visual_oracle_test.go", + "witnessSelectors": [ + { + "selector": "TestCIBrowserVisualRejectsWeakenedReferenceGate", + "command": "go test ./scripts -run '^TestCIBrowserVisualRejectsWeakenedReferenceGate$'" + }, + { + "selector": "TestCIBrowserVisualRunsTheRecordedReferenceEnvironment", + "command": "go test ./scripts -run '^TestCIBrowserVisualRunsTheRecordedReferenceEnvironment$'" + } + ], + "commandIds": ["proofkit.go-test"], + "environmentClasses": ["local-go"] + }, + { + "requirementId": "REQ-PROOFKIT-QUALITY-028", + "scenarioId": "proofkit.supply-chain-quality.visual-exact-comparator", + "witnessId": "proofkit.browser-visual.exact-comparator-falsifiers", + "witnessKind": "technical", + "witnessPath": "scripts/browser-visual-references.test.mjs", + "commandIds": ["proofkit.browser-check"], + "environmentClasses": ["local-node-browser"] + }, + { + "requirementId": "REQ-PROOFKIT-QUALITY-028", + "scenarioId": "proofkit.supply-chain-quality.visual-reference-gate", + "witnessId": "proofkit.browser-visual.reference-gate", + "witnessKind": "technical", + "witnessPath": "tests/visual/references.spec.mjs", + "commandIds": ["proofkit.browser-visual"], + "environmentClasses": ["container-linux-arm64-browser"] + }, { "requirementId": "REQ-PROOFKIT-SPEC-019", "scenarioId": "proofkit.spec-proof-core.requirement-context-composition", @@ -10468,6 +10623,11 @@ "command": "npm run browser:check", "environmentClass": "local-node-browser" }, + { + "commandId": "proofkit.browser-visual", + "command": "npm run browser:visual -- --require-reference-environment", + "environmentClass": "container-linux-arm64-browser" + }, { "commandId": "proofkit.command-contract-check", "command": "npm run command-contract:check", diff --git a/proofkit/witness-plan.json b/proofkit/witness-plan.json index 6754d295..147099ae 100644 --- a/proofkit/witness-plan.json +++ b/proofkit/witness-plan.json @@ -12,6 +12,7 @@ "none" ], "environmentClasses": [ + "container-linux-arm64-browser", "local-go", "local-go-legacy-positive", "local-go-node", @@ -21,6 +22,18 @@ "local-python" ], "environmentClassPolicies": [ + { + "environmentClass": "container-linux-arm64-browser", + "networkPolicies": [ + "none" + ], + "credentialClasses": [ + "none" + ], + "cachePolicies": [ + "disabled" + ] + }, { "environmentClass": "local-go", "networkPolicies": [ @@ -116,6 +129,7 @@ "parallelGroups": [ "boundary-contract", "browser-runtime", + "browser-visual", "compact-contract", "coverage-metrics", "graph-contract", @@ -877,6 +891,43 @@ ] } }, + { + "schemaVersion": 1, + "id": "proofkit.browser-visual", + "cwd": ".", + "argv": [ + "npm", + "run", + "browser:visual", + "--", + "--require-reference-environment" + ], + "environment": { + "inherit": "none", + "allowlist": [], + "classes": [ + "container-linux-arm64-browser" + ] + }, + "timeoutMs": 600000, + "networkPolicy": "none", + "credentialClass": "none", + "cachePolicy": "disabled", + "expectedArtifacts": [ + { + "kind": "report", + "path": "artifacts/proofkit/browser-visual-proof.json", + "required": true + } + ], + "parallelGroup": "browser-visual", + "exitCodePolicy": { + "kind": "zero", + "successCodes": [ + 0 + ] + } + }, { "schemaVersion": 1, "id": "proofkit.receipt-contract-check", @@ -1130,6 +1181,98 @@ "Pinned Chromium, Firefox, and Playwright WebKit engines do not prove branded Safari compatibility." ] }, + { + "commandId": "proofkit.browser-visual", + "inputSelectors": [ + "go.mod", + "go.sum", + "internal/command/adoptionmaterialization", + "internal/command/adoptionplan", + "internal/command/capabilitymapadmission", + "internal/command/nativeevidenceguidance", + "internal/command/projectstatus", + "internal/command/repositoryinventory", + "internal/command/requirementbinding", + "internal/command/requirementbrowser", + "internal/command/requirementcontext", + "internal/command/requirementcoverageview", + "internal/command/requirementdiff", + "internal/command/requirementgraph", + "internal/command/requirementproofview", + "internal/command/requirementsourceadmission", + "internal/command/requirementsourceview", + "internal/command/requirementspectree", + "internal/command/stackpreset", + "internal/command/testevidenceinventory", + "internal/kernel/admission", + "internal/kernel/admit", + "internal/kernel/agentenvelope", + "internal/kernel/browserdoc", + "internal/kernel/cliexec", + "internal/kernel/compactproofcontract", + "internal/kernel/diagnostic", + "internal/kernel/digest", + "internal/kernel/jsonshape", + "internal/kernel/markdownfmt", + "internal/kernel/pathidentity", + "internal/kernel/proofvocab", + "internal/kernel/report", + "internal/kernel/repositorytransaction", + "internal/kernel/requirementsourcecodec", + "internal/kernel/requirementsourcemodel", + "internal/kernel/rootpath", + "internal/kernel/scenarioidentity", + "internal/kernel/secretjson", + "internal/kernel/stablejson", + "internal/kernel/unicodepolicy", + "internal/kernel/witnesscommand", + "internal/testsupport/browserfixture", + "internal/tools/browserproofverify", + "internal/tools/browsertestserver", + "package-lock.json", + "package.json", + "playwright.visual.config.mjs", + "proofkit/witness-plan.json", + "scripts/browser-proof-execution.mjs", + "scripts/browser-proof-inputs.mjs", + "scripts/browser-visual-proof-inputs.v1.json", + "scripts/browser-visual-references.test.mjs", + "scripts/diagnostic.mjs", + "scripts/stable-json.mjs", + "scripts/write-browser-visual-proof.mjs", + "tests/visual" + ], + "outputSelectors": [ + "artifacts/proofkit/browser-visual-proof.json" + ], + "resourceReads": [ + "resource.proofkit.source" + ], + "resourceWrites": [ + "resource.proofkit.local-artifacts" + ], + "exclusiveLocks": [], + "sideEffectClass": "local_write", + "deterministicOutput": false, + "cacheAdmissionRefs": [], + "retryPolicy": { + "kind": "none", + "maxAttempts": 1 + }, + "cancellationPolicy": { + "kind": "cooperative", + "graceMs": 5000 + }, + "timeoutPolicy": { + "kind": "bounded", + "timeoutMs": 600000 + }, + "nonClaims": [ + "The release candidate and the package gate do not run visual reference proof.", + "Visual reference proof compares rendered views only in the recorded reference environment; any other host reports the comparison as unverified and makes no pixel claim.", + "Visual reference proof does not cover content below the first 2400 CSS pixels of a view, identify who reviewed a reference, or prove usability or WCAG conformance." + ] + }, { "commandId": "proofkit.ci-receipt-anchor", "inputSelectors": [ @@ -1414,12 +1557,16 @@ "go.mod", "go.sum", "internal", + "package-lock.json", "package.json", "proofkit/cli-contract.v2.json", "proofkit/requirement-bindings.json", "proofkit/witness-plan.json", "scripts/browser-runtime-proof-inputs.v1.json", - "scripts/validate-self-hosting-receipts.go" + "scripts/browser-visual-capture-digests.mjs", + "scripts/browser-visual-proof-inputs.v1.json", + "scripts/validate-self-hosting-receipts.go", + "tests/visual" ], "outputSelectors": [], "resourceReads": [ diff --git a/scripts/browser-proof-execution.mjs b/scripts/browser-proof-execution.mjs index 4ba25410..795762e3 100644 --- a/scripts/browser-proof-execution.mjs +++ b/scripts/browser-proof-execution.mjs @@ -5,7 +5,7 @@ import {join, resolve} from "node:path"; import {assertInputSnapshotUnchanged, materializeInputSnapshot, snapshotInputAssets} from "./browser-proof-inputs.mjs"; import {createUTF8StrictStreamDecoder, decodeUTF8Strict} from "./stable-json.mjs"; -export async function executeBrowserProof({environment, execFile, inputPaths, nodeExecutable, resolution, runDirectory, spawnProcess, startServer = startBrowserServer, testCommand}) { +export async function executeBrowserProof({admissionArguments = [], environment, execFile, inputPaths, nodeExecutable, resolution, runDirectory, spawnProcess, startServer = startBrowserServer, testCommand}) { const sourceDirectory = resolve(runDirectory, "source"); const assets = materializeInputSnapshot(inputPaths, ".", sourceDirectory); symlinkSync(resolve("node_modules"), join(sourceDirectory, "node_modules"), process.platform === "win32" ? "junction" : "dir"); @@ -18,7 +18,7 @@ export async function executeBrowserProof({environment, execFile, inputPaths, no testResult = runBrowserProofTests({environment, nodeExecutable, runDirectory, serverURL: server.url, sourceDirectory, spawnProcess, testCommand}); if (testResult.error) throw testResult.error; if (testResult.status !== 0) throw new Error(`browser test command failed; diagnostics retained in ${runDirectory}`); - projects = admitBrowserProofReport({execFile, reportPath: resolve(runDirectory, "playwright-report.json"), sourceDirectory}); + projects = admitBrowserProofReport({admissionArguments, execFile, reportPath: resolve(runDirectory, "playwright-report.json"), sourceDirectory}); } catch (error) { executionError = error; } @@ -57,10 +57,10 @@ export function runBrowserProofTests({environment, nodeExecutable, runDirectory, }); } -function admitBrowserProofReport({execFile, reportPath, sourceDirectory}) { +function admitBrowserProofReport({admissionArguments, execFile, reportPath, sourceDirectory}) { const output = execFile( "go", - ["run", "./internal/tools/browserproofverify", "--admit-playwright-report", reportPath], + ["run", "./internal/tools/browserproofverify", ...admissionArguments, "--admit-playwright-report", reportPath], {cwd: sourceDirectory, maxBuffer: 8 << 20}, ); const encoded = typeof output === "string" ? output : decodeUTF8Strict(output, "browser proof report admission output"); diff --git a/scripts/browser-visual-capture-digests.mjs b/scripts/browser-visual-capture-digests.mjs new file mode 100644 index 00000000..954a8335 --- /dev/null +++ b/scripts/browser-visual-capture-digests.mjs @@ -0,0 +1,16 @@ +import {stdout} from "node:process"; + +import {captureInventory} from "../tests/visual/capture-inventory.mjs"; +import {runDiagnosticEntrypoint} from "./diagnostic.mjs"; +import {stableJSONStringify} from "./stable-json.mjs"; + +// Prints the canonical digest and dimensions of every capture below one +// capture directory, recomputed from the PNG files. Reference admission calls +// it; the visual gate never does. +await runDiagnosticEntrypoint(async () => { + const [directory] = process.argv.slice(2); + if (typeof directory !== "string" || directory === "" || process.argv.length !== 3) { + throw new Error("usage: browser-visual-capture-digests.mjs CAPTURE_DIRECTORY"); + } + stdout.write(`${stableJSONStringify(captureInventory(directory))}\n`); +}); diff --git a/scripts/browser-visual-proof-inputs.v1.json b/scripts/browser-visual-proof-inputs.v1.json new file mode 100644 index 00000000..985503ca --- /dev/null +++ b/scripts/browser-visual-proof-inputs.v1.json @@ -0,0 +1,20 @@ +{ + "schemaVersion": 1, + "proofKind": "proofkit.browser-visual-proof-inputs", + "serverTarget": "./internal/tools/browsertestserver", + "testRoot": "tests/visual", + "writerPath": "scripts/write-browser-visual-proof.mjs", + "paths": [ + "go.mod", + "go.sum", + "package-lock.json", + "package.json", + "playwright.visual.config.mjs", + "proofkit/witness-plan.json", + "scripts/browser-proof-execution.mjs", + "scripts/browser-proof-inputs.mjs", + "scripts/browser-visual-references.test.mjs", + "scripts/diagnostic.mjs", + "scripts/stable-json.mjs" + ] +} diff --git a/scripts/browser-visual-references.test.mjs b/scripts/browser-visual-references.test.mjs new file mode 100644 index 00000000..fd6ba433 --- /dev/null +++ b/scripts/browser-visual-references.test.mjs @@ -0,0 +1,232 @@ +import assert from "node:assert/strict"; +import {createHash} from "node:crypto"; +import {mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync} from "node:fs"; +import {tmpdir} from "node:os"; +import {join} from "node:path"; +import test from "node:test"; +import {crc32, deflateSync} from "node:zlib"; + +import {captureInventory} from "../tests/visual/capture-inventory.mjs"; +import {comparedEnvironmentFields, environmentDivergence, observeEnvironment} from "../tests/visual/environment-probe.mjs"; +import {canonicalRGBADigest, decodePNG} from "../tests/visual/exact-png.mjs"; + +function chunk(type, body) { + const header = Buffer.alloc(8); + header.writeUInt32BE(body.length, 0); + header.write(type, 4, "latin1"); + const checksum = Buffer.alloc(4); + checksum.writeUInt32BE(crc32(Buffer.concat([header.subarray(4), body])), 0); + return Buffer.concat([header, body, checksum]); +} + +// An independent Paeth predictor for the test encoder (PNG specification 9.4). +function paeth(left, up, upLeft) { + const estimate = left + up - upLeft; + const distances = [Math.abs(estimate - left), Math.abs(estimate - up), Math.abs(estimate - upLeft)]; + if (distances[0] <= distances[1] && distances[0] <= distances[2]) return left; + return distances[1] <= distances[2] ? up : upLeft; +} + +// encodePNG writes 8-bit images with one row filter per row. +function encodePNG({width, height, colorType, channels, pixels, filters = [], palette, transparency}) { + const stride = width * channels; + const rows = []; + for (let row = 0; row < height; row += 1) { + const filter = filters[row % Math.max(filters.length, 1)] ?? 0; + const line = pixels.subarray(row * stride, (row + 1) * stride); + const previous = row > 0 ? pixels.subarray((row - 1) * stride, row * stride) : Buffer.alloc(stride); + const encoded = Buffer.alloc(stride); + for (let index = 0; index < stride; index += 1) { + const left = index >= channels ? line[index - channels] : 0; + const up = previous[index]; + const upLeft = index >= channels ? previous[index - channels] : 0; + const predictor = [0, left, up, (left + up) >> 1, paeth(left, up, upLeft)][filter]; + encoded[index] = (line[index] - predictor) & 0xff; + } + rows.push(Buffer.from([filter]), encoded); + } + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header.set([8, colorType, 0, 0, 0], 8); + const parts = [Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), chunk("IHDR", header)]; + if (palette) parts.push(chunk("PLTE", palette)); + if (transparency) parts.push(chunk("tRNS", transparency)); + parts.push(chunk("IDAT", deflateSync(Buffer.concat(rows))), chunk("IEND", Buffer.alloc(0))); + return Buffer.concat(parts); +} + +function gradient(width, height) { + const pixels = Buffer.alloc(width * height * 4); + for (let pixel = 0; pixel < width * height; pixel += 1) { + pixels.set([(pixel * 7) & 0xff, (pixel * 13) & 0xff, (pixel * 29) & 0xff, 255], pixel * 4); + } + return pixels; +} + +test("exact PNG decoding reverses every row filter and color type", () => { + const width = 9; + const height = 6; + const rgba = gradient(width, height); + const decoded = decodePNG(encodePNG({width, height, colorType: 6, channels: 4, pixels: rgba, filters: [0, 1, 2, 3, 4]})); + assert.deepEqual([decoded.width, decoded.height], [width, height]); + assert.ok(decoded.rgba.equals(rgba)); + const rgb = Buffer.alloc(width * height * 3); + for (let pixel = 0; pixel < width * height; pixel += 1) rgb.set(rgba.subarray(pixel * 4, pixel * 4 + 3), pixel * 3); + assert.ok(decodePNG(encodePNG({width, height, colorType: 2, channels: 3, pixels: rgb, filters: [4, 3, 2, 1, 0]})).rgba.equals(rgba)); + const palette = Buffer.from([10, 20, 30, 40, 50, 60]); + const indexed = decodePNG(encodePNG({width: 2, height: 1, colorType: 3, channels: 1, pixels: Buffer.from([0, 1]), palette, transparency: Buffer.from([128])})); + assert.deepEqual([...indexed.rgba], [10, 20, 30, 128, 40, 50, 60, 255]); +}); + +test("the canonical RGBA digest covers dimensions and every channel of every pixel", () => { + const width = 16; + const height = 8; + const reference = decodePNG(encodePNG({width, height, colorType: 6, channels: 4, pixels: gradient(width, height)})); + // A known-answer vector for a 2x1 image fixes the encoded bytes. + assert.equal(canonicalRGBADigest({width: 2, height: 1, rgba: Buffer.from([1, 2, 3, 4, 5, 6, 7, 8])}), "5c3a27f684eb264dbc461a30e8265ffd30c21207e24e8aa1756e04bb5ed61636"); + const digest = canonicalRGBADigest(reference); + assert.match(digest, /^[0-9a-f]{64}$/); + assert.equal(canonicalRGBADigest({...reference, rgba: Buffer.from(reference.rgba)}), digest); + const differs = (rgba, size = {width, height}) => canonicalRGBADigest({...size, rgba}) !== digest; + const unit = Buffer.from(reference.rgba); + unit[5 * 4 + 2] = (unit[5 * 4 + 2] + 1) & 0xff; + assert.ok(differs(unit), "one unit in one channel"); + const alpha = Buffer.from(reference.rgba); + alpha[3] = 254; + assert.ok(differs(alpha), "alpha"); + const shifted = Buffer.alloc(reference.rgba.length); + for (let row = 0; row < height; row += 1) { + reference.rgba.copy(shifted, row * width * 4 + 4, row * width * 4, (row + 1) * width * 4 - 4); + reference.rgba.copy(shifted, row * width * 4, row * width * 4, row * width * 4 + 4); + } + assert.ok(differs(shifted), "one-pixel shift"); + assert.ok(differs(reference.rgba, {width: height, height: width}), "same bytes, transposed dimensions"); + assert.throws(() => canonicalRGBADigest({width, height: height + 1, rgba: reference.rgba}), /exactly width\*height/); +}); + +test("the capture inventory digests every PNG capture and refuses other files", () => { + const directory = mkdtempSync(join(tmpdir(), "proofkit-capture-inventory-")); + try { + mkdirSync(join(directory, "webkit")); + mkdirSync(join(directory, "chromium")); + const pixels = gradient(3, 2); + writeFileSync(join(directory, "webkit", "view.dark.320.png"), encodePNG({width: 3, height: 2, colorType: 6, channels: 4, pixels})); + writeFileSync(join(directory, "chromium", "view.light.320.png"), encodePNG({width: 3, height: 2, colorType: 6, channels: 4, pixels, filters: [4, 1]})); + const entries = captureInventory(directory); + assert.deepEqual(entries.map((entry) => entry.id), ["chromium/view.light.320", "webkit/view.dark.320"]); + assert.equal(entries[0].rgbaSHA256, entries[1].rgbaSHA256, "filters do not change the canonical digest"); + assert.deepEqual([entries[0].width, entries[0].height], [3, 2]); + writeFileSync(join(directory, "notes.txt"), "x"); + assert.throws(() => captureInventory(directory), /only PNG captures/); + rmSync(join(directory, "notes.txt")); + symlinkSync(join(directory, "webkit", "view.dark.320.png"), join(directory, "webkit", "link.png")); + assert.throws(() => captureInventory(directory), /symlinks/); + } finally { + rmSync(directory, {recursive: true, force: true}); + } +}); + +test("exact PNG decoding refuses malformed, unsupported or oversized images", () => { + const valid = encodePNG({width: 2, height: 2, colorType: 6, channels: 4, pixels: gradient(2, 2)}); + const corrupt = Buffer.from(valid); + corrupt[30] ^= 0xff; + assert.throws(() => decodePNG(corrupt), /checksum/); + assert.throws(() => decodePNG(valid.subarray(0, valid.length - 4)), /truncated|IEND/); + assert.throws(() => decodePNG(Buffer.from("not a png")), /not a PNG/); + const sixteenBit = Buffer.from(valid); + sixteenBit[24] = 16; + sixteenBit.writeUInt32BE(crc32(sixteenBit.subarray(12, 29)), 29); + assert.throws(() => decodePNG(sixteenBit), /8-bit/); + const huge = Buffer.from(valid); + huge.writeUInt32BE(1441, 16); + huge.writeUInt32BE(2400, 20); + huge.writeUInt32BE(crc32(huge.subarray(12, 29)), 29); + assert.throws(() => decodePNG(huge), /bounds/); +}); + +test("exact PNG decoding refuses transparency outside palette images and a second header", () => { + const rgb = Buffer.from([1, 2, 3, 4, 5, 6]); + assert.throws(() => decodePNG(encodePNG({width: 2, height: 1, colorType: 2, channels: 3, pixels: rgb, transparency: Buffer.from([0, 1, 0, 2, 0, 3])})), /tRNS is admitted only for palette images/); + assert.throws(() => decodePNG(encodePNG({width: 2, height: 1, colorType: 0, channels: 1, pixels: Buffer.from([7, 9]), transparency: Buffer.from([0, 7])})), /tRNS is admitted only for palette images/); + assert.throws(() => decodePNG(encodePNG({width: 1, height: 1, colorType: 6, channels: 4, pixels: Buffer.from([1, 2, 3, 4]), transparency: Buffer.from([0])})), /tRNS is admitted only for palette images/); + const valid = encodePNG({width: 2, height: 1, colorType: 2, channels: 3, pixels: rgb}); + assert.deepEqual([...decodePNG(valid).rgba], [1, 2, 3, 255, 4, 5, 6, 255]); + // A second IHDR after the first whose 3x1 grayscale-alpha layout fits the + // same image data, so only refusing it keeps the data from being reread. + const secondHeader = Buffer.alloc(13); + secondHeader.writeUInt32BE(3, 0); + secondHeader.writeUInt32BE(1, 4); + secondHeader.set([8, 4, 0, 0, 0], 8); + const headerEnd = 8 + 12 + 13; + const twoHeaders = Buffer.concat([valid.subarray(0, headerEnd), chunk("IHDR", secondHeader), valid.subarray(headerEnd)]); + assert.throws(() => decodePNG(twoHeaders), /exactly one IHDR/); +}); + +test("environment divergence names exactly the compared fields that differ", () => { + const reference = {architecture: "aarch64", dockerImage: {dockerImageName: "image:tag"}, emulation: "none", engines: {chromium: "1"}, fontConfig: "sha256:1", fontMatches: {serif: "a"}, os: {id: "ubuntu", versionId: "24.04"}, packages: {fontconfig: "1"}}; + assert.deepEqual(environmentDivergence(reference, structuredClone(reference)), []); + assert.deepEqual(environmentDivergence(reference, {...structuredClone(reference), architecture: "x86_64", packages: null}), ["architecture", "packages"]); + assert.deepEqual(environmentDivergence(reference, {...structuredClone(reference), engines: {chromium: "2"}, cpu: {model: "other"}}), ["engines"]); + assert.deepEqual(environmentDivergence(reference, {...structuredClone(reference), fontConfig: "sha256:2"}), ["fontConfig"]); + assert.deepEqual(comparedEnvironmentFields, ["architecture", "dockerImage", "emulation", "engines", "fontConfig", "fontMatches", "os", "packages"]); +}); + +test("environment observation marks emulation and missing observations without launching a browser", () => { + const files = { + "/proc/cpuinfo": "processor\t: 0\nvendor_id\t: GenuineIntel\nmodel name\t: Example CPU\nflags\t\t: fpu avx2 hypervisor\n", + "/proc/self/maps": "", + "/etc/os-release": "ID=ubuntu\nVERSION_ID=\"24.04\"\n", + "/ms-playwright/.docker-info": "{\"dockerImageName\":\"image:tag\",\"driverVersion\":\"1.0.0\"}", + }; + const read = (path) => files[path] ?? null; + const commands = {"uname -m": "aarch64"}; + const run = (command, args) => commands[[command, ...args].join(" ")] ?? null; + const observed = observeEnvironment({platform: "linux", read, run, environment: {}}); + assert.equal(observed.emulation, "foreign-cpu"); + assert.deepEqual(observed.cpu, {avx2: true, avx512f: false, features: "fpu avx2 hypervisor", hypervisor: true, model: "Example CPU", vendor: "GenuineIntel"}); + assert.equal(observed.fontConfig, null); + assert.deepEqual(observed.os, {id: "ubuntu", versionId: "24.04"}); + assert.deepEqual(observed.dockerImage, {dockerImageName: "image:tag", driverVersion: "1.0.0"}); + assert.equal(observed.packages, null); + assert.equal(observed.fontMatches, null); + commands["uname -m"] = "x86_64"; + assert.equal(observeEnvironment({platform: "linux", read, run, environment: {}}).emulation, "none"); + files["/proc/self/maps"] = "00400000-00401000 r-xp 00000000 00:00 0 /run/rosetta/rosetta\n"; + assert.equal(observeEnvironment({platform: "linux", read, run, environment: {}}).emulation, "translator-mapped"); +}); + +test("font matches query fc-match with each family's whole name", () => { + const queries = []; + const run = (command, args) => { + if (command !== "fc-match") return null; + queries.push(args); + return `match for ${args[0]}`; + }; + const observed = observeEnvironment({platform: "linux", read: () => null, run, environment: {}}); + // fc-match reads text after an unescaped hyphen as a point size, so + // "sans-serif" would query the family "sans"; every hyphen is escaped. + assert.deepEqual(queries, [["monospace"], ["sans\\-serif"], ["serif"], ["system\\-ui"], ["ui\\-monospace"]]); + assert.deepEqual(observed.fontMatches, { + "monospace": "match for monospace", + "sans-serif": "match for sans\\-serif", + "serif": "match for serif", + "system-ui": "match for system\\-ui", + "ui-monospace": "match for ui\\-monospace", + }); +}); + +test("arm64 CPU observation records the Features line and leaves x86 flags unobserved", () => { + const files = {"/proc/cpuinfo": "processor\t: 0\nBogoMIPS\t: 48.00\nFeatures\t: fp asimd sve sve2\nCPU implementer\t: 0x41\nCPU part\t: 0xd49\n"}; + const directory = mkdtempSync(join(tmpdir(), "proofkit-fontconfig-")); + try { + const config = join(directory, "fonts.conf"); + writeFileSync(config, "\n"); + const observed = observeEnvironment({platform: "linux", read: (path) => files[path] ?? null, run: (command) => (command === "uname" ? "aarch64" : null), environment: {FONTCONFIG_FILE: config}}); + assert.deepEqual(observed.cpu, {avx2: null, avx512f: null, features: "fp asimd sve sve2", hypervisor: null, model: "part 0xd49", vendor: "implementer 0x41"}); + assert.equal(observed.emulation, "none"); + assert.equal(observed.fontConfig, `sha256:${createHash("sha256").update("\n").digest("hex")}`); + } finally { + rmSync(directory, {recursive: true, force: true}); + } +}); diff --git a/scripts/diagnostic-entrypoints.test.mjs b/scripts/diagnostic-entrypoints.test.mjs index f6a0d80c..28132a3e 100644 --- a/scripts/diagnostic-entrypoints.test.mjs +++ b/scripts/diagnostic-entrypoints.test.mjs @@ -6,16 +6,21 @@ import {runDiagnosticEntrypoint} from "./diagnostic.mjs"; test("repository-owned JavaScript entrypoints use the diagnostic boundary", () => { const manifest = JSON.parse(readFileSync("package.json", "utf8")); - const browserInputs = JSON.parse(readFileSync("scripts/browser-runtime-proof-inputs.v1.json", "utf8")); - const entrypoints = new Set([browserInputs.writerPath]); + const browserWriters = ["scripts/browser-runtime-proof-inputs.v1.json", "scripts/browser-visual-proof-inputs.v1.json"] + .map((path) => JSON.parse(readFileSync(path, "utf8")).writerPath); + // Reference admission runs the capture digest owner from Go. + const goInvoked = ["scripts/browser-visual-capture-digests.mjs"]; + const entrypoints = new Set([...browserWriters, ...goInvoked]); for (const command of Object.values(manifest.scripts)) { for (const match of command.matchAll(/(?:^|&&|\|\|)\s*node\s+(scripts\/[^\s]+\.mjs)(?:\s|$)/gu)) { entrypoints.add(match[1]); } } assert.deepEqual([...entrypoints].sort(), [ + "scripts/browser-visual-capture-digests.mjs", "scripts/source-hygiene.mjs", "scripts/write-browser-proof.mjs", + "scripts/write-browser-visual-proof.mjs", ]); for (const path of entrypoints) { const source = readFileSync(path, "utf8"); diff --git a/scripts/workflow_browser_visual_oracle_test.go b/scripts/workflow_browser_visual_oracle_test.go new file mode 100644 index 00000000..a5aea885 --- /dev/null +++ b/scripts/workflow_browser_visual_oracle_test.go @@ -0,0 +1,327 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +const browserVisualJob = "browser-visual" + +// readVisualReferenceImageForTest projects name:tag@index digest from the +// recorded visual reference environment. +func readVisualReferenceImageForTest(t *testing.T) (string, string) { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "tests", "visual", "environment.json")) + if err != nil { + t.Fatal(err) + } + var environment struct { + Image struct { + Name string `json:"name"` + Tag string `json:"tag"` + IndexDigest string `json:"indexDigest"` + Platform string `json:"platform"` + PlatformDigests map[string]string `json:"platformDigests"` + } `json:"image"` + } + if err := json.Unmarshal(raw, &environment); err != nil { + t.Fatal(err) + } + image := environment.Image + if image.Platform != "linux/arm64" || image.PlatformDigests[image.Platform] == "" { + t.Fatalf("visual reference environment must record its linux/arm64 platform digest: %+v", image) + } + return image.Name + ":" + image.Tag + "@" + image.IndexDigest, image.Platform +} + +// validateCIBrowserVisualJob checks the visual reference job against the +// recorded environment: the exact digest-pinned container on the native ARM +// runner, the exact ordered setup and gate steps, the capture gallery upload +// after every outcome with 30-day retention, and the record upload only after +// success. +func validateCIBrowserVisualJob(workflow githubWorkflow, referenceImage string) error { + job, ok := workflow.Jobs[browserVisualJob] + if !ok { + return errors.New("ci workflow missing browser-visual job") + } + if runner, ok := job.RunsOn.(string); !ok || runner != "ubuntu-24.04-arm" { + return fmt.Errorf("browser-visual runs-on=%v, want the native ARM runner ubuntu-24.04-arm", job.RunsOn) + } + container, ok := job.Container.(map[string]any) + if !ok || container["image"] != referenceImage || container["options"] != ownedVisualContainerOptions || len(container) != 2 { + return fmt.Errorf("browser-visual container=%v, want the recorded reference image %s", job.Container, referenceImage) + } + if ownedVisualContainerImage != referenceImage { + return errors.New("owner container image differs from the recorded reference environment") + } + expected := []githubStep{ + {Name: "Checkout", Uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", With: map[string]any{"persist-credentials": false}}, + {Name: "Setup Node", Uses: "actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1", With: map[string]any{"node-version": "26.11.1", "package-manager-cache": false}}, + {Name: "Install verified npm", Uses: "./.github/actions/setup-verified-npm"}, + {Name: "Setup Go", Uses: "actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e", With: map[string]any{"go-version-file": "go.mod", "cache": true}}, + {Name: "Install locked browser tooling", Run: "npm ci --ignore-scripts"}, + {Name: "Run visual reference proof", Run: "npm run browser:visual -- --require-reference-environment"}, + {Name: "Upload visual reference captures", If: "${{ always() }}", Uses: "actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9", With: map[string]any{ + "name": "browser-visual-captures-${{ github.sha }}-${{ github.run_attempt }}", + "path": "artifacts/browser-run-*/captures\nartifacts/browser-run-*/candidate-references.json\nartifacts/browser-run-*/observed-environment.json\n", + "if-no-files-found": "error", + "retention-days": 30, + }}, + {Name: "Upload visual reference proof", Uses: "actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9", With: map[string]any{ + "name": "browser-visual-proof-${{ github.sha }}", + "path": "artifacts/proofkit/browser-visual-proof.json", + "if-no-files-found": "error", + "retention-days": 14, + }}, + } + if len(job.Steps) != len(expected) { + return fmt.Errorf("browser-visual step count=%d, want exact %d", len(job.Steps), len(expected)) + } + for index, want := range expected { + actual := job.Steps[index] + if actual.Name != want.Name || actual.Uses != want.Uses || actual.Run != want.Run || actual.If != want.If || + !reflect.DeepEqual(actual.With, want.With) || actual.runPresent != (want.Run != "") || + actual.usesPresent != (want.Uses != "") || actual.withPresent != (want.With != nil) || actual.ifPresent != (want.If != "") { + return fmt.Errorf("browser-visual step %d differs from exact owner step %q", index, want.Name) + } + } + return nil +} + +func TestCIBrowserVisualRunsTheRecordedReferenceEnvironment(t *testing.T) { + workflow := readWorkflowForTest(t, filepath.Join("..", ".github", "workflows", "ci.yml")) + referenceImage, _ := readVisualReferenceImageForTest(t) + if err := validateCIRequiredAggregate(workflow); err != nil { + t.Fatal(err) + } + if err := validateCIBrowserVisualJob(workflow, referenceImage); err != nil { + t.Fatal(err) + } + // The contributor instructions run the same digest-pinned image. + contributing, err := os.ReadFile(filepath.Join("..", "CONTRIBUTING.md")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(contributing), "--platform linux/arm64 --user 1001 --ipc=host --init") || !strings.Contains(string(contributing), referenceImage) { + t.Fatalf("CONTRIBUTING.md must run the recorded reference image %s", referenceImage) + } +} + +func TestCIBrowserVisualRejectsWeakenedReferenceGate(t *testing.T) { + base := readWorkflowForTest(t, filepath.Join("..", ".github", "workflows", "ci.yml")) + referenceImage, _ := readVisualReferenceImageForTest(t) + validate := func(workflow githubWorkflow) error { + return errors.Join(validateCIRequiredAggregate(workflow), validateCIBrowserVisualJob(workflow, referenceImage)) + } + if err := validate(cloneWorkflow(t, base)); err != nil { + t.Fatalf("owner workflow clone: %v", err) + } + stepIndex := func(job githubJob, name string) int { + index, err := uniqueStepIndex(job.Steps, name) + if err != nil || index < 0 { + t.Fatalf("locate %q: index=%d err=%v", name, index, err) + } + return index + } + withContainer := func(mutate func(map[string]any)) func(*githubWorkflow) { + return func(workflow *githubWorkflow) { + job := workflow.Jobs[browserVisualJob] + container := map[string]any{} + for key, value := range job.Container.(map[string]any) { + container[key] = value + } + mutate(container) + job.Container = container + workflow.Jobs[browserVisualJob] = job + } + } + visualAggregateLine := "test \"${{ needs.browser-visual.result }}\" = \"success\"\n" + cases := []struct { + name string + mutate func(*githubWorkflow) + }{ + {"unpinned image tag", withContainer(func(container map[string]any) { + container["image"] = "mcr.microsoft.com/playwright:v1.63.0-noble" + })}, + {"other image digest", withContainer(func(container map[string]any) { + container["image"] = "mcr.microsoft.com/playwright:v1.63.0-noble@sha256:" + strings.Repeat("0", 64) + })}, + {"platform manifest digest instead of the index", withContainer(func(container map[string]any) { + container["image"] = "mcr.microsoft.com/playwright:v1.63.0-noble@sha256:a0f4498920a5dbac63196d9140ed738ef00470f27e2e74029abd8850b7bd5717" + })}, + {"other image tag", withContainer(func(container map[string]any) { + container["image"] = strings.Replace(container["image"].(string), "v1.63.0-noble", "v1.64.0-noble", 1) + })}, + {"root container user", withContainer(func(container map[string]any) { + container["options"] = "--ipc=host --init" + })}, + {"container credentials", withContainer(func(container map[string]any) { + container["credentials"] = map[string]any{"username": "shadow"} + })}, + {"container removed", func(workflow *githubWorkflow) { + job := workflow.Jobs[browserVisualJob] + job.Container, job.containerPresent = nil, false + workflow.Jobs[browserVisualJob] = job + }}, + {"container on another job", func(workflow *githubWorkflow) { + job := workflow.Jobs["browser-runtime"] + job.Container, job.containerPresent = workflow.Jobs[browserVisualJob].Container, true + workflow.Jobs["browser-runtime"] = job + }}, + {"x64 runner", func(workflow *githubWorkflow) { + job := workflow.Jobs[browserVisualJob] + job.RunsOn = "ubuntu-24.04" + workflow.Jobs[browserVisualJob] = job + }}, + {"timeout lowered", func(workflow *githubWorkflow) { + job := workflow.Jobs[browserVisualJob] + job.TimeoutMinutes = 1 + workflow.Jobs[browserVisualJob] = job + }}, + {"reference environment no longer required", func(workflow *githubWorkflow) { + job := workflow.Jobs[browserVisualJob] + job.Steps[stepIndex(job, "Run visual reference proof")].Run = "npm run browser:visual" + workflow.Jobs[browserVisualJob] = job + }}, + {"gate step neutralized", func(workflow *githubWorkflow) { + job := workflow.Jobs[browserVisualJob] + job.Steps[stepIndex(job, "Run visual reference proof")].Run = "npm run browser:visual -- --require-reference-environment || true" + workflow.Jobs[browserVisualJob] = job + }}, + {"captures uploaded only on failure", func(workflow *githubWorkflow) { + job := workflow.Jobs[browserVisualJob] + job.Steps[stepIndex(job, "Upload visual reference captures")].If = "${{ failure() }}" + workflow.Jobs[browserVisualJob] = job + }}, + {"captures uploaded only on success", func(workflow *githubWorkflow) { + job := workflow.Jobs[browserVisualJob] + index := stepIndex(job, "Upload visual reference captures") + job.Steps[index].If, job.Steps[index].ifPresent = "", false + workflow.Jobs[browserVisualJob] = job + }}, + {"record uploaded after a failure", func(workflow *githubWorkflow) { + job := workflow.Jobs[browserVisualJob] + index := stepIndex(job, "Upload visual reference proof") + job.Steps[index].If, job.Steps[index].ifPresent = "${{ always() }}", true + workflow.Jobs[browserVisualJob] = job + }}, + {"observed environment left out of the gallery", func(workflow *githubWorkflow) { + job := workflow.Jobs[browserVisualJob] + index := stepIndex(job, "Upload visual reference captures") + with := map[string]any{} + for key, value := range job.Steps[index].With { + with[key] = value + } + with["path"] = "artifacts/browser-run-*/captures\nartifacts/browser-run-*/candidate-references.json\n" + job.Steps[index].With = with + workflow.Jobs[browserVisualJob] = job + }}, + {"unbounded gallery retention", func(workflow *githubWorkflow) { + job := workflow.Jobs[browserVisualJob] + index := stepIndex(job, "Upload visual reference captures") + with := map[string]any{} + for key, value := range job.Steps[index].With { + with[key] = value + } + delete(with, "retention-days") + job.Steps[index].With = with + workflow.Jobs[browserVisualJob] = job + }}, + {"record uploaded before the gate", func(workflow *githubWorkflow) { + job := workflow.Jobs[browserVisualJob] + gate, upload := stepIndex(job, "Run visual reference proof"), stepIndex(job, "Upload visual reference proof") + job.Steps[gate], job.Steps[upload] = job.Steps[upload], job.Steps[gate] + workflow.Jobs[browserVisualJob] = job + }}, + {"dropped from aggregate needs", func(workflow *githubWorkflow) { + gate := workflow.Jobs["ci-required-gate"] + needs := []any{} + for _, need := range gate.Needs.([]any) { + if need != browserVisualJob { + needs = append(needs, need) + } + } + gate.Needs = needs + workflow.Jobs["ci-required-gate"] = gate + }}, + {"result unchecked by aggregate", func(workflow *githubWorkflow) { + gate := workflow.Jobs["ci-required-gate"] + gate.Steps[0].Run = strings.Replace(gate.Steps[0].Run, visualAggregateLine, "", 1) + workflow.Jobs["ci-required-gate"] = gate + }}, + {"job continue on error", func(workflow *githubWorkflow) { + job := workflow.Jobs[browserVisualJob] + job.ContinueOnError = true + workflow.Jobs[browserVisualJob] = job + }}, + {"job environment", func(workflow *githubWorkflow) { + job := workflow.Jobs[browserVisualJob] + job.Env = map[string]any{"PLAYWRIGHT_BROWSERS_PATH": "/shadow"} + workflow.Jobs[browserVisualJob] = job + }}, + } + for _, item := range cases { + t.Run(item.name, func(t *testing.T) { + workflow := cloneWorkflow(t, base) + if !strings.Contains(workflow.Jobs["ci-required-gate"].Steps[0].Run, visualAggregateLine) { + t.Fatal("owner aggregate lacks the browser-visual result line") + } + item.mutate(&workflow) + if err := validate(workflow); err == nil { + t.Fatal("weakened visual reference gate was admitted") + } + }) + } +} + +func TestWorkflowContainerImagesArePinnedByDigest(t *testing.T) { + for _, reference := range []string{ + ownedVisualContainerImage, + "python:3.11.0-slim@sha256:" + strings.Repeat("a", 64), + "registry.example:5000/team/image:1.0@sha256:" + strings.Repeat("b", 64), + } { + if err := validateContainerImageReference(reference); err != nil { + t.Fatalf("digest-pinned reference %q was refused: %v", reference, err) + } + } + for _, reference := range []string{ + "mcr.microsoft.com/playwright:v1.63.0-noble", + "mcr.microsoft.com/playwright@sha256:" + strings.Repeat("a", 64), + "mcr.microsoft.com/playwright:v1.63.0-noble@sha256:" + strings.Repeat("A", 64), + "mcr.microsoft.com/playwright:v1.63.0-noble@sha256:" + strings.Repeat("a", 63), + "mcr.microsoft.com/playwright:latest@sha512:" + strings.Repeat("a", 64), + "", + } { + if err := validateContainerImageReference(reference); err == nil { + t.Fatalf("unpinned or malformed container image reference %q was admitted", reference) + } + } + ciPath := filepath.Join("..", ".github", "workflows", "ci.yml") + ciRaw := readWorkflowBytesForTest(t, ciPath) + ownerImageLine := " image: " + ownedVisualContainerImage + "\n" + ownerOptionsLine := " options: " + ownedVisualContainerOptions + "\n" + if !strings.Contains(string(ciRaw), " container:\n"+ownerImageLine+ownerOptionsLine) { + t.Fatal("owner ci workflow lacks the exact visual reference container") + } + for _, mutant := range []struct { + name, anchor, replacement string + }{ + {"unpinned image", ownerImageLine, " image: mcr.microsoft.com/playwright:v1.63.0-noble\n"}, + {"floating image tag", ownerImageLine, " image: mcr.microsoft.com/playwright:latest\n"}, + {"scalar container", " container:\n" + ownerImageLine + ownerOptionsLine, " container: " + ownedVisualContainerImage + "\n"}, + {"changed options", ownerOptionsLine, " options: --ipc=host --init\n"}, + {"container credentials", ownerOptionsLine, ownerOptionsLine + " credentials:\n username: shadow\n"}, + {"container on the source job", " source-quality:\n", " source-quality:\n container:\n" + ownerImageLine + ownerOptionsLine}, + {"unpinned container on the aggregate", " ci-required-gate:\n", " ci-required-gate:\n container: ubuntu:24.04\n"}, + } { + t.Run(mutant.name, func(t *testing.T) { + assertClosedWorkflowMutationRejected(t, ciPath, ciRaw, mutant.anchor, mutant.replacement) + }) + } +} diff --git a/scripts/workflow_oracle_support_test.go b/scripts/workflow_oracle_support_test.go index f38d854b..37ef07a6 100644 --- a/scripts/workflow_oracle_support_test.go +++ b/scripts/workflow_oracle_support_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "reflect" + "regexp" "strings" "testing" @@ -39,6 +40,7 @@ type githubRunDefaults struct { } type githubJob struct { + Container any `yaml:"container,omitempty"` ContinueOnError any `yaml:"continue-on-error,omitempty"` Defaults *githubDefaults `yaml:"defaults,omitempty"` Env map[string]any `yaml:"env"` @@ -53,6 +55,7 @@ type githubJob struct { TimeoutMinutes any `yaml:"timeout-minutes,omitempty"` Uses string `yaml:"uses,omitempty"` + containerPresent bool `yaml:"-"` continueOnErrorPresent bool `yaml:"-"` defaultsPresent bool `yaml:"-"` environmentPresent bool `yaml:"-"` @@ -106,6 +109,7 @@ func (job *githubJob) UnmarshalYAML(node *yaml.Node) error { return err } *job = githubJob(decoded) + job.containerPresent = yamlMappingHasKey(node, "container") job.continueOnErrorPresent = yamlMappingHasKey(node, "continue-on-error") job.defaultsPresent = yamlMappingHasKey(node, "defaults") job.environmentPresent = yamlMappingHasKey(node, "environment") @@ -270,9 +274,15 @@ func validateWorkflowClosedKeys(path string, raw []byte) error { "steps", "strategy", "environment", + "container", ); err != nil { return err } + if container := yamlMappingValue(job, "container"); container != nil || yamlMappingHasKey(job, "container") { + if err := validateOwnedJobContainer(path, jobID, container); err != nil { + return err + } + } if strategy := yamlMappingValue(job, "strategy"); strategy != nil { if filepath.Base(path) != "codeql.yml" || jobID != "analyze" { return fmt.Errorf("workflow %q job %q contains unowned strategy", path, jobID) @@ -385,6 +395,61 @@ func validateOwnedReleaseEnvironment(path string, jobID string, environment *yam return nil } +// The only admitted job container is the visual reference environment of the +// ci workflow's browser-visual job: an exact digest-pinned image reference and +// exact options. Every container image reference must carry a sha256 digest. +const ( + ownedVisualContainerJob = "browser-visual" + ownedVisualContainerImage = "mcr.microsoft.com/playwright:v1.63.0-noble@sha256:eff16c30e6f3f4af0a03fa4b706120d5e9b0891c344a27d64559aff5900a4a27" + ownedVisualContainerOptions = "--user 1001 --ipc=host --init" +) + +var digestPinnedContainerImage = regexp.MustCompile(`^[a-z0-9]+(?:[._-][a-z0-9]+)*(?:\.[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[0-9]+)?(?:/[a-z0-9]+(?:[._-][a-z0-9]+)*)*:[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}@sha256:[0-9a-f]{64}$`) + +func workflowAllowsVisualContainer(path string) bool { + actual, err := filepath.Abs(path) + if err != nil { + return false + } + expected, err := filepath.Abs(filepath.Join("..", ".github", "workflows", "ci.yml")) + return err == nil && actual == expected +} + +func validateContainerImageReference(reference string) error { + if !digestPinnedContainerImage.MatchString(reference) { + return fmt.Errorf("container image reference %q must be name:tag pinned by a sha256 digest", reference) + } + return nil +} + +func validateOwnedJobContainer(path, jobID string, container *yaml.Node) error { + if !workflowAllowsVisualContainer(path) || jobID != ownedVisualContainerJob { + return fmt.Errorf("workflow %q job %q contains unowned key %q", path, jobID, "container") + } + label := fmt.Sprintf("workflow %q job %q container", path, jobID) + if container == nil { + return fmt.Errorf("%s must be an exact mapping", label) + } + if err := validateClosedYAMLMapping(label, container, "image", "options"); err != nil { + return err + } + image := yamlMappingValue(container, "image") + options := yamlMappingValue(container, "options") + if image == nil || image.Kind != yaml.ScalarNode || image.Tag != "!!str" { + return fmt.Errorf("%s image must be a string", label) + } + if err := validateContainerImageReference(image.Value); err != nil { + return fmt.Errorf("%s: %w", label, err) + } + if image.Value != ownedVisualContainerImage { + return fmt.Errorf("%s image must equal the exact owner-reviewed reference environment image", label) + } + if options == nil || options.Kind != yaml.ScalarNode || options.Tag != "!!str" || options.Value != ownedVisualContainerOptions { + return fmt.Errorf("%s options must equal exact %q", label, ownedVisualContainerOptions) + } + return nil +} + func validateDefaultsMapping(label string, defaults *yaml.Node) error { if defaults == nil || defaults.Kind == yaml.ScalarNode && defaults.Tag == "!!null" { return nil diff --git a/scripts/workflow_package_gate_oracle_test.go b/scripts/workflow_package_gate_oracle_test.go index a1da8a7e..da020e93 100644 --- a/scripts/workflow_package_gate_oracle_test.go +++ b/scripts/workflow_package_gate_oracle_test.go @@ -22,6 +22,7 @@ const requiredPlatformSmokeOwnerCommand = "go run ./internal/tools/packagebuild const setupVerifiedNPMActionSHA256 = "e5b863b6c0c9fac02ed3e00708d760cb3bc3ef6dc33c138b6ca13670d5895339" const ciSourceQualityStepInventorySHA256 = "9e7fdac5768a868929325052379a0b6382d18fc47052af7c1a148663b46ad153" const ciBrowserRuntimeStepInventorySHA256 = "c157f7ee933119596eecbce7dd03478c84389736fd99248ef884e338f24d9ee4" +const ciBrowserVisualStepInventorySHA256 = "6f19d5ff18a70bcff69441bdbf8a783b8e0ec9456c971cfb4df3c75e688b741a" const releaseCandidateStepInventorySHA256 = "8e5002e046eab875b8549f4e90238f37a5d4d9bc77b4a9abe86d221fe6c7c2d5" type packageGateWorkflowExpectation struct { @@ -753,7 +754,7 @@ func TestCIWorkflowDeclaresFailClosedRequiredAggregate(t *testing.T) { if !usesAlwaysStatusCheck(gate.If) { t.Fatalf("ci-required-gate if=%q, want always() so failed or skipped needs are inspected", gate.If) } - wantNeeds := []string{"browser-runtime", "platform-smoke", "platform-smoke-intel", "platform-smoke-linux-arm64", "source-quality"} + wantNeeds := []string{"browser-runtime", "browser-visual", "platform-smoke", "platform-smoke-intel", "platform-smoke-linux-arm64", "source-quality"} if got := needsList(gate.Needs); !reflect.DeepEqual(got, wantNeeds) { t.Fatalf("ci-required-gate needs=%#v, want %#v", got, wantNeeds) } @@ -1850,6 +1851,7 @@ func singleLineRun(value string) string { const requiredAggregateShell = `set -euo pipefail test "${{ needs.browser-runtime.result }}" = "success" +test "${{ needs.browser-visual.result }}" = "success" test "${{ needs.source-quality.result }}" = "success" test "${{ needs.platform-smoke.result }}" = "success" test "${{ needs.platform-smoke-intel.result }}" = "success" @@ -1862,7 +1864,7 @@ npm run platform:smoke ` // The exact required leaf jobs of the CI aggregate, in needsList order. -var ciRequiredLeafJobs = []string{"browser-runtime", "platform-smoke", "platform-smoke-intel", "platform-smoke-linux-arm64", "source-quality"} +var ciRequiredLeafJobs = []string{"browser-runtime", "browser-visual", "platform-smoke", "platform-smoke-intel", "platform-smoke-linux-arm64", "source-quality"} const linuxArm64PlatformSmokeJob = "platform-smoke-linux-arm64" @@ -2096,6 +2098,7 @@ func validateCIRequiredAggregate(workflow githubWorkflow) error { workflow, map[string]string{ "browser-runtime": "", + "browser-visual": "", "platform-smoke": "", "platform-smoke-intel": "", "platform-smoke-linux-arm64": "", @@ -2106,6 +2109,9 @@ func validateCIRequiredAggregate(workflow githubWorkflow) error { "browser-runtime": { "Upload browser failure diagnostics": "failure()", }, + "browser-visual": { + "Upload visual reference captures": "always()", + }, }, ); err != nil { return err @@ -2120,6 +2126,7 @@ func validateCIRequiredAggregate(workflow githubWorkflow) error { "platform-smoke-intel": {name: "quality / platform smoke / macos-15-intel", runner: "macos-15-intel", timeoutMinutes: 20}, "platform-smoke-linux-arm64": {name: "quality / platform smoke / linux-arm64", runner: "ubuntu-24.04-arm", timeoutMinutes: 15}, "browser-runtime": {name: "quality / browser runtime", runner: "ubuntu-24.04", timeoutMinutes: 20}, + "browser-visual": {name: "quality / browser visual references", runner: "ubuntu-24.04-arm", timeoutMinutes: 20}, "ci-required-gate": {name: "quality / required aggregate", runner: "ubuntu-24.04", timeoutMinutes: 5}, } for jobID, expected := range exactJobTopology { @@ -2167,6 +2174,16 @@ func validateCIRequiredAggregate(workflow githubWorkflow) error { ); err != nil { return err } + if err := validateExactStepInventory( + "ci workflow browser-visual", + workflow.Jobs["browser-visual"].Steps, + ciBrowserVisualStepInventorySHA256, + ); err != nil { + return err + } + if err := validateCIJobContainers(workflow); err != nil { + return err + } gate, ok := workflow.Jobs["ci-required-gate"] if !ok { return errors.New("ci workflow missing ci-required-gate") @@ -2197,6 +2214,37 @@ func validateCIRequiredAggregate(workflow githubWorkflow) error { return nil } +// validateCIJobContainers admits exactly one job container: the digest-pinned +// visual reference environment of browser-visual with its exact options. +func validateCIJobContainers(workflow githubWorkflow) error { + for jobID, job := range workflow.Jobs { + if jobID != ownedVisualContainerJob { + if job.Container != nil || job.containerPresent { + return fmt.Errorf("ci workflow job %s must not run in a container", jobID) + } + continue + } + container, ok := job.Container.(map[string]any) + if !ok || !job.containerPresent || len(container) != 2 { + return fmt.Errorf("ci workflow job %s must declare the exact reference environment container", jobID) + } + image, imageOK := container["image"].(string) + if !imageOK { + return fmt.Errorf("ci workflow job %s container image must be a string", jobID) + } + if err := validateContainerImageReference(image); err != nil { + return fmt.Errorf("ci workflow job %s: %w", jobID, err) + } + if image != ownedVisualContainerImage || container["options"] != ownedVisualContainerOptions { + return fmt.Errorf("ci workflow job %s container differs from the exact owner-reviewed reference environment", jobID) + } + } + if _, ok := workflow.Jobs[ownedVisualContainerJob]; !ok { + return fmt.Errorf("ci workflow missing %s", ownedVisualContainerJob) + } + return nil +} + func validateExactWorkflowConcurrency( label string, actual *githubConcurrency, diff --git a/scripts/workflow_python_minimum_test.go b/scripts/workflow_python_minimum_test.go index 017d8065..b4b47bb7 100644 --- a/scripts/workflow_python_minimum_test.go +++ b/scripts/workflow_python_minimum_test.go @@ -17,7 +17,7 @@ func TestCIMinimumFailureCannotSatisfyAggregate(t *testing.T) { // Both jobs that run the CPython 3.11.0 installed smoke are required // leaves: a minimum failure in either job is a job result the aggregate // must refuse, whatever the other leaves report. - leaves := []string{"source-quality", "browser-runtime", "platform-smoke", "platform-smoke-intel", "platform-smoke-linux-arm64"} + leaves := []string{"source-quality", "browser-runtime", "browser-visual", "platform-smoke", "platform-smoke-intel", "platform-smoke-linux-arm64"} for _, failing := range ciMinimumPythonJobIDs() { for _, status := range []string{"success", "failure", "skipped", "cancelled"} { t.Run(failing+"/"+status, func(t *testing.T) { diff --git a/scripts/write-browser-visual-proof.mjs b/scripts/write-browser-visual-proof.mjs new file mode 100644 index 00000000..8369b8b7 --- /dev/null +++ b/scripts/write-browser-visual-proof.mjs @@ -0,0 +1,149 @@ +import {execFileSync, spawnSync} from "node:child_process"; +import {createHash} from "node:crypto"; +import {chmodSync, constants, copyFileSync, existsSync, readdirSync, readFileSync, rmSync, writeFileSync} from "node:fs"; +import {join, resolve} from "node:path"; + +import {captureInventory} from "../tests/visual/capture-inventory.mjs"; +import {comparedEnvironmentFields, environmentDivergence, observeEnvironment} from "../tests/visual/environment-probe.mjs"; +import {executeBrowserProof} from "./browser-proof-execution.mjs"; +import {isSafeRepoPath, loadBrowserProofInputResolution, snapshotInputAssets} from "./browser-proof-inputs.mjs"; +import {runDiagnosticEntrypoint} from "./diagnostic.mjs"; +import {decodeUTF8Strict, stableJSONStringify} from "./stable-json.mjs"; + +const manifestPath = "scripts/browser-visual-proof-inputs.v1.json"; +const testRoot = "tests/visual"; +const testCommand = ["node_modules/@playwright/test/cli.js", "test", "--config", "playwright.visual.config.mjs"]; + +await runDiagnosticEntrypoint(async () => { + const resolution = loadBrowserProofInputResolution(); + const inputPaths = resolution.inputPaths; + const runDirectory = requiredSafePath("PROOFKIT_BROWSER_RUN_DIRECTORY"); + const proofCandidatePath = requiredSafePath("PROOFKIT_BROWSER_PROOF_CANDIDATE_PATH"); + if (proofCandidatePath !== `${runDirectory}/browser-visual-proof.candidate.json`) { + throw new Error("browser visual proof candidate path must belong to the admitted run directory"); + } + const environment = JSON.parse(readFileSync(`${testRoot}/environment.json`, "utf8")); + const environmentId = `sha256:${createHash("sha256").update(stableJSONStringify(environment)).digest("hex")}`; + const references = JSON.parse(readFileSync(`${testRoot}/references.json`, "utf8")); + // The font configuration is copied into the run directory and set before the + // environment is observed and recorded and before any browser launches, so + // the hashed bytes are the bytes that fc-match and every browser read; the + // working-tree file is never read again. + const fontConfigPath = resolve(runDirectory, "fonts.conf"); + copyFileSync(`${testRoot}/fonts.conf`, fontConfigPath, constants.COPYFILE_EXCL); + chmodSync(fontConfigPath, 0o444); + process.env.FONTCONFIG_FILE = fontConfigPath; + const observedEnvironment = observeEnvironment(); + const divergentFields = environmentDivergence(environment.observed, observedEnvironment); + const observedRecord = { + ...Object.fromEntries(["cpu", ...comparedEnvironmentFields].map((field) => [field, observedEnvironment[field]])), + divergentFields, + }; + writeFileSync(join(runDirectory, "observed-environment.json"), `${JSON.stringify(observedRecord, null, 2)}\n`, {flag: "wx", mode: 0o644}); + const {cpu} = observedEnvironment; + process.stdout.write(`browser visual environment: cpu vendor=${cpu.vendor} model=${cpu.model} features=${cpu.features}; divergent fields: ${divergentFields.length === 0 ? "none" : divergentFields.join(", ")}\n`); + let assets; + let command = null; + let engines = []; + let projects = []; + if (divergentFields.length === 0) { + let execution; + try { + execution = await executeBrowserProof({ + admissionArguments: ["--manifest", manifestPath], + environment: process.env, + execFile: execFileSync, + inputPaths, + nodeExecutable: process.execPath, + resolution, + runDirectory, + spawnProcess: spawnSync, + testCommand, + }); + } finally { + // Passing or failing, the run keeps its captures and a candidate + // inventory derived from them for review and owner admission. + writeCandidateInventory(runDirectory, environmentId); + } + assertNoUnlistedTestFiles(execution.sourceDirectory, inputPaths); + for (const generated of [execution.sourceDirectory, join(runDirectory, "server"), join(runDirectory, "visual-fixture-server")]) { + rmSync(generated, {force: true, recursive: true}); + } + ({assets, projects} = execution); + command = {argv: testCommand, exitCode: execution.testResult.status, inputMode: "materialized_snapshot", runner: "node"}; + engines = projects.map(({browserName, browserVersion}) => ({name: browserName, version: browserVersion})); + } else { + assets = snapshotInputAssets(inputPaths); + } + const inputResolution = {serverTarget: resolution.serverTarget, writerPath: resolution.writerPath}; + const inputDigest = createHash("sha256").update(stableJSONStringify({assets, inputResolution})).digest("hex"); + const sourceRevision = decodeUTF8Strict(execFileSync("git", ["rev-parse", "HEAD"]), "git source revision output").trim(); + const sourceTreeState = decodeUTF8Strict( + execFileSync("git", ["status", "--porcelain=v1", "--untracked-files=all"]), + "git source tree status output", + ).trim() === "" ? "clean" : "dirty"; + const record = { + assets, + command, + divergentFields, + engines, + inputDigest: `sha256:${inputDigest}`, + inputResolution, + nonClaims: [ + "A passed record proves only that every listed reference view produced, in two byte-equal captures, its admitted canonical RGBA digest and dimensions in the recorded reference environment before record creation.", + "Only the first 2400 CSS pixels of each view are compared; content below that height is not covered by a reference.", + "An unverified record makes no pixel claim; it lists the compared environment fields that differ from the recorded reference environment.", + "The CPU vendor, model and features are recorded but are not part of the compared environment identity; pixel identity is calibrated only on Apple M1 (implementer 0x61) and Arm Neoverse N2 (implementer 0x41, part 0xd49).", + "Admitted images are not stored in the repository; they are reproducible from the admitting commit in the recorded environment on a calibrated CPU model while the pinned image and the pinned toolchain and dependency downloads remain available, and an uploaded capture gallery is derived, retention-limited evidence.", + "This record does not identify who reviewed or admitted a reference.", + "Playwright WebKit is not branded Safari compatibility proof.", + "A dirty-tree record is content-bound only to its listed assets; sourceRevision is contextual.", + "This record does not independently authenticate or sandbox the same-user proof runner, the container image beyond its observed tuple, external toolchain packages, or browser binaries.", + ], + observedEnvironment: Object.fromEntries(["cpu", ...comparedEnvironmentFields].map((field) => [field, observedEnvironment[field]])), + projects, + proofKind: "proofkit.browser-visual-proof", + referenceEnvironment: { + environmentId: references.environmentId, + referenceCount: Array.isArray(references.references) ? references.references.length : 0, + referencesDigest: `sha256:${createHash("sha256").update(readFileSync(`${testRoot}/references.json`)).digest("hex")}`, + }, + schemaVersion: 1, + sourceRevision, + sourceTreeState, + state: divergentFields.length === 0 ? "passed" : "unverified", + }; + writeFileSync(proofCandidatePath, `${JSON.stringify(record, null, 2)}\n`, {flag: "wx", mode: 0o600}); +}); + +function writeCandidateInventory(runDirectory, environmentId) { + const captures = join(runDirectory, "captures"); + const entries = existsSync(captures) ? captureInventory(captures) : []; + const candidate = {clipHeight: 2400, environmentId, kind: "proofkit.browser-visual-references", references: entries, schemaVersion: 1}; + writeFileSync(join(runDirectory, "candidate-references.json"), `${JSON.stringify(candidate, null, 2)}\n`, {flag: "wx", mode: 0o644}); +} + +// The materialized test directory must hold exactly its content-bound input +// files after the run: no capture, reference or other file may appear in it. +function assertNoUnlistedTestFiles(sourceDirectory, inputPaths) { + const listed = new Set(inputPaths.filter((path) => path.startsWith(`${testRoot}/`))); + const present = []; + const walk = (relativeDirectory) => { + for (const entry of readdirSync(join(sourceDirectory, relativeDirectory), {withFileTypes: true})) { + const path = `${relativeDirectory}/${entry.name}`; + if (entry.isDirectory()) walk(path); + else present.push(path); + } + }; + walk(testRoot); + const unlisted = present.filter((path) => !listed.has(path)); + if (unlisted.length > 0 || present.length !== listed.size) { + throw new Error("the visual test directory gained or lost files during the run"); + } +} + +function requiredSafePath(name) { + const value = process.env[name]; + if (!value || !isSafeRepoPath(value)) throw new Error(`${name} must be a safe repository-relative path`); + return value; +} diff --git a/tests/visual/capture-inventory.mjs b/tests/visual/capture-inventory.mjs new file mode 100644 index 00000000..612c8a0a --- /dev/null +++ b/tests/visual/capture-inventory.mjs @@ -0,0 +1,27 @@ +import {lstatSync, readdirSync, readFileSync} from "node:fs"; +import {join} from "node:path"; + +import {canonicalRGBADigest, decodePNG} from "./exact-png.mjs"; + +// captureInventory decodes every PNG capture below a capture directory and +// returns its entries sorted by ID (/..). Any +// file that is not a regular PNG capture is refused. +export function captureInventory(directory) { + const entries = []; + const walk = (relative) => { + for (const entry of readdirSync(join(directory, relative), {withFileTypes: true})) { + const path = relative === "" ? entry.name : `${relative}/${entry.name}`; + const metadata = lstatSync(join(directory, path)); + if (metadata.isSymbolicLink()) throw new Error("capture directory must not contain symlinks"); + if (metadata.isDirectory()) { + walk(path); + continue; + } + if (!metadata.isFile() || !path.endsWith(".png")) throw new Error("capture directory may contain only PNG captures"); + const image = decodePNG(readFileSync(join(directory, path))); + entries.push({height: image.height, id: path.slice(0, -".png".length), rgbaSHA256: canonicalRGBADigest(image), width: image.width}); + } + }; + walk(""); + return entries.sort((left, right) => (left.id < right.id ? -1 : left.id > right.id ? 1 : 0)); +} diff --git a/tests/visual/controls.spec.mjs b/tests/visual/controls.spec.mjs new file mode 100644 index 00000000..08582fd0 --- /dev/null +++ b/tests/visual/controls.spec.mjs @@ -0,0 +1,71 @@ +import {expect} from "@playwright/test"; + +import {captureTwice, controlDigest, openView, referenceId, test} from "./visual-harness.mjs"; + +// Controls over a copied workspace stylesheet served in place of the original. +// The routed unchanged copy must keep the admitted digest, so routing alone +// cannot change the pixels; each mutated copy must then yield a canonical +// digest that differs from the admitted digest of the unmutated view. A +// mutation control fails when its mutation is not visible in the captured +// pixels, which also checks that the recorded font environment renders +// weight, family and monospace differences. +const view = "workspace-specifications"; +const scheme = "light"; +const width = 1440; + +async function routeCopiedStylesheet(page, mutate) { + await page.route("**/assets/workspace.css", async (route) => { + const response = await route.fetch(); + const original = await response.text(); + const copy = mutate(original); + if (copy === original) throw new Error("control copy did not change the copied stylesheet"); + await route.fulfill({response, body: copy}); + }); +} + +function replaceOnce(stylesheet, from, to) { + const occurrences = stylesheet.split(from).length - 1; + if (occurrences !== 1) throw new Error(`copied stylesheet must contain exactly one ${JSON.stringify(from)}`); + return stylesheet.replace(from, to); +} + +// Raises the blue channel of the light text color token by one unit. +function shiftTextColorByOneUnit(stylesheet) { + const declarations = [...stylesheet.matchAll(/--text: #([0-9a-f]{6});/g)]; + if (declarations.length === 0) throw new Error("copied stylesheet has no text color token"); + const [declaration, hex] = declarations[0]; + const blue = Number.parseInt(hex.slice(4), 16); + const shifted = hex.slice(0, 4) + (blue === 255 ? blue - 1 : blue + 1).toString(16).padStart(2, "0"); + return stylesheet.replace(declaration, `--text: #${shifted};`); +} + +async function captureControl(page, visualServers, testInfo, browserName, name, copy) { + await routeCopiedStylesheet(page, copy); + await openView(page, visualServers, view, scheme, width); + return controlDigest(testInfo, referenceId(browserName, view, scheme, width), name, await captureTwice(page, {fullPage: true})); +} + +// The copy differs from the original only by a trailing comment. +test("a routed unchanged stylesheet keeps its reference", async ({page, browserName, visualServers}, testInfo) => { + const {actual, admitted} = await captureControl(page, visualServers, testInfo, browserName, "routed-unchanged", (stylesheet) => `${stylesheet}\n/* routed unchanged copy */\n`); + expect(actual, "routing an unchanged stylesheet copy must keep the admitted digest and dimensions").toEqual({height: admitted.height, rgbaSHA256: admitted.rgbaSHA256, width: admitted.width}); +}); + +const controls = [ + ["a one-unit text color change differs from its reference", "text-color-one-unit", shiftTextColorByOneUnit], + ["a one-pixel shift differs from its reference", "one-pixel-shift", (stylesheet) => `${stylesheet}\nhtml { padding-inline-start: 1px; }\n`], + // Every element renders at weight 700, so the text at weight 400 becomes 700 + // and no relative weight such as bolder reaches a heavier synthesized face. + ["a font-weight change from 400 to 700 differs from its reference", "font-weight-700", (stylesheet) => `${stylesheet}\n* { font-weight: 700 !important; }\n`], + // The root family stack ends in sans-serif; the copy makes it serif. + ["a change of the sans-serif family to serif differs from its reference", "serif-family", (stylesheet) => replaceOnce(stylesheet, "font-family: ui-sans-serif, system-ui, sans-serif;", "font-family: serif;")], + // The requirement IDs use the monospace stack; the copy makes them sans-serif. + ["a change of the monospace family to sans-serif differs from its reference", "monospace-family", (stylesheet) => replaceOnce(stylesheet, "font-family: ui-monospace, monospace;", "font-family: sans-serif;")], +]; + +for (const [title, name, mutate] of controls) { + test(title, async ({page, browserName, visualServers}, testInfo) => { + const {actual, admitted} = await captureControl(page, visualServers, testInfo, browserName, name, mutate); + expect(actual.rgbaSHA256, `${name} control capture must differ from the admitted digest`).not.toBe(admitted.rgbaSHA256); + }); +} diff --git a/tests/visual/environment-probe.mjs b/tests/visual/environment-probe.mjs new file mode 100644 index 00000000..9cb76f36 --- /dev/null +++ b/tests/visual/environment-probe.mjs @@ -0,0 +1,180 @@ +import {execFileSync} from "node:child_process"; +import {createHash} from "node:crypto"; +import {existsSync, readFileSync} from "node:fs"; +import {chromium, firefox, webkit} from "@playwright/test"; + +import {stableJSONStringify} from "../../scripts/stable-json.mjs"; + +// Observes the reference-environment tuple without launching any browser. +// Every field the host cannot observe is null, which differs from the +// recorded reference environment and makes the comparison unverified. +// fontConfig is the SHA-256 of the fontconfig file named by FONTCONFIG_FILE, +// which the caller sets to its run directory's copy of tests/visual/fonts.conf +// before observing, so that fc-match and every browser resolve fonts through +// the hashed bytes. fontMatches records the fc-match result for each named +// family queried by its whole name: fc-match reads the text after an +// unescaped hyphen as a point size, so every hyphen is escaped. + +export const comparedEnvironmentFields = Object.freeze(["architecture", "dockerImage", "emulation", "engines", "fontConfig", "fontMatches", "os", "packages"]); +const fontFamilies = ["monospace", "sans-serif", "serif", "system-ui", "ui-monospace"]; +const commandOptions = {encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 10_000, maxBuffer: 1 << 20}; + +export function observeEnvironment({platform = process.platform, run = runCommand, read = readText, environment = process.env} = {}) { + const architecture = run("uname", ["-m"]); + const cpuinfo = platform === "linux" ? read("/proc/cpuinfo") : null; + return { + architecture, + cpu: observeCPU(platform, cpuinfo, run), + dockerImage: observeDockerImage(read), + emulation: observeEmulation(platform, architecture, cpuinfo, run, read), + engines: observeEngines(), + fontConfig: observeFontConfig(environment.FONTCONFIG_FILE), + fontMatches: observeFontMatches(run), + os: platform === "linux" ? observeOS(read) : null, + packages: platform === "linux" ? observePackages(run) : null, + }; +} + +// environmentDivergence returns the compared fields whose observed value +// differs from the reference value, in the fixed field order. +export function environmentDivergence(reference, observed) { + return comparedEnvironmentFields.filter((field) => stableJSONStringify(reference[field] ?? null) !== stableJSONStringify(observed[field] ?? null)); +} + +function runCommand(command, args) { + try { + const output = execFileSync(command, args, commandOptions).trim(); + return output === "" ? null : output; + } catch { + return null; + } +} + +function readText(path) { + try { + return readFileSync(path, "utf8"); + } catch { + return null; + } +} + +function observeOS(read) { + const text = read("/etc/os-release"); + if (text === null) return null; + const fields = Object.fromEntries(text.split("\n").map((line) => line.match(/^([A-Z_]+)=(.*)$/)).filter(Boolean) + .map(([, key, value]) => [key, value.replace(/^"(.*)"$/, "$1")])); + if (!fields.ID || !fields.VERSION_ID) return null; + return {id: fields.ID, versionId: fields.VERSION_ID}; +} + +function observeDockerImage(read) { + const text = read("/ms-playwright/.docker-info"); + if (text === null) return null; + try { + const value = JSON.parse(text); + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const result = {}; + for (const key of Object.keys(value).sort()) { + if (typeof value[key] !== "string" || value[key] === "") return null; + result[key] = value[key]; + } + return Object.keys(result).length === 0 ? null : result; + } catch { + return null; + } +} + +function observePackages(run) { + const output = run("dpkg-query", ["-W", "-f", "${db:Status-Abbrev}\t${Package}\t${Version}\n", "fontconfig", "libfreetype6", "libharfbuzz0b", "fonts-*"]); + if (output === null) return null; + const packages = {}; + for (const line of output.split("\n")) { + const [status, name, version] = line.split("\t"); + if (status?.trim() === "ii" && name && version) packages[name] = version; + } + for (const required of ["fontconfig", "libfreetype6", "libharfbuzz0b"]) { + if (!(required in packages)) return null; + } + return Object.fromEntries(Object.keys(packages).sort().map((name) => [name, packages[name]])); +} + +function observeFontMatches(run) { + const matches = {}; + for (const family of fontFamilies) { + const match = run("fc-match", [family.replaceAll("-", "\\-")]); + if (match === null) return null; + matches[family] = match; + } + return matches; +} + +function observeFontConfig(path) { + if (typeof path !== "string" || path === "") return null; + try { + return `sha256:${createHash("sha256").update(readFileSync(path)).digest("hex")}`; + } catch { + return null; + } +} + +// Installed engine revisions: the revision directory of each executable that +// Playwright reports for the engine. Chromium's headless shell is installed +// at the same revision; a missing executable is unobserved. +function observeEngines() { + const engines = {}; + for (const [name, type] of [["chromium", chromium], ["firefox", firefox], ["webkit", webkit]]) { + let revision = null; + try { + const executable = type.executablePath(); + const match = executable.match(/(?:^|[\\/])(?:chromium|firefox|webkit)-(\d+)[\\/]/); + if (match && existsSync(executable)) revision = match[1]; + } catch { + revision = null; + } + engines[name] = revision; + } + return engines; +} + +function observeEmulation(platform, architecture, cpuinfo, run, read) { + if (platform === "darwin") { + return run("sysctl", ["-in", "sysctl.proc_translated"]) === "1" ? "rosetta" : "none"; + } + if (platform !== "linux" || cpuinfo === null || architecture === null) return null; + const maps = read("/proc/self/maps") ?? ""; + if (/\/rosetta\b|qemu-/.test(maps)) return "translator-mapped"; + const x86CPU = /^vendor_id\s*:/m.test(cpuinfo); + const armCPU = /^CPU implementer\s*:/m.test(cpuinfo); + if (/VirtualApple/.test(cpuinfo)) return "rosetta"; + if (architecture === "aarch64" && !armCPU) return "foreign-cpu"; + if (architecture === "x86_64" && !x86CPU) return "foreign-cpu"; + return "none"; +} + +// The CPU is recorded in every run but not compared. A value the host cannot +// observe is null: x86 instruction-set and hypervisor flags do not exist on +// arm64, where the cpuinfo Features line is recorded instead. +function observeCPU(platform, cpuinfo, run) { + if (platform === "darwin") { + const hypervisor = run("sysctl", ["-in", "kern.hv_vmm_present"]); + return {avx2: null, avx512f: null, features: null, hypervisor: hypervisor === null ? null : hypervisor === "1", model: run("sysctl", ["-n", "machdep.cpu.brand_string"]), vendor: "apple"}; + } + if (cpuinfo === null) return {avx2: null, avx512f: null, features: null, hypervisor: null, model: null, vendor: null}; + const first = (pattern) => cpuinfo.match(pattern)?.[1]?.trim() || null; + const implementer = first(/^CPU implementer\s*:(.*)$/m); + const part = first(/^CPU part\s*:(.*)$/m); + const vendor = first(/^vendor_id\s*:(.*)$/m); + if (vendor === null && implementer !== null) { + return {avx2: null, avx512f: null, features: first(/^Features\s*:(.*)$/m), hypervisor: null, model: part === null ? null : `part ${part}`, vendor: `implementer ${implementer}`}; + } + const flagsLine = first(/^flags\s*:(.*)$/m); + const flags = flagsLine === null ? null : new Set(flagsLine.split(/\s+/).filter(Boolean)); + return { + avx2: flags === null ? null : flags.has("avx2"), + avx512f: flags === null ? null : flags.has("avx512f"), + features: flagsLine, + hypervisor: flags === null ? null : flags.has("hypervisor"), + model: first(/^model name\s*:(.*)$/m), + vendor, + }; +} diff --git a/tests/visual/environment.json b/tests/visual/environment.json new file mode 100644 index 00000000..2e798762 --- /dev/null +++ b/tests/visual/environment.json @@ -0,0 +1,56 @@ +{ + "browserVersions": { + "chromium": "153.0.8010.12", + "firefox": "155.0", + "webkit": "26.6" + }, + "image": { + "indexDigest": "sha256:eff16c30e6f3f4af0a03fa4b706120d5e9b0891c344a27d64559aff5900a4a27", + "name": "mcr.microsoft.com/playwright", + "platform": "linux/arm64", + "platformDigests": { + "linux/amd64": "sha256:bc6ab0d6d44ff4826e4cb8c1e6d801e185bfc42bb0753f8e2a30efc70db054c7", + "linux/arm64": "sha256:a0f4498920a5dbac63196d9140ed738ef00470f27e2e74029abd8850b7bd5717" + }, + "tag": "v1.63.0-noble" + }, + "kind": "proofkit.browser-visual-environment", + "observed": { + "architecture": "aarch64", + "dockerImage": { + "dockerImageName": "mcr.microsoft.com/playwright:v1.63.0-noble", + "driverVersion": "1.63.0" + }, + "emulation": "none", + "engines": { + "chromium": "1243", + "firefox": "1543", + "webkit": "2359" + }, + "fontConfig": "sha256:5813e51e798ed34bdd02105f221499b368814bf181c792b249a579a36f3321e2", + "fontMatches": { + "monospace": "LiberationMono-Regular.ttf: \"Liberation Mono\" \"Regular\"", + "sans-serif": "LiberationSans-Regular.ttf: \"Liberation Sans\" \"Regular\"", + "serif": "LiberationSerif-Regular.ttf: \"Liberation Serif\" \"Regular\"", + "system-ui": "LiberationSans-Regular.ttf: \"Liberation Sans\" \"Regular\"", + "ui-monospace": "LiberationMono-Regular.ttf: \"Liberation Mono\" \"Regular\"" + }, + "os": { + "id": "ubuntu", + "versionId": "24.04" + }, + "packages": { + "fontconfig": "2.15.0-1.1ubuntu2", + "fonts-freefont-ttf": "20211204+svn4273-2", + "fonts-ipafont-gothic": "00303-21ubuntu1", + "fonts-liberation": "1:2.1.5-3", + "fonts-noto-color-emoji": "2.047-0ubuntu0.24.04.1", + "fonts-tlwg-loma-otf": "1:0.7.3-1", + "fonts-unifont": "1:15.1.01-1build1", + "fonts-wqy-zenhei": "0.9.45-8", + "libfreetype6": "2.13.2+dfsg-1ubuntu0.1", + "libharfbuzz0b": "8.3.0-2build2" + } + }, + "schemaVersion": 1 +} diff --git a/tests/visual/exact-png.mjs b/tests/visual/exact-png.mjs new file mode 100644 index 00000000..a84b9ae2 --- /dev/null +++ b/tests/visual/exact-png.mjs @@ -0,0 +1,149 @@ +import {createHash} from "node:crypto"; +import {crc32, inflateSync} from "node:zlib"; + +// PNG decoding and the canonical pixel digest of the visual reference gate. +// Decoding uses only node:zlib; no image library and no tolerance exist here. + +const signature = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const maximumPixels = 1440 * 2400; +const channelsByColorType = new Map([[0, 1], [2, 3], [3, 1], [4, 2], [6, 4]]); + +export function decodePNG(bytes) { + const buffer = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes); + if (buffer.length < signature.length || !buffer.subarray(0, signature.length).equals(signature)) { + throw new Error("image is not a PNG file"); + } + let offset = signature.length; + let header; + let palette; + let transparency; + const data = []; + let ended = false; + while (offset < buffer.length) { + if (offset + 12 > buffer.length) throw new Error("PNG chunk is truncated"); + const length = buffer.readUInt32BE(offset); + const type = buffer.toString("latin1", offset + 4, offset + 8); + const end = offset + 12 + length; + if (length > 0x7fffffff || end > buffer.length) throw new Error("PNG chunk is truncated"); + const body = buffer.subarray(offset + 8, offset + 8 + length); + if (crc32(buffer.subarray(offset + 4, offset + 8 + length)) !== buffer.readUInt32BE(offset + 8 + length)) { + throw new Error("PNG chunk checksum mismatch"); + } + if (header === undefined && type !== "IHDR") throw new Error("PNG must start with IHDR"); + if (type === "IHDR") { + if (header !== undefined) throw new Error("PNG must contain exactly one IHDR"); + header = decodeHeader(body); + } else if (type === "PLTE") palette = body; + else if (type === "tRNS") { + // Transparency of a grayscale or RGB image would change alpha without a + // palette; only palette images may carry tRNS here. + if (header.colorType !== 3) throw new Error("PNG tRNS is admitted only for palette images"); + transparency = body; + } + else if (type === "IDAT") data.push(body); + else if (type === "IEND") { + ended = true; + offset = end; + break; + } else if ((type.charCodeAt(0) & 0x20) === 0) { + throw new Error("PNG contains an unsupported critical chunk"); + } + offset = end; + } + if (!ended || offset !== buffer.length) throw new Error("PNG must end with IEND"); + if (data.length === 0) throw new Error("PNG has no image data"); + if (header.colorType === 3 && palette === undefined) throw new Error("palette PNG has no palette"); + const channels = channelsByColorType.get(header.colorType); + const stride = header.width * channels; + const raw = inflateSync(Buffer.concat(data), {maxOutputLength: (stride + 1) * header.height + 1}); + if (raw.length !== (stride + 1) * header.height) throw new Error("PNG image data has an unexpected length"); + const pixels = unfilter(raw, stride, channels, header.height); + return {width: header.width, height: header.height, rgba: toRGBA(pixels, header, palette, transparency)}; +} + +function decodeHeader(body) { + if (body.length !== 13) throw new Error("PNG header is invalid"); + const width = body.readUInt32BE(0); + const height = body.readUInt32BE(4); + const [bitDepth, colorType, compression, filter, interlace] = body.subarray(8, 13); + if (width === 0 || height === 0 || width * height > maximumPixels) throw new Error("PNG dimensions are outside the admitted bounds"); + if (bitDepth !== 8 || !channelsByColorType.has(colorType)) throw new Error("PNG must be 8-bit grayscale, RGB, palette, grayscale-alpha or RGBA"); + if (compression !== 0 || filter !== 0 || interlace !== 0) throw new Error("PNG must use standard compression, filtering and no interlacing"); + return {width, height, colorType}; +} + +function unfilter(raw, stride, channels, height) { + const output = Buffer.alloc(stride * height); + for (let row = 0; row < height; row += 1) { + const filter = raw[row * (stride + 1)]; + const source = row * (stride + 1) + 1; + const target = row * stride; + for (let index = 0; index < stride; index += 1) { + const value = raw[source + index]; + const left = index >= channels ? output[target + index - channels] : 0; + const up = row > 0 ? output[target - stride + index] : 0; + const upLeft = row > 0 && index >= channels ? output[target - stride + index - channels] : 0; + let predictor; + if (filter === 0) predictor = 0; + else if (filter === 1) predictor = left; + else if (filter === 2) predictor = up; + else if (filter === 3) predictor = (left + up) >> 1; + else if (filter === 4) predictor = paeth(left, up, upLeft); + else throw new Error("PNG row filter is invalid"); + output[target + index] = (value + predictor) & 0xff; + } + } + return output; +} + +function paeth(left, up, upLeft) { + const estimate = left + up - upLeft; + const toLeft = Math.abs(estimate - left); + const toUp = Math.abs(estimate - up); + const toUpLeft = Math.abs(estimate - upLeft); + if (toLeft <= toUp && toLeft <= toUpLeft) return left; + return toUp <= toUpLeft ? up : upLeft; +} + +function toRGBA(pixels, header, palette, transparency) { + const count = header.width * header.height; + const rgba = Buffer.alloc(count * 4); + for (let pixel = 0; pixel < count; pixel += 1) { + let red, green, blue, alpha = 255; + if (header.colorType === 6) { + [red, green, blue, alpha] = pixels.subarray(pixel * 4, pixel * 4 + 4); + } else if (header.colorType === 2) { + [red, green, blue] = pixels.subarray(pixel * 3, pixel * 3 + 3); + } else if (header.colorType === 0) { + red = green = blue = pixels[pixel]; + } else if (header.colorType === 4) { + red = green = blue = pixels[pixel * 2]; + alpha = pixels[pixel * 2 + 1]; + } else { + const entry = pixels[pixel]; + if (entry * 3 + 2 >= palette.length) throw new Error("PNG palette index is out of range"); + [red, green, blue] = palette.subarray(entry * 3, entry * 3 + 3); + if (transparency !== undefined && entry < transparency.length) alpha = transparency[entry]; + } + rgba[pixel * 4] = red; + rgba[pixel * 4 + 1] = green; + rgba[pixel * 4 + 2] = blue; + rgba[pixel * 4 + 3] = alpha; + } + return rgba; +} + +// canonicalRGBADigest is the single code owner of the canonical pixel +// encoding: the 14 ASCII bytes "PROOFKIT-RGBA8", the width and the height as +// big-endian uint32, then the 8-bit non-premultiplied RGBA rows from top to +// bottom with no padding. A capture matches its reference only when this +// digest and the dimensions are equal: no pixel may differ in any channel. +export function canonicalRGBADigest({width, height, rgba}) { + if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0 || rgba.length !== width * height * 4) { + throw new Error("canonical RGBA digest needs positive dimensions and exactly width*height RGBA pixels"); + } + const dimensions = Buffer.alloc(8); + dimensions.writeUInt32BE(width, 0); + dimensions.writeUInt32BE(height, 4); + return createHash("sha256").update("PROOFKIT-RGBA8", "ascii").update(dimensions).update(rgba).digest("hex"); +} diff --git a/tests/visual/font-faces.spec.mjs b/tests/visual/font-faces.spec.mjs new file mode 100644 index 00000000..67df8743 --- /dev/null +++ b/tests/visual/font-faces.spec.mjs @@ -0,0 +1,103 @@ +import {expect} from "@playwright/test"; + +import {openView, test} from "./visual-harness.mjs"; + +// Every product font stack must draw its intended face in every engine. The +// stacks are read from the rendered workspace and static views, so a stack +// change in either style owner is checked. Each stack is then drawn in a probe +// page beside the named Liberation faces, in regular and bold, and identified +// by exact pixel equality. A monospace stack must also advance i and M +// equally, and a sans-serif stack must not. +const stacks = Object.freeze([ + {name: "workspace root", view: "workspace-specifications", selector: ":root", face: "Liberation Sans", monospaced: false}, + {name: "workspace requirement ID", view: "workspace-specifications", selector: "article h3", face: "Liberation Mono", monospaced: true}, + {name: "static root", view: "static-source-cards", selector: ":root", face: "Liberation Sans", monospaced: false}, + {name: "static code", view: "static-source-cards", selector: "code", face: "Liberation Mono", monospaced: true}, + {name: "static record ID", view: "static-source-cards", selector: ".proofkit-id", face: "Liberation Mono", monospaced: true}, +]); +const faces = Object.freeze(["Liberation Sans", "Liberation Serif", "Liberation Mono"]); +const weights = Object.freeze([400, 700]); +const text = "REQ-BROWSER-001 iiii MMMM Wq"; + +// readStacks returns each stack's computed font-family in its rendered view. +async function readStacks(page, visualServers) { + const rendered = []; + for (const view of [...new Set(stacks.map((stack) => stack.view))]) { + await openView(page, visualServers, view, "light", 1440); + for (const stack of stacks.filter((candidate) => candidate.view === view)) { + const family = await page.evaluate((selector) => { + const element = document.querySelector(selector); + return element === null ? null : getComputedStyle(element).fontFamily; + }, stack.selector); + expect(family, `${stack.name} (${stack.selector}) must be rendered in ${view}`).not.toBeNull(); + rendered.push({...stack, family}); + } + } + return rendered; +} + +test("each product font stack draws its intended face", async ({page, visualServers}) => { + const rendered = await readStacks(page, visualServers); + const samples = [ + ...faces.map((face) => ({label: face, family: JSON.stringify(face)})), + ...rendered.map((stack) => ({label: `${stack.name} (${stack.family})`, family: stack.family})), + ]; + const probe = await page.context().newPage(); + try { + await probe.setViewportSize({width: 1000, height: 1200}); + await probe.setContent(""); + await probe.evaluate(({samples, weights, text}) => { + samples.forEach(({family}, index) => { + for (const weight of weights) { + const box = document.createElement("div"); + box.dataset.sample = `${index}-${weight}`; + Object.assign(box.style, {width: "900px", height: "40px", overflow: "hidden", fontSize: "24px", lineHeight: "40px", whiteSpace: "pre", fontFamily: family, fontWeight: String(weight)}); + box.textContent = text; + document.body.append(box); + } + for (const glyphs of ["iiiiiiiiii", "MMMMMMMMMM"]) { + const span = document.createElement("span"); + span.dataset.advance = `${index}-${glyphs[0]}`; + Object.assign(span.style, {fontSize: "40px", whiteSpace: "pre", fontFamily: family}); + span.textContent = glyphs; + document.body.append(span); + } + }); + }, {samples, weights, text}); + const advances = await probe.evaluate(async () => { + await document.fonts.ready; + await new Promise((resolveFrame) => requestAnimationFrame(() => requestAnimationFrame(resolveFrame))); + return Object.fromEntries([...document.querySelectorAll("[data-advance]")].map((span) => [span.dataset.advance, span.getBoundingClientRect().width])); + }); + const capture = (index, weight) => probe.locator(`[data-sample="${index}-${weight}"]`).screenshot({animations: "disabled", caret: "hide", scale: "css"}); + const findings = []; + // The named faces must be installed and distinguishable, regular and bold. + const faceCaptures = new Map(); + for (const weight of weights) { + for (const [index, face] of faces.entries()) faceCaptures.set(`${face} ${weight}`, await capture(index, weight)); + } + const captures = [...faceCaptures.entries()]; + for (const [left, leftBytes] of captures) { + for (const [right, rightBytes] of captures) { + if (left < right && leftBytes.equals(rightBytes)) findings.push(`${left} and ${right} draw identically`); + } + } + for (const [offset, stack] of rendered.entries()) { + const index = faces.length + offset; + for (const weight of weights) { + const bytes = await capture(index, weight); + const drawn = faces.filter((face) => faceCaptures.get(`${face} ${weight}`).equals(bytes)); + if (drawn.length !== 1 || drawn[0] !== stack.face) { + findings.push(`${samples[index].label} at weight ${weight} draws ${drawn.length === 0 ? "no named face" : drawn.join(" and ")}, not ${stack.face}`); + } + } + const monospaced = advances[`${index}-i`] === advances[`${index}-M`]; + if (monospaced !== stack.monospaced) { + findings.push(`${samples[index].label} is ${monospaced ? "monospaced" : "proportional"}, not ${stack.monospaced ? "monospaced" : "proportional"}`); + } + } + expect(findings, "every product font stack must draw its intended face").toEqual([]); + } finally { + await probe.close(); + } +}); diff --git a/tests/visual/fonts.conf b/tests/visual/fonts.conf new file mode 100644 index 00000000..5c8df6bf --- /dev/null +++ b/tests/visual/fonts.conf @@ -0,0 +1,24 @@ + + + + + /etc/fonts/fonts.conf + sans-serifLiberation Sans + system-uiLiberation Sans + serifLiberation Serif + monospaceLiberation Mono + ui-monospaceLiberation Mono + diff --git a/tests/visual/readme-scene.spec.mjs b/tests/visual/readme-scene.spec.mjs new file mode 100644 index 00000000..15800703 --- /dev/null +++ b/tests/visual/readme-scene.spec.mjs @@ -0,0 +1,28 @@ +import {expect} from "@playwright/test"; + +import {assertEmulation, assertMatchesReference, captureTwice, navigate, readmeViewport, test} from "./visual-harness.mjs"; + +const question = "Which negative case should reject a request without an idempotency key?"; + +// The README scene: the committed example.dispatch project served through the +// project view path, its first invariant selected and a question typed into +// the inspector, in a 1440x960 light window. Only the Chromium project runs it. +test("readme-scene light 1440x960", async ({page, browserName, visualServers}, testInfo) => { + expect(browserName).toBe("chromium"); + await page.setViewportSize(readmeViewport); + await navigate(page, await visualServers.url("--readme-project"), "workspace", "example.dispatch", "light"); + await expect(page.locator("body")).toHaveAttribute("data-state", "specifications"); + await expect(page.locator("#workspace-content")).toHaveAttribute("aria-busy", "false"); + await page.getByRole("button", {name: "Select invariant"}).first().click(); + const questionInput = page.locator("#annotation-question"); + await questionInput.fill(question); + await expect(questionInput).toHaveValue(question); + await page.mouse.move(readmeViewport.width - 1, readmeViewport.height - 1); + await page.evaluate(async () => { + await document.fonts.ready; + await new Promise((resolveFrame) => requestAnimationFrame(() => requestAnimationFrame(resolveFrame))); + }); + await assertEmulation(page, "light"); + const captures = await captureTwice(page, {fullPage: false}); + assertMatchesReference(testInfo, "chromium/readme-scene.light.1440", captures); +}); diff --git a/tests/visual/references.json b/tests/visual/references.json new file mode 100644 index 00000000..f99279b5 --- /dev/null +++ b/tests/visual/references.json @@ -0,0 +1,1022 @@ +{ + "clipHeight": 2400, + "environmentId": "sha256:70a8d9e8054360d159fbb5bfff68feaf53b48d20df98b5c72e7dcc5bb84cedd7", + "kind": "proofkit.browser-visual-references", + "references": [ + { + "height": 960, + "id": "chromium/readme-scene.light.1440", + "rgbaSHA256": "9052880808daa4dfaab00c20a4aa6ca84407e4135e583ec3fff94e45ffb0d1b9", + "width": 1440 + }, + { + "height": 1918, + "id": "chromium/static-coverage-cards.dark.1440", + "rgbaSHA256": "aea257f7b695563f015cbea1a62729bd6ab07aa263159b5d87c7ed7af4f64379", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-coverage-cards.dark.320", + "rgbaSHA256": "8ca3f44a38b39b05b4a8222cb2a347d042558dbbc98d115d30a7da45f189acd3", + "width": 320 + }, + { + "height": 1918, + "id": "chromium/static-coverage-cards.light.1440", + "rgbaSHA256": "17ddacebcc73bf0fc052da83825156d358baa174852fc0ad78297b5046e67582", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-coverage-cards.light.320", + "rgbaSHA256": "17f489d5c15b8dfec54b0cd580dd6aeb949a0b0b73ad667c482e38a3e0bea7bc", + "width": 320 + }, + { + "height": 1548, + "id": "chromium/static-coverage-table.dark.1440", + "rgbaSHA256": "424fa2d144afc0a53ea865d67da03175d20c6d957eca685ef59ad37f6913d0f9", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-coverage-table.dark.320", + "rgbaSHA256": "9e16299e0e23420d732741be74de738fc6cb488256c31339f0b048f196f79654", + "width": 320 + }, + { + "height": 1548, + "id": "chromium/static-coverage-table.light.1440", + "rgbaSHA256": "91784702bc12307ba18bf243fd76253c83f50e8a974b412cf5dd5dbd7daf2c8b", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-coverage-table.light.320", + "rgbaSHA256": "bfd61a5abc3b91d1ad52e94647ad152e2517f6b1d1a76f8f7499d7011403ce71", + "width": 320 + }, + { + "height": 1682, + "id": "chromium/static-proof-compact-cards.dark.1440", + "rgbaSHA256": "e9a6a5a92a9242ff820e70813bb222263e3bf7ecaec0f4c404b14cef6c35d78a", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-proof-compact-cards.dark.320", + "rgbaSHA256": "77a51e0a96839cbaf0592cff59677d9d2301e784535bd45f6a5a5aaae8b8c46f", + "width": 320 + }, + { + "height": 1682, + "id": "chromium/static-proof-compact-cards.light.1440", + "rgbaSHA256": "852bc05d40da4a8bf2055cbbcac05f4f6e222f7ddb6b022d373a936f29d9dc04", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-proof-compact-cards.light.320", + "rgbaSHA256": "6c330483b4d0e40b3d58611e89c3f303dfa81f365ffc0ac44cb644c719720f5a", + "width": 320 + }, + { + "height": 1409, + "id": "chromium/static-proof-compact-table.dark.1440", + "rgbaSHA256": "7a049c4dca16133474ae3c83d4d3cba8ad7f56453f9eb1b1a35fcc523d1ecfbf", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-proof-compact-table.dark.320", + "rgbaSHA256": "46adcbdd6d1078841ee87aaed193b9897c9139deb874e5b4bb6bd42a6ee77079", + "width": 320 + }, + { + "height": 1409, + "id": "chromium/static-proof-compact-table.light.1440", + "rgbaSHA256": "8238ea1eb4b092c79bd1f30012b972ad8f3c420d3747247aad13a71e5df0bcb0", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-proof-compact-table.light.320", + "rgbaSHA256": "9a55ea5e04c1fb9e08a69c1742ebd0e89b3b93cf53fd29439f45c41db22b168d", + "width": 320 + }, + { + "height": 1535, + "id": "chromium/static-proof-structured-cards.dark.1440", + "rgbaSHA256": "e5a44cc43cb632d6347cac29b401bcf21a2ed1538389618caf5504bb510c059d", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-proof-structured-cards.dark.320", + "rgbaSHA256": "8f573932ec123afbe96b51ca29aa97fd55ade5bbd820861845fb1e89a5813a63", + "width": 320 + }, + { + "height": 1535, + "id": "chromium/static-proof-structured-cards.light.1440", + "rgbaSHA256": "7bdf9b90cfe579bf38a3d89a134eacf250ebe730e4e9411cecbb0d68d1ebd7c7", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-proof-structured-cards.light.320", + "rgbaSHA256": "85d6ac88793f30a17443a9a55f6ef48410d5c47723945c48de38cc82656ba759", + "width": 320 + }, + { + "height": 1176, + "id": "chromium/static-proof-structured-table.dark.1440", + "rgbaSHA256": "4356b9d26b97ba5dfaed137ef2c191c7505e1ef8989ea06510c2b13da95952fc", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-proof-structured-table.dark.320", + "rgbaSHA256": "ce1c0194cd991130e59e223ad5e6d96ec9aa60ebd6bb421c22ab2bb64b6f6122", + "width": 320 + }, + { + "height": 1176, + "id": "chromium/static-proof-structured-table.light.1440", + "rgbaSHA256": "f491d954faa1bded7ae2f7e15033e8e8b8bc40c4290e2dbeeb571cd576c45921", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-proof-structured-table.light.320", + "rgbaSHA256": "aab05118acb3ec8fc8dc0300d1b52b141093e088029e2ef9f21558933a4471e3", + "width": 320 + }, + { + "height": 2400, + "id": "chromium/static-source-cards.dark.1440", + "rgbaSHA256": "585d260ed4d6faeaa2baeaaa366c6f52040df74d7d0161901a6a699267213401", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-source-cards.dark.320", + "rgbaSHA256": "8e8226cff3d0a3a83a64763e02272af567168dc8407a0514bc076ad6daf562b1", + "width": 320 + }, + { + "height": 2400, + "id": "chromium/static-source-cards.light.1440", + "rgbaSHA256": "c8a96ac517e82e3cdb18964a19bd29a9ac45025c6e4ecb9d446bfe833d1442c6", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-source-cards.light.320", + "rgbaSHA256": "e3d078a5cf37cde05972ca86516112894d358d07aa3409639b9fb5ed8977cb98", + "width": 320 + }, + { + "height": 1961, + "id": "chromium/static-source-table.dark.1440", + "rgbaSHA256": "0e2d8aadd6e3cd1af0de9c7c0bcec90e44a4c703bfbf50e7d103d50e8a5a9519", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-source-table.dark.320", + "rgbaSHA256": "77d2a88f5fde4e00d900a4f15cbaf17b3e1e97c51d85cb599e86cea1dea0b1e8", + "width": 320 + }, + { + "height": 1961, + "id": "chromium/static-source-table.light.1440", + "rgbaSHA256": "d8707cea4dbbd96ceb87c1ee906aa14e8222d87fac8ebbf663631bd278d9c067", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-source-table.light.320", + "rgbaSHA256": "152f39a809df4e3e8cd262a16b4851340b6d0d30190b052eaecd70086ef8d7fe", + "width": 320 + }, + { + "height": 1625, + "id": "chromium/static-spec-tree-cards.dark.1440", + "rgbaSHA256": "1bb776b91a0c3d93c6e8705ef00d0b45a80759e9487aeda41ce90588d4dd663e", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-spec-tree-cards.dark.320", + "rgbaSHA256": "66d1dc992771e0a899cdb31550571352dfbd72e0e0482558a0348b839b9ae936", + "width": 320 + }, + { + "height": 1625, + "id": "chromium/static-spec-tree-cards.light.1440", + "rgbaSHA256": "33d034f491aac0126142c9fe297c789239fd01457b4a5003d7530247081a2464", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-spec-tree-cards.light.320", + "rgbaSHA256": "cf688885cad196898988495ba6653c7c9f351b978118846fc1fcdba693ef1061", + "width": 320 + }, + { + "height": 1375, + "id": "chromium/static-spec-tree-table.dark.1440", + "rgbaSHA256": "f330e75b8282e90ef3e8cecc2a68d397a3b4c8cecb9d51550d40891893d53f7c", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-spec-tree-table.dark.320", + "rgbaSHA256": "366418789834a3cc6d466496a337575d34674de718c78b369b260828ce5c7348", + "width": 320 + }, + { + "height": 1375, + "id": "chromium/static-spec-tree-table.light.1440", + "rgbaSHA256": "f45f4a01dd73420751ed3539ac56442c2dfc45af29e751f7d188a8191b24ef52", + "width": 1440 + }, + { + "height": 2400, + "id": "chromium/static-spec-tree-table.light.320", + "rgbaSHA256": "6f8fdc9e0e5e43ff2018af0f2112af0ae733ffc7d181e47fd6ccb129ea881b0b", + "width": 320 + }, + { + "height": 973, + "id": "chromium/workspace-coverage.dark.1440", + "rgbaSHA256": "f4e634ffb9567e2ade0bbb1a2cac049effcce0bd1c8e9c8cbed22039bf35b4c7", + "width": 1440 + }, + { + "height": 1245, + "id": "chromium/workspace-coverage.dark.320", + "rgbaSHA256": "36fe0181ded1645a25f269673d7bc23d8de6cdbcef8ec2b909c6d958f5c244b1", + "width": 320 + }, + { + "height": 973, + "id": "chromium/workspace-coverage.light.1440", + "rgbaSHA256": "5819a8e77335e77927f8844af4cfcf79c07d41b0e8ba097ab111d0466253c79a", + "width": 1440 + }, + { + "height": 1245, + "id": "chromium/workspace-coverage.light.320", + "rgbaSHA256": "c5185570cd5bddd4d8862fefca5ec6b93960d7ebf5fe48d7d89cb4357357895b", + "width": 320 + }, + { + "height": 900, + "id": "chromium/workspace-diff.dark.1440", + "rgbaSHA256": "bcff1044fe77c9cd9cea5ea4138ef7f5e00b934d4e3a0e26c0590ef402c3925e", + "width": 1440 + }, + { + "height": 1281, + "id": "chromium/workspace-diff.dark.320", + "rgbaSHA256": "d09da6908258f21b3c343c066a7fa2e063e061cd73f268d2850de173c8f4f5b8", + "width": 320 + }, + { + "height": 900, + "id": "chromium/workspace-diff.light.1440", + "rgbaSHA256": "bef512ce700a63dc24a90eb5af8d77cc52ea16686a02b0e2872660933edba7cc", + "width": 1440 + }, + { + "height": 1281, + "id": "chromium/workspace-diff.light.320", + "rgbaSHA256": "4b8f0b97f76c5b165bcb1e4e4a6b51cdd5d1103b9556bd1fc2f6bfe745aa9d37", + "width": 320 + }, + { + "height": 900, + "id": "chromium/workspace-specifications.dark.1440", + "rgbaSHA256": "523693ce5642851139b9a75f83cf14c672448624672b727c91ce7fd20ec4289b", + "width": 1440 + }, + { + "height": 905, + "id": "chromium/workspace-specifications.dark.320", + "rgbaSHA256": "30c8b7814d46093e8714b9964c6de366dbe78fc2ab8e9bfae4e7995081ebe93b", + "width": 320 + }, + { + "height": 900, + "id": "chromium/workspace-specifications.light.1440", + "rgbaSHA256": "ea3a6ce0862006589b81fc7941741c85ef0c6cb7dd2ebc6c602e53cee6d69918", + "width": 1440 + }, + { + "height": 905, + "id": "chromium/workspace-specifications.light.320", + "rgbaSHA256": "0a1bdc63fef4ec6c5ca228e58f930626aac06f3c08debcd0e6cb046d80df0e91", + "width": 320 + }, + { + "height": 2041, + "id": "chromium/workspace-traceability.dark.1440", + "rgbaSHA256": "42b84851e529537c10f707a5148830f17293a0cb756d4279ae7bd8268e264069", + "width": 1440 + }, + { + "height": 2206, + "id": "chromium/workspace-traceability.dark.320", + "rgbaSHA256": "6c036648f76ca676008043de3532c398d1e57da4bdb583a34ec9885ae9cc4417", + "width": 320 + }, + { + "height": 2041, + "id": "chromium/workspace-traceability.light.1440", + "rgbaSHA256": "6c903c632b394a701a93543425f0a2fef665102d8c9a3aa65c27120d8ec9df97", + "width": 1440 + }, + { + "height": 2206, + "id": "chromium/workspace-traceability.light.320", + "rgbaSHA256": "9b6b3d28b6ee6d6935218f9a2aeb3d2f61efd3c0e2db1be393406d1821504201", + "width": 320 + }, + { + "height": 1928, + "id": "firefox/static-coverage-cards.dark.1440", + "rgbaSHA256": "0fca71b50b9a5798c81282eb44d673fd9c61f7cd0d676482ef92ab9f79988b24", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-coverage-cards.dark.320", + "rgbaSHA256": "357c5d64d311d4bb6399cf56dce37f320b089c82de211fb7fce18306a4f9ad91", + "width": 320 + }, + { + "height": 1928, + "id": "firefox/static-coverage-cards.light.1440", + "rgbaSHA256": "cedf63daa472e63fadb11fd4a429b02710978a929ff0fcf0245cfb1ff4e37d95", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-coverage-cards.light.320", + "rgbaSHA256": "c2ebde73b4a18ff07da923e965fdd118400a2a35ea37d30f5ee097fa3de97b0a", + "width": 320 + }, + { + "height": 1556, + "id": "firefox/static-coverage-table.dark.1440", + "rgbaSHA256": "3b8630df1bed65bbc0fe0b00ee991d7d84b3c424b8c0740397e67ff9e9e284c8", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-coverage-table.dark.320", + "rgbaSHA256": "3ae89ff4616a7ed0334d8becdb46285a278792e5f8fbcc7f429332a542383160", + "width": 320 + }, + { + "height": 1556, + "id": "firefox/static-coverage-table.light.1440", + "rgbaSHA256": "b118f66fa88d9d703bc7d2d7b168c8ce60880d5a7bf5ac3d13f070b3b841ad0d", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-coverage-table.light.320", + "rgbaSHA256": "1f89ed2b167b0a3f7f7762de78282a7788a1b24bac382760e7c3297b285b7ce3", + "width": 320 + }, + { + "height": 1689, + "id": "firefox/static-proof-compact-cards.dark.1440", + "rgbaSHA256": "80a57c8d7f30f35dfaf34e7741091191692a3c2b966410c5e78066bbdf904391", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-proof-compact-cards.dark.320", + "rgbaSHA256": "c21314ddbcd08555ad5f6a74350c9600e69d8990962d1adaff18bf7ef5efcaa7", + "width": 320 + }, + { + "height": 1689, + "id": "firefox/static-proof-compact-cards.light.1440", + "rgbaSHA256": "280a7ae630aaaa5d4cde8cb5364863b40946c83f43d19f34cb93b855887631c5", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-proof-compact-cards.light.320", + "rgbaSHA256": "998094bc8f6d4b3f64b1f4fff0134d8700a87a1d52506864126cd6b5d88a141d", + "width": 320 + }, + { + "height": 1416, + "id": "firefox/static-proof-compact-table.dark.1440", + "rgbaSHA256": "1e155fe775caf294d7be6aa3676316d2ed4f2fbb709069b030d726b07b4ac361", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-proof-compact-table.dark.320", + "rgbaSHA256": "487f042a2cb12a178290a35304ad2e181ad1ff2ea6ea95ae30744670730278fa", + "width": 320 + }, + { + "height": 1416, + "id": "firefox/static-proof-compact-table.light.1440", + "rgbaSHA256": "f4d0b122755485b8fbe92d2903347fbd717be0b3e76d8a21ed339501e2ab7089", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-proof-compact-table.light.320", + "rgbaSHA256": "5f4d44c665ae7af69ca999043316487bba8ed26b50a6bf25315e134abc2b0c21", + "width": 320 + }, + { + "height": 1542, + "id": "firefox/static-proof-structured-cards.dark.1440", + "rgbaSHA256": "aeb30db9d14bded83be089b2d10327273fb2fa96ed55c5add13da132d287ab92", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-proof-structured-cards.dark.320", + "rgbaSHA256": "29e62d9fce0a4078280e6414abf095fb1161248e38ff6d1b68e5ece5e8a52530", + "width": 320 + }, + { + "height": 1542, + "id": "firefox/static-proof-structured-cards.light.1440", + "rgbaSHA256": "134ef17b455a98635db08cc402d9289126a169822159ba3c2e815b95ed5bdb73", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-proof-structured-cards.light.320", + "rgbaSHA256": "ebebbc982177d1b77113d46d88fb9022739fa3820837640f1121380855482eb7", + "width": 320 + }, + { + "height": 1180, + "id": "firefox/static-proof-structured-table.dark.1440", + "rgbaSHA256": "eebd1e214d1916a32bf397d6d695c1e7aad2882344353632477b215be6d99a91", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-proof-structured-table.dark.320", + "rgbaSHA256": "beeea4f1702efbb8b82dad055a2ac3d5bec1fbeb655b8c26451f9740ce84ef4e", + "width": 320 + }, + { + "height": 1180, + "id": "firefox/static-proof-structured-table.light.1440", + "rgbaSHA256": "138e24162fd36d59e3bce4088c56c3b77d8678507cad545e5659f2da2268b0ce", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-proof-structured-table.light.320", + "rgbaSHA256": "6006f678e7b90ccf6d0fdfb9a6b36ba8b0547c80aa8d2d21602978519f455d4e", + "width": 320 + }, + { + "height": 2400, + "id": "firefox/static-source-cards.dark.1440", + "rgbaSHA256": "cb54c9b6d3c89f0b89525c87cc3764f1d92e68c32244a46d153c6cb278a971c3", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-source-cards.dark.320", + "rgbaSHA256": "9d4136c2a2921be6af7ae52c576ae5894b136fdc6216e399269715e6230c4729", + "width": 320 + }, + { + "height": 2400, + "id": "firefox/static-source-cards.light.1440", + "rgbaSHA256": "f2a9c8200852424f4987e1c3655a4fca651ba82a8508f0cdced6142128bb900a", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-source-cards.light.320", + "rgbaSHA256": "31b5bff495979a86337553e01fe52c920e605b437eec6b2deda1712fa5fb5202", + "width": 320 + }, + { + "height": 2089, + "id": "firefox/static-source-table.dark.1440", + "rgbaSHA256": "6e60ab0800f055cd50dd5bb20ec8309e39a2c128c2386cd48421a82c925ed90d", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-source-table.dark.320", + "rgbaSHA256": "9d4136c2a2921be6af7ae52c576ae5894b136fdc6216e399269715e6230c4729", + "width": 320 + }, + { + "height": 2089, + "id": "firefox/static-source-table.light.1440", + "rgbaSHA256": "7de658cdaf62079f9e84109ba4bce50348b9ce9fc3599b300bbeaa1abedc937c", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-source-table.light.320", + "rgbaSHA256": "31b5bff495979a86337553e01fe52c920e605b437eec6b2deda1712fa5fb5202", + "width": 320 + }, + { + "height": 1633, + "id": "firefox/static-spec-tree-cards.dark.1440", + "rgbaSHA256": "fcaa948e629e289ab68c98b67b020e78102d5182330113274abae64e3468d4e8", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-spec-tree-cards.dark.320", + "rgbaSHA256": "75a3c1bd351b4d68fbb053f9a62042c2b453dfb2fa42c4cb149a48d918539713", + "width": 320 + }, + { + "height": 1633, + "id": "firefox/static-spec-tree-cards.light.1440", + "rgbaSHA256": "8c08cbccea18e56c9c1b6676296ee5d6a4e58cb252da059d3153516d643c4b2f", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-spec-tree-cards.light.320", + "rgbaSHA256": "222865c55d23c814d45f17e5ab7e4edd9861405d24499d36475f0cc44ef23932", + "width": 320 + }, + { + "height": 1382, + "id": "firefox/static-spec-tree-table.dark.1440", + "rgbaSHA256": "90f8bf7fe1a3078e4d88af884a9a0bd32d8acab55dae745ae3c98af5f72a336d", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-spec-tree-table.dark.320", + "rgbaSHA256": "f2ac36bf5b3f1e883a42a482d4f64a6d80793c51365947f5729bb528fc7b50e7", + "width": 320 + }, + { + "height": 1382, + "id": "firefox/static-spec-tree-table.light.1440", + "rgbaSHA256": "9da1f9c8fa9fc60b4bbdfa2f51470aad11be4016bf56b2afd83e5acb9360a397", + "width": 1440 + }, + { + "height": 2400, + "id": "firefox/static-spec-tree-table.light.320", + "rgbaSHA256": "e0e8add2323516ffd563a182626a02ada9d279d7f1f7bb2e188528a284aa1d6b", + "width": 320 + }, + { + "height": 976, + "id": "firefox/workspace-coverage.dark.1440", + "rgbaSHA256": "773c1490c72328260aa2ec21d7ae4e63ef7febd53263d4c0a4811e325343f8e3", + "width": 1440 + }, + { + "height": 1248, + "id": "firefox/workspace-coverage.dark.320", + "rgbaSHA256": "5d0c0dcc1114d5290fb77e84d8591f2614c680fcb8f600eae94becfe0e6836c1", + "width": 320 + }, + { + "height": 976, + "id": "firefox/workspace-coverage.light.1440", + "rgbaSHA256": "1d65819575f83143fdee476507ec4e1f6567ad38392c7aac23ef4a4b14ded888", + "width": 1440 + }, + { + "height": 1248, + "id": "firefox/workspace-coverage.light.320", + "rgbaSHA256": "c9a5d4930b298fd2d1da8170a76a54141cd3c9a35b2308a8449557a2c1e195e6", + "width": 320 + }, + { + "height": 900, + "id": "firefox/workspace-diff.dark.1440", + "rgbaSHA256": "cd3a480372a36260889d8bdcb1f821582fc8bf08c2f41690a674cded19158778", + "width": 1440 + }, + { + "height": 1284, + "id": "firefox/workspace-diff.dark.320", + "rgbaSHA256": "8c037ba0f5ae90f9a7e5980828cbf0bd3ae714556867b2f11ef5229bd298f9d5", + "width": 320 + }, + { + "height": 900, + "id": "firefox/workspace-diff.light.1440", + "rgbaSHA256": "18e4011939d3c254305bfdfc24910fa566dbff1cb757bbd56b8fab276012e170", + "width": 1440 + }, + { + "height": 1284, + "id": "firefox/workspace-diff.light.320", + "rgbaSHA256": "83e21870706b6a2c27e4f8ebd1413457480b9959a547cd461caefc6f241b6f7e", + "width": 320 + }, + { + "height": 900, + "id": "firefox/workspace-specifications.dark.1440", + "rgbaSHA256": "45632c7557d786044d4781acd1ace06c3eee1260603855e2ce79a996b9746192", + "width": 1440 + }, + { + "height": 908, + "id": "firefox/workspace-specifications.dark.320", + "rgbaSHA256": "0d939edf2fe47b481cef946fedfaa1cd730d5fd5f31099ae2011db2296b900f4", + "width": 320 + }, + { + "height": 900, + "id": "firefox/workspace-specifications.light.1440", + "rgbaSHA256": "3036d7295a75307b99967bd6d9d2b9204f33ce0a30574f437bca3826be8231eb", + "width": 1440 + }, + { + "height": 908, + "id": "firefox/workspace-specifications.light.320", + "rgbaSHA256": "69351aa02aa6439c4c9529cbdcf9f96b1375909d7e9883cc98d8ac24d6916960", + "width": 320 + }, + { + "height": 2048, + "id": "firefox/workspace-traceability.dark.1440", + "rgbaSHA256": "749d598d57d4097fe9aea5702d44aa3e3f7339357be3ada67b6104f26ceb4e3c", + "width": 1440 + }, + { + "height": 2256, + "id": "firefox/workspace-traceability.dark.320", + "rgbaSHA256": "213d177a68936200f79f099489b1c674530aa3c1fffcaf578259a386fa0e352d", + "width": 320 + }, + { + "height": 2048, + "id": "firefox/workspace-traceability.light.1440", + "rgbaSHA256": "29a55b10723228ca307802a02748413bf221a7c7fd50ed3c3e3ceea222350ef9", + "width": 1440 + }, + { + "height": 2256, + "id": "firefox/workspace-traceability.light.320", + "rgbaSHA256": "af3aa365960d1ab790bfdfbbc18dc0aa5539e3d8151857945a15157edda42b1f", + "width": 320 + }, + { + "height": 1938, + "id": "webkit/static-coverage-cards.dark.1440", + "rgbaSHA256": "eac55269a6b7a563848b9a6bd51c001a3503a350cb38aa4a6e972cfb775c1b37", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-coverage-cards.dark.320", + "rgbaSHA256": "52e9b90341fec93f188f53833a73ce1a0a494abc6b064906f004ef6149a2032f", + "width": 320 + }, + { + "height": 1938, + "id": "webkit/static-coverage-cards.light.1440", + "rgbaSHA256": "d7162cb14916459882c41548a8ab88e41e1d1c25fb8a945b893e32219dd90c1e", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-coverage-cards.light.320", + "rgbaSHA256": "ecc81b2895a5db746137254c0a74b23328c17cf7f8c29f297545ebd49a2cf57f", + "width": 320 + }, + { + "height": 1571, + "id": "webkit/static-coverage-table.dark.1440", + "rgbaSHA256": "98abce2e60ac23d3feafe192fbc33ee74868d54a2b9c60919b33abc520015a40", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-coverage-table.dark.320", + "rgbaSHA256": "2233ad64338b9e5925b1e1f2e061da061e740ae2917a53f52a67f570eff611f9", + "width": 320 + }, + { + "height": 1571, + "id": "webkit/static-coverage-table.light.1440", + "rgbaSHA256": "18b9df82cac5e27a87ac6795732d067baab20e76a1188e082ed85d49d02c2963", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-coverage-table.light.320", + "rgbaSHA256": "d09ebdbfbebcb899a905e638c649f340a3e0c6fc7f17b63cbc97a51777695a35", + "width": 320 + }, + { + "height": 1703, + "id": "webkit/static-proof-compact-cards.dark.1440", + "rgbaSHA256": "6e3bbe2306c29e1bd553f225d21ad3523a28c44fb7119a2f84af7235fd28202f", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-proof-compact-cards.dark.320", + "rgbaSHA256": "beede2ed8aa3c40337f2bba34bb460a41aa96e08cc890a472bdabb0e3248e7e2", + "width": 320 + }, + { + "height": 1703, + "id": "webkit/static-proof-compact-cards.light.1440", + "rgbaSHA256": "6e1e54d31ca3b48983d8db19f1a9b73a658e41814ebf7cbb9508fbddeb5b01eb", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-proof-compact-cards.light.320", + "rgbaSHA256": "266ffe402710f00476deee68abe330a03c31d3c80b6285eebf651b89be706a36", + "width": 320 + }, + { + "height": 1485, + "id": "webkit/static-proof-compact-table.dark.1440", + "rgbaSHA256": "463736a2f0c533527b817705a9eabfabc40a22dcc2af82c36d3925db2011f34d", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-proof-compact-table.dark.320", + "rgbaSHA256": "c26cad10b2281eebb7aa493d42cebb822016b7b83dec89d141bc6386eb35c82c", + "width": 320 + }, + { + "height": 1485, + "id": "webkit/static-proof-compact-table.light.1440", + "rgbaSHA256": "2f259bc85bbf8071de3e115f4b0f6721109f244469cb9751436e2716c7a345ee", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-proof-compact-table.light.320", + "rgbaSHA256": "62df58ac12d5ebcc385e230edd8eafa42464e36daa4a57acadfc790b1be2d0c4", + "width": 320 + }, + { + "height": 1556, + "id": "webkit/static-proof-structured-cards.dark.1440", + "rgbaSHA256": "aa1c645ee3b0b6f1ab1bbcb1941914c84650b566c2dfe7031e2d8f25bb51a46f", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-proof-structured-cards.dark.320", + "rgbaSHA256": "d9be4eef29ca4d37eda8ba294adbbca096ba0c1f9d3fc6fdd8c005fb6962b8b2", + "width": 320 + }, + { + "height": 1556, + "id": "webkit/static-proof-structured-cards.light.1440", + "rgbaSHA256": "081cc73d397b3af26c144e30ffc6c4363b3bdad1c7b0087a6bfb021cc76069d6", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-proof-structured-cards.light.320", + "rgbaSHA256": "22798f81096aac2c7d8218ea39c9808a1f180fa9008ff9fd91007ff1681d6d9f", + "width": 320 + }, + { + "height": 1199, + "id": "webkit/static-proof-structured-table.dark.1440", + "rgbaSHA256": "308fcad92879fdbd8be6183c05ec46fdb9b54ecb58ecbf82eba462e9a9fe980c", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-proof-structured-table.dark.320", + "rgbaSHA256": "9e32ec583c13f3851762d153ad6c483cc6d6fc4d7717a463d42281d0b60312c7", + "width": 320 + }, + { + "height": 1199, + "id": "webkit/static-proof-structured-table.light.1440", + "rgbaSHA256": "25df37662192cc3aded5a8fc69b6e9e882b5c362f96a679a2265527424086368", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-proof-structured-table.light.320", + "rgbaSHA256": "bbfd98713c32381c6eb09395422e11666819fd07beb94ca40899a2fa08c9b913", + "width": 320 + }, + { + "height": 2400, + "id": "webkit/static-source-cards.dark.1440", + "rgbaSHA256": "2fbd10216cec8505515787ae2a3d37ea8db8c14e216e3a1048a3bb3b78b888ed", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-source-cards.dark.320", + "rgbaSHA256": "8e6cdf99ccaab8fa5e219ff9ec4dfe3b73eb3b9ae1f04fd953aa8dda7910f591", + "width": 320 + }, + { + "height": 2400, + "id": "webkit/static-source-cards.light.1440", + "rgbaSHA256": "3ad57ba23adbf2f3871d024fc4614c18f89489c9a1be6b2c8ba736c95db7a0e7", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-source-cards.light.320", + "rgbaSHA256": "0b078f3ba06f7b52f517c0f18218eea285895cf26ea44955cfd164baaa972ca2", + "width": 320 + }, + { + "height": 1983, + "id": "webkit/static-source-table.dark.1440", + "rgbaSHA256": "db9719f5bca2a64e78b43a1f51ba1820d4fbac87365e20cc5b54b338be974e28", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-source-table.dark.320", + "rgbaSHA256": "2cb93a0cce75844dceff4a6957dd36d7bc2f233f80b9b85d63534af4b5990f6e", + "width": 320 + }, + { + "height": 1983, + "id": "webkit/static-source-table.light.1440", + "rgbaSHA256": "7d80f8158864571c61ea75217aba30dd126eecbe6c4b19628452439ae998d599", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-source-table.light.320", + "rgbaSHA256": "2c29cb9e0eae186fc3b6c5d5e78ca06a0e63635b2f410115374d9c572234c694", + "width": 320 + }, + { + "height": 1648, + "id": "webkit/static-spec-tree-cards.dark.1440", + "rgbaSHA256": "55e9200467dacf50fc082aa1bc8c39cb7433921f109e63dfce30423da2244afc", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-spec-tree-cards.dark.320", + "rgbaSHA256": "e566bbb7a4f41e4c2d45ed84cd1afa16a0ce5119d6a81e7a3f09ee466aaf3d41", + "width": 320 + }, + { + "height": 1648, + "id": "webkit/static-spec-tree-cards.light.1440", + "rgbaSHA256": "e14affd991f238f7181b89f0624a47eab39a9051dabc683bdf9395258ccf7ac6", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-spec-tree-cards.light.320", + "rgbaSHA256": "c8d91328fd8e68c6d7d9070bdadc5f97d99abd59365b0bfb63d2490fe70ea5bb", + "width": 320 + }, + { + "height": 1400, + "id": "webkit/static-spec-tree-table.dark.1440", + "rgbaSHA256": "de7a3e2cd1b54a10ce2ae6c4c5042d6ff6c42ab0229b45bc65efd39b25fd67f4", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-spec-tree-table.dark.320", + "rgbaSHA256": "88ba1e733ef14be16ee7d47eac1946a546d91bcb9979e6199987c72c21971849", + "width": 320 + }, + { + "height": 1400, + "id": "webkit/static-spec-tree-table.light.1440", + "rgbaSHA256": "b3b97365e210d8629fe3082324b702381b2e8f4728aad9a7cbf2b13022b8ab9d", + "width": 1440 + }, + { + "height": 2400, + "id": "webkit/static-spec-tree-table.light.320", + "rgbaSHA256": "7893b75d18e97b0eedb64c8959729a4273f39c9a0d6ba18f5c2cd9b4b950ffc7", + "width": 320 + }, + { + "height": 973, + "id": "webkit/workspace-coverage.dark.1440", + "rgbaSHA256": "e51ba3e30226af0f355f205d8a0faf4e300a58456223abb51a12d68ff2fbf4d4", + "width": 1440 + }, + { + "height": 1245, + "id": "webkit/workspace-coverage.dark.320", + "rgbaSHA256": "c3384457bd8b6007ac23f58fe3b9f2789961ddb9089fa30fc78e083eaccfc5f3", + "width": 320 + }, + { + "height": 973, + "id": "webkit/workspace-coverage.light.1440", + "rgbaSHA256": "2e42d3b05d4299344c1a54f4b7ed59e11ddde7a6641473aa8b04a6fda013ce41", + "width": 1440 + }, + { + "height": 1245, + "id": "webkit/workspace-coverage.light.320", + "rgbaSHA256": "38d25105b2b1613a63d29ef4c3510dfa2d8e43f0bc6b01f989e1622d6b0c173d", + "width": 320 + }, + { + "height": 900, + "id": "webkit/workspace-diff.dark.1440", + "rgbaSHA256": "3725bae03ebc117296e05a36af06424e807b04f8b3d812b063d714f6e8a1ca63", + "width": 1440 + }, + { + "height": 1281, + "id": "webkit/workspace-diff.dark.320", + "rgbaSHA256": "6f1d6e864f1f89365a966bfe2d624c5d796a6db8363408658f09479e29064c4e", + "width": 320 + }, + { + "height": 900, + "id": "webkit/workspace-diff.light.1440", + "rgbaSHA256": "5d7936346f9ebab4a22b896a37382248eb32f03d388be26edb2e39f4c9f9cb3e", + "width": 1440 + }, + { + "height": 1281, + "id": "webkit/workspace-diff.light.320", + "rgbaSHA256": "0a57ba466e6221f8c5aad16758a67f3cb34c49de442a99804b50ab5fbd46c66e", + "width": 320 + }, + { + "height": 900, + "id": "webkit/workspace-specifications.dark.1440", + "rgbaSHA256": "5ab7589084e19ed7cbf74fa2d8637d40ca0ff8df8f76c849171688458e8bd5e4", + "width": 1440 + }, + { + "height": 905, + "id": "webkit/workspace-specifications.dark.320", + "rgbaSHA256": "803edd4d09e2c77de43db72ad61b54ea41d1b0380a0fabb5f2251582907594b8", + "width": 320 + }, + { + "height": 900, + "id": "webkit/workspace-specifications.light.1440", + "rgbaSHA256": "de81e9bcd21e5db202e984d2368a148cd17a52a861da1bb39eae63e57e719f54", + "width": 1440 + }, + { + "height": 905, + "id": "webkit/workspace-specifications.light.320", + "rgbaSHA256": "0ba0f25023171f4aee939e1e0b03d20b5592964bad3f0595a836ee423b5f24bf", + "width": 320 + }, + { + "height": 2042, + "id": "webkit/workspace-traceability.dark.1440", + "rgbaSHA256": "4d58c35c9a83ec99a897ae6981b966a67cae2164ee97d612494520dba9ee8a80", + "width": 1440 + }, + { + "height": 2206, + "id": "webkit/workspace-traceability.dark.320", + "rgbaSHA256": "cd1bb4479e0d036967caf8161a11687c4334adc8d5e39d9a5a6caa988f76a8f2", + "width": 320 + }, + { + "height": 2042, + "id": "webkit/workspace-traceability.light.1440", + "rgbaSHA256": "08018415f70353b6e364c8d1699218f4fe8456fd197ebefde129cdbe18b312c1", + "width": 1440 + }, + { + "height": 2206, + "id": "webkit/workspace-traceability.light.320", + "rgbaSHA256": "7c4b0676a3e7e455019d155b7ea0e73e42c7c678fc17655640292e39e3f860e3", + "width": 320 + } + ], + "schemaVersion": 1 +} diff --git a/tests/visual/references.spec.mjs b/tests/visual/references.spec.mjs new file mode 100644 index 00000000..540d6460 --- /dev/null +++ b/tests/visual/references.spec.mjs @@ -0,0 +1,14 @@ +import {assertMatchesReference, captureTwice, openView, referenceId, schemes, test, viewNames, widths} from "./visual-harness.mjs"; + +// One exact digest comparison per view, color scheme and width in every engine. +for (const view of viewNames) { + for (const scheme of schemes) { + for (const width of widths) { + test(`${view} ${scheme} ${width}`, async ({page, browserName, visualServers}, testInfo) => { + await openView(page, visualServers, view, scheme, width); + const captures = await captureTwice(page, {fullPage: true}); + assertMatchesReference(testInfo, referenceId(browserName, view, scheme, width), captures); + }); + } + } +} diff --git a/tests/visual/visual-harness.mjs b/tests/visual/visual-harness.mjs new file mode 100644 index 00000000..f9c52035 --- /dev/null +++ b/tests/visual/visual-harness.mjs @@ -0,0 +1,286 @@ +import {execFileSync, spawn} from "node:child_process"; +import {createHash} from "node:crypto"; +import {existsSync, mkdirSync, readFileSync, writeFileSync} from "node:fs"; +import {dirname, resolve, sep} from "node:path"; +import {fileURLToPath} from "node:url"; +import {expect, test as base} from "@playwright/test"; + +import {startBrowserServer} from "../../scripts/browser-proof-execution.mjs"; +import {canonicalRGBADigest, decodePNG} from "./exact-png.mjs"; + +// The visual reference harness. It captures each view twice, requires both +// captures to be byte-equal, decodes the capture and compares its canonical +// RGBA digest and dimensions with the admitted inventory entry. References are +// digests only; every capture is written below the run directory, never into +// the test directory. + +export const clipHeight = 2400; +export const viewportHeight = 900; +export const readmeViewport = Object.freeze({width: 1440, height: 960}); + +const repositoryRoot = fileURLToPath(new URL("../../", import.meta.url)); +const testDirectory = resolve(repositoryRoot, "tests/visual"); +const environment = JSON.parse(readFileSync(new URL("./environment.json", import.meta.url), "utf8")); +const fontConfig = readFileSync(new URL("./fonts.conf", import.meta.url)); +const inventory = JSON.parse(readFileSync(new URL("./references.json", import.meta.url), "utf8")); +const staticScriptDigest = createHash("sha256").update(readFileSync(new URL("../../internal/kernel/browserdoc/browser.js", import.meta.url))).digest("base64"); +const expectedCSP = Object.freeze({ + static: `default-src 'none'; script-src 'sha256-${staticScriptDigest}'; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'`, + workspace: "default-src 'none'; script-src 'self'; style-src 'self'; connect-src 'self'; img-src 'self'; worker-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'", +}); +const firefoxPreferences = Object.freeze({"browser.chrome.site_icons": false, "fission.webContentIsolationStrategy": 1}); +// Chromium re-rasters only invalidated tile regions by default; which regions +// an asynchronous state change invalidates varies between launches and +// changed rounded-corner pixels by one unit. Full tile raster is deterministic. +const chromiumArguments = Object.freeze(["--disable-partial-raster"]); + +// Five static renderings, each shown in Cards and Table modes. +const staticRenderings = Object.freeze({ + "static-source": {selector: "--static-source", heading: "Requirement Source View: browser.fixture.requirements"}, + "static-spec-tree": {selector: "--static-tree", heading: "Requirement Spec Tree View: browser.fixture.tree"}, + "static-proof-structured": {selector: "--static-proof", heading: "Requirement Proof View: proofkit.browser.coverage.binding"}, + "static-proof-compact": {selector: "--static-proof-compact", heading: "Compact Requirement Proof View: proofkit.browser.coverage.compact"}, + "static-coverage": {selector: "--static-coverage", heading: "Requirement Coverage View: proofkit.browser.coverage.view"}, +}); +// Four workspace tabs over the workspace fixture that carries coverage. +const workspaceTabs = Object.freeze({ + "workspace-specifications": {view: "specifications", state: "specifications"}, + "workspace-coverage": {view: "coverage", state: "coverage"}, + "workspace-diff": {view: "diff", state: "diff"}, + "workspace-traceability": {view: "graph", state: "graph"}, +}); +const workspaceSelector = "--coverage-structured"; +const workspaceHeading = "browser.fixture.workspace"; + +export const viewNames = Object.freeze([ + ...Object.keys(staticRenderings).flatMap((name) => [`${name}-cards`, `${name}-table`]), + ...Object.keys(workspaceTabs), +]); +export const schemes = Object.freeze(["light", "dark"]); +export const widths = Object.freeze([320, 1440]); + +export function referenceId(engine, view, scheme, width) { + return `${engine}/${view}.${scheme}.${width}`; +} + +export const test = base.extend({ + engineEvidence: [async ({browser, browserName, channel, connectOptions, launchOptions}, use, testInfo) => { + testInfo.annotations.push({type: "proofkit.browser-engine", description: browserName}); + testInfo.annotations.push({type: "proofkit.browser-version", description: browser.version()}); + expect(browserName).toBe(testInfo.project.name); + expect(channel).toBeUndefined(); + expect(connectOptions).toBeUndefined(); + expect(launchOptions.channel).toBeUndefined(); + expect(launchOptions.executablePath).toBeUndefined(); + expect(launchOptions.firefoxUserPrefs).toEqual(browserName === "firefox" ? firefoxPreferences : undefined); + expect(launchOptions.args).toEqual(browserName === "chromium" ? chromiumArguments : undefined); + // The probe compared engine revisions before launch; the launched + // version must also equal the recorded reference environment. + expect(browser.version()).toBe(environment.browserVersions[browserName]); + // Every browser reads the font configuration from the run directory's + // copy, whose bytes the probe hashed, and that copy must equal the + // content-bound fonts.conf of this test snapshot. Browsers inherit this + // worker's environment only while no project sets launchOptions.env. + const fontConfigCopy = resolve(testInfo.project.outputDir, "..", "fonts.conf"); + expect(launchOptions.env, "browsers must inherit the environment that names the font configuration copy").toBeUndefined(); + expect(process.env.FONTCONFIG_FILE, "FONTCONFIG_FILE must name the run directory's font configuration copy").toBe(fontConfigCopy); + expect(readFileSync(fontConfigCopy).equals(fontConfig), "the run directory's font configuration must equal tests/visual/fonts.conf").toBe(true); + await use(); + }, {auto: true}], + visualServers: [async ({}, use, workerInfo) => { + // One fixture server binary per run, built beside the output directory; + // a worker restarted after a failure reuses it. + const binary = resolve(workerInfo.project.outputDir, "..", "visual-fixture-server"); + if (!existsSync(binary)) execFileSync("go", ["build", "-o", binary, "./internal/tools/browsertestserver"], {stdio: "pipe"}); + const servers = new Map(); + try { + await use({ + async url(selector) { + if (!servers.has(selector)) { + servers.set(selector, await startBrowserServer(binary, { + spawnProcess: (path, args, options) => spawn(path, [selector, ...args], options), + })); + } + return servers.get(selector).url; + }, + }); + } finally { + const failures = []; + for (const server of servers.values()) { + try { await server.stop(); } catch (error) { failures.push(error); } + } + if (failures.length > 0) throw new AggregateError(failures, "visual fixture servers did not stop"); + } + }, {scope: "worker"}], +}); + +// navigate loads the view with the color scheme and reduced motion in effect. +// Firefox drops color-scheme emulation at a navigation into or out of a +// document served with Cross-Origin-Opener-Policy: same-origin, unless both +// documents have the same origin and send that header: the first navigation +// from about:blank, a navigation to another fixture server (each has its own +// port) and history traversal across such a boundary. Emulation applied after +// the last such navigation holds across reload and across navigation between +// documents of the same fixture. The emulation is therefore applied after the +// first navigation, the document is reloaded under it so that scripts reading +// the scheme at load see it, and it is applied again after the reload, the +// last navigation of every captured view; assertEmulation checks it before +// each capture. +export async function navigate(page, url, profile, heading, scheme) { + await emulate(page, scheme); + const first = await page.goto(url, {waitUntil: "load"}); + if (first === null || first.status() !== 200) throw new Error("visual view navigation did not return 200"); + await emulate(page, scheme); + const response = await page.reload({waitUntil: "load"}); + if (response === null || response.status() !== 200) throw new Error("visual view reload did not return 200"); + await emulate(page, scheme); + expect(response.headers()["content-security-policy"]).toBe(expectedCSP[profile]); + expect(response.headers()["x-content-type-options"]).toBe("nosniff"); + await expect(page.getByRole("heading", {name: heading, exact: true})).toBeVisible(); + await assertEmulation(page, scheme); +} + +async function emulate(page, scheme) { + await page.emulateMedia({colorScheme: scheme, reducedMotion: "reduce"}); +} + +// assertEmulation requires the requested color scheme and reduced motion to +// be in effect in the document about to be captured. +export async function assertEmulation(page, scheme) { + expect(await page.evaluate((colorScheme) => [ + matchMedia(`(prefers-color-scheme: ${colorScheme})`).matches, + matchMedia("(prefers-reduced-motion: reduce)").matches, + ], scheme), `color scheme ${scheme} and reduced motion must be in effect in the captured document`).toEqual([true, true]); +} + +async function settle(page) { + await page.evaluate(async () => { + if (document.activeElement instanceof HTMLElement) document.activeElement.blur(); + await document.fonts.ready; + await new Promise((resolveFrame) => requestAnimationFrame(() => requestAnimationFrame(resolveFrame))); + }); +} + +// openView renders one inventory view at the given scheme and width. +export async function openView(page, visualServers, view, scheme, width) { + await page.setViewportSize({width, height: viewportHeight}); + const tab = workspaceTabs[view]; + if (tab !== undefined) { + await navigate(page, await visualServers.url(workspaceSelector), "workspace", workspaceHeading, scheme); + await expect(page.locator("body")).toHaveAttribute("data-state", "specifications"); + if (tab.view !== "specifications") { + // A programmatic click selects the tab without leaving hover or focus. + await page.locator(`[data-view="${tab.view}"]`).evaluate((button) => button.click()); + await expect(page.locator("body")).toHaveAttribute("data-state", tab.state); + } + await expect(page.locator("#workspace-content")).toHaveAttribute("aria-busy", "false"); + await settle(page); + await assertEmulation(page, scheme); + return; + } + const match = view.match(/^(.*)-(cards|table)$/); + const rendering = match === null ? undefined : staticRenderings[match[1]]; + if (rendering === undefined) throw new Error(`unknown visual view ${view}`); + await navigate(page, await visualServers.url(rendering.selector), "static", rendering.heading, scheme); + if (match[2] === "table") await page.locator("#proofkit-view-mode").selectOption("table"); + await expect(page.locator("#proofkit-view-mode")).toHaveValue(match[2]); + await settle(page); + await assertEmulation(page, scheme); +} + +// fitViewportToDocument makes the window exactly as tall as the document, cut +// at clipHeight, so a plain viewport capture is the full page cut at 2400 CSS +// pixels. Capturing beyond the viewport would re-run layout at another +// viewport height during the capture, which changed page heights between two +// otherwise identical Chromium captures. +async function fitViewportToDocument(page) { + const {width} = page.viewportSize(); + let height = page.viewportSize().height; + for (let attempt = 0; attempt < 8; attempt += 1) { + const documentHeight = await page.evaluate(() => Math.max( + document.documentElement.scrollHeight, document.body.scrollHeight, + document.documentElement.offsetHeight, document.body.offsetHeight, + )); + const target = Math.min(clipHeight, documentHeight); + if (target === height) return; + height = target; + await page.setViewportSize({width, height}); + await settle(page); + } + throw new Error("document height did not converge while fitting the window to the page"); +} + +// captureTwice takes two captures with identical options; the gate admits a +// capture only when both are byte-equal. A full-page capture is the whole +// document cut at clipHeight CSS pixels. +export async function captureTwice(page, {fullPage}) { + if (fullPage) await fitViewportToDocument(page); + const options = {animations: "disabled", caret: "hide", scale: "css", fullPage: false}; + const first = await page.screenshot(options); + const second = await page.screenshot(options); + return {first, second}; +} + +// The run directory is the parent of the Playwright output directory. Stable +// captures go to /captures/.png, unstable pairs to /unstable, +// and mutated control captures to /controls. No path resolves into the +// test directory. +function runFile(testInfo, area, relativePath) { + const runDirectory = resolve(testInfo.project.outputDir, ".."); + const root = resolve(runDirectory, area); + const target = resolve(root, relativePath); + if (!target.startsWith(root + sep) || target === testDirectory || target.startsWith(testDirectory + sep)) { + throw new Error("visual capture path escaped its run directory area"); + } + return target; +} + +function writeRunFile(testInfo, area, relativePath, bytes) { + const target = runFile(testInfo, area, relativePath); + mkdirSync(dirname(target), {recursive: true}); + writeFileSync(target, bytes, {flag: "wx"}); +} + +function admittedEntry(id) { + return Array.isArray(inventory.references) ? inventory.references.find((entry) => entry.id === id) : undefined; +} + +function describe(digest) { + return `${digest.rgbaSHA256} (${digest.width}x${digest.height})`; +} + +// stableDigest requires the two captures to be byte-equal and returns the +// canonical digest and dimensions of the decoded capture. +function stableDigest(testInfo, id, {first, second}) { + if (!first.equals(second)) { + writeRunFile(testInfo, "unstable", `${id}.first.png`, first); + writeRunFile(testInfo, "unstable", `${id}.second.png`, second); + throw new Error(`${id}: two captures of the same state differ; the view is not stable`); + } + const image = decodePNG(first); + return {height: image.height, rgbaSHA256: canonicalRGBADigest(image), width: image.width}; +} + +// assertMatchesReference keeps the capture for the gallery and admission, +// then requires equal dimensions and an equal canonical digest. +export function assertMatchesReference(testInfo, id, captures) { + const actual = stableDigest(testInfo, id, captures); + writeRunFile(testInfo, "captures", `${id}.png`, captures.first); + const admitted = admittedEntry(id); + if (admitted === undefined) { + throw new Error(`${id}: has no admitted reference; captured ${describe(actual)}`); + } + if (admitted.width !== actual.width || admitted.height !== actual.height || admitted.rgbaSHA256 !== actual.rgbaSHA256) { + throw new Error(`${id}: captured ${describe(actual)} differs from the admitted ${describe(admitted)}`); + } +} + +// controlDigest returns the digest of a mutated capture and the admitted +// entry of the unmutated view; a control passes only when they differ. +export function controlDigest(testInfo, id, controlName, captures) { + const actual = stableDigest(testInfo, `controls/${id}.${controlName}`, captures); + writeRunFile(testInfo, "controls", `${id}.${controlName}.png`, captures.first); + const admitted = admittedEntry(id); + if (admitted === undefined) throw new Error(`${id}: has no admitted reference for the ${controlName} control`); + return {actual, admitted}; +} From 23697c63e15a9f713f6f46248f34960c31d70574 Mon Sep 17 00:00:00 2001 From: iperev Date: Fri, 23 Oct 2026 22:26:23 +0200 Subject: [PATCH 4/6] test(browser): assert media emulation in the document under test Pinned Firefox drops color-scheme emulation at a navigation into or out of a document served with Cross-Origin-Opener-Policy: same-origin, unless both documents have the same origin and send that header. That includes the first navigation from about:blank, a navigation to another fixture (each fixture server has its own port) and history traversal across such a boundary; every fixture document sends the header. Emulation applied after the last such navigation holds across reload and across navigation between documents of the same fixture. Chromium and WebKit are unaffected, and reduced-motion and forced-colors emulation survived every such navigation tested in every engine. Playwright's default light scheme is applied before navigation as well, so pinned Firefox drops it; tests that do not call the helper run in Firefox under its own preference. The reflow spec tests/browser/workspace-evidence-layout.spec.mjs, the native reflow witness of REQ-PROOFKIT-SPEC-039 and REQ-PROOFKIT-SPEC-040, emulated before openWorkspace, so its two Firefox dark rows (390x844 and 1280x800) ran in the light scheme. No emulation site asserted that its emulation took effect. - tests/browser/workspace-navigation-harness.mjs: emulateMediaInEffect applies page.emulateMedia and then asserts, in one observation, that the page holds a served http document and that matchMedia matches every given color-scheme, reduced-motion and forced-colors value. It refuses any other feature or value, and the http check refuses a call made in the initial about:blank document, where a match says nothing about the next document. Every pinned engine reflects forced-colors emulation in matchMedia, so that feature is asserted in all three. The assertion proves the media feature, which switches the author forced-colors rules, not an engine forced-color palette, which WebKit does not apply. - tests/browser/workspace-evidence-layout.spec.mjs: each reflow row emulates after openWorkspace through the helper, without a reload. The workspace reads the color scheme and the motion preference only through CSS (media queries and the root color-scheme property), which follows emulation in the loaded document. - tests/browser/workspace.spec.mjs (rendered light and dark contrast), tests/browser/static-view.spec.mjs (static control contrast) and tests/browser/workspace-graph.spec.mjs (dark and forced-colors graph paint) emulate through the helper. They already emulated after navigation. Test titles, requirement statements, bindings and the browser proof input manifest are unchanged. No statement of REQ-PROOFKIT-SPEC-039 or REQ-PROOFKIT-SPEC-040 names a color scheme, the light and dark contrast oracles of REQ-PROOFKIT-QUALITY-022 already ran under the intended scheme, and the runtime proof inputs select tests/browser as a whole. Proof on this tree: A local npm run check sequence on darwin/arm64 passed (exit 0) on tree 14809a0c67de with Node 26.11.1, npm 12.2.0, CPython 3.15.0 and Go 1.27.3: 561/561 runtime browser cases in Chromium, Firefox and WebKit with zero retries. One recorded deviation: the host firewall blocks Go-built binaries from the network, so go:vulncheck ran against a vulnerability-database snapshot fetched with curl (modified 2026-10-21T00:27:29Z): no vulnerabilities. --- tests/browser/static-view.spec.mjs | 4 +- .../workspace-evidence-layout.spec.mjs | 7 +++- tests/browser/workspace-graph.spec.mjs | 4 +- .../browser/workspace-navigation-harness.mjs | 37 +++++++++++++++++++ tests/browser/workspace.spec.mjs | 4 +- 5 files changed, 48 insertions(+), 8 deletions(-) diff --git a/tests/browser/static-view.spec.mjs b/tests/browser/static-view.spec.mjs index 6ccf76f1..68c659ba 100644 --- a/tests/browser/static-view.spec.mjs +++ b/tests/browser/static-view.spec.mjs @@ -2,7 +2,7 @@ import {readFile} from "node:fs/promises"; import {createHash} from "node:crypto"; import {expect} from "@playwright/test"; import {staticViewTest as test} from "./workspace-test-harness.mjs"; -import {openWorkspace} from "./workspace-navigation-harness.mjs"; +import {emulateMediaInEffect, openWorkspace} from "./workspace-navigation-harness.mjs"; const heading = "Requirement Source View: browser.fixture.requirements"; const treeHeading = "Requirement Spec Tree View: browser.fixture.tree"; @@ -146,7 +146,7 @@ test("static hierarchy links resolve once and remain deterministic", async ({sta for (const colorScheme of ["light", "dark"]) { test(`static control boundaries have computed contrast in ${colorScheme}`, async ({staticTreeURL, page}, testInfo) => { await openWorkspace(page, staticTreeURL, treeHeading, "static-view"); - await page.emulateMedia({colorScheme}); + await emulateMediaInEffect(page, {colorScheme}); const controls = page.locator('input[type="search"], select, button'); expect(await controls.count()).toBeGreaterThanOrEqual(4); const inspect = () => controls.evaluateAll(nodes => { diff --git a/tests/browser/workspace-evidence-layout.spec.mjs b/tests/browser/workspace-evidence-layout.spec.mjs index 25be0094..280dd86b 100644 --- a/tests/browser/workspace-evidence-layout.spec.mjs +++ b/tests/browser/workspace-evidence-layout.spec.mjs @@ -1,6 +1,6 @@ import {expect} from "@playwright/test"; import {coverageTest as test} from "./workspace-test-harness.mjs"; -import {openWorkspace} from "./workspace-navigation-harness.mjs"; +import {emulateMediaInEffect, openWorkspace} from "./workspace-navigation-harness.mjs"; import {analyzeAxe, assertAxeTestComplete, initializeAxe} from "./axe-harness.mjs"; const viewports = [ @@ -16,11 +16,14 @@ const viewports = [ for (const viewport of viewports) { test(`evidence ${viewport.view} reflows at ${viewport.width}x${viewport.height} ${viewport.colorScheme}`, async ({compactURL, page}, testInfo) => { await page.setViewportSize({width: viewport.width, height: viewport.height}); - await page.emulateMedia({colorScheme: viewport.colorScheme, reducedMotion: "reduce"}); await initializeAxe(page); const errors = []; page.on("pageerror", error => errors.push(error.message)); await openWorkspace(page, compactURL); + // The workspace reads the color scheme and motion preference only through + // CSS (media queries and the root color-scheme property), which follows + // emulation in the loaded document, so no reload is needed. + await emulateMediaInEffect(page, {colorScheme: viewport.colorScheme, reducedMotion: "reduce"}); await expect(page.locator("html")).toHaveCSS("font-size", "16px"); await page.getByRole("button", {name: viewport.view === "coverage" ? "Coverage" : "Traceability", exact: true}).click(); await expect(page.locator("body")).toHaveAttribute("data-state", viewport.view); diff --git a/tests/browser/workspace-graph.spec.mjs b/tests/browser/workspace-graph.spec.mjs index 8cda5530..d45b686b 100644 --- a/tests/browser/workspace-graph.spec.mjs +++ b/tests/browser/workspace-graph.spec.mjs @@ -1,6 +1,6 @@ import {expect} from "@playwright/test"; import {capacityTest, graphLayoutTest, test} from "./workspace-test-harness.mjs"; -import {openWorkspace} from "./workspace-navigation-harness.mjs"; +import {emulateMediaInEffect, openWorkspace} from "./workspace-navigation-harness.mjs"; import {analyzeAxe, assertAxeTestComplete, initializeAxe} from "./axe-harness.mjs"; import {admitGraphCommitMilliseconds, assertGraphGeometry, assertGraphPaint, normalizeGraphGeometry, readGraphGeometry} from "./graph-geometry-oracle.mjs"; @@ -209,7 +209,7 @@ graphLayoutTest("graph geometry preserves paint in dark and forced colors and de await page.setViewportSize({width: 1920, height: 1080}); const graph = await openGraph(page, graphMixedLayoutURL); for (const media of [{colorScheme: "dark", forcedColors: "none"}, {colorScheme: "light", forcedColors: "active"}]) { - await page.emulateMedia(media); + await emulateMediaInEffect(page, media); await assertRenderedGraph(page, graph); await attachGraphScreenshot(page, testInfo, `graph-${media.forcedColors === "active" ? "forced" : "dark"}.png`); } diff --git a/tests/browser/workspace-navigation-harness.mjs b/tests/browser/workspace-navigation-harness.mjs index 8f66be42..67d25686 100644 --- a/tests/browser/workspace-navigation-harness.mjs +++ b/tests/browser/workspace-navigation-harness.mjs @@ -165,6 +165,43 @@ export async function openWorkspace(page, baseURL, heading, securityProfile = "w ); } +// Media features whose emulation every pinned engine reflects in matchMedia, +// with their admitted values. For forced colors this proves the media feature, +// which switches the author forced-colors rules, not an engine forced-color +// palette: pinned WebKit applies none. +const emulatedMediaQueries = Object.freeze({ + colorScheme: Object.freeze({light: "(prefers-color-scheme: light)", dark: "(prefers-color-scheme: dark)"}), + reducedMotion: Object.freeze({reduce: "(prefers-reduced-motion: reduce)", "no-preference": "(prefers-reduced-motion: no-preference)"}), + forcedColors: Object.freeze({active: "(forced-colors: active)", none: "(forced-colors: none)"}), +}); + +// emulateMediaInEffect applies media emulation and asserts through matchMedia +// that every given feature is in effect in the document under test. Call it +// after the last navigation into that document: pinned Firefox drops +// color-scheme emulation at a navigation into or out of a document served +// with Cross-Origin-Opener-Policy: same-origin, which every fixture document +// sends, unless both documents have the same origin and send that header; +// each fixture server has its own origin. The document must be a served http +// document, because a match in the initial about:blank document says nothing +// about the next one. +export async function emulateMediaInEffect(page, media) { + const queries = Object.entries(media).map(([feature, value]) => { + const values = Object.hasOwn(emulatedMediaQueries, feature) ? emulatedMediaQueries[feature] : {}; + if (!Object.hasOwn(values, value)) throw new Error(`Unsupported media emulation ${feature}: ${String(value)}`); + return values[value]; + }); + if (queries.length === 0) throw new Error("Media emulation names no feature"); + await page.emulateMedia(media); + const observed = await page.evaluate((queries) => ({ + protocol: location.protocol, + matches: Object.fromEntries(queries.map((query) => [query, matchMedia(query).matches])), + }), queries); + expect(observed, "media emulation in effect in the document under test").toEqual({ + protocol: "http:", + matches: Object.fromEntries(queries.map((query) => [query, true])), + }); +} + export async function reloadWorkspace(page, baseURL) { const workspaceURL = admittedWorkspaceURL(baseURL); await navigateWorkspace( diff --git a/tests/browser/workspace.spec.mjs b/tests/browser/workspace.spec.mjs index 943b92d9..05e36c01 100644 --- a/tests/browser/workspace.spec.mjs +++ b/tests/browser/workspace.spec.mjs @@ -5,7 +5,7 @@ import {test} from "./workspace-test-harness.mjs"; import {analyzeAxe, assertAxeTestComplete, initializeAxe} from "./axe-harness.mjs"; -import {admittedWorkspaceURL, isWorkspaceNavigationResponse, navigateWorkspace, openWorkspace, reloadWorkspace, staticViewCSP} from "./workspace-navigation-harness.mjs"; +import {admittedWorkspaceURL, emulateMediaInEffect, isWorkspaceNavigationResponse, navigateWorkspace, openWorkspace, reloadWorkspace, staticViewCSP} from "./workspace-navigation-harness.mjs"; const fixtureWorkspaceCSP = "default-src 'none'; script-src 'self'; style-src 'self'; connect-src 'self'; img-src 'self'; worker-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'"; @@ -1467,7 +1467,7 @@ async function assertReflow(page, row) { async function assertRenderedContrast(page, row) { for (const colorScheme of ["light", "dark"]) { - await page.emulateMedia({colorScheme}); + await emulateMediaInEffect(page, {colorScheme}); await assertWorkspaceState(page, row); const samples = await page.evaluate(() => { const parse = (value) => { From be34b7f38f17ba6cf453ae1c0ae9a3ddd9c2a4fb Mon Sep 17 00:00:00 2001 From: iperev Date: Sat, 24 Oct 2026 05:37:59 +0200 Subject: [PATCH 5/6] fix(test): give spawned CLI children their input as a file, not a socket The Node suites passed CLI input with spawnSync's `input` option. Node then gives the child one end of a Unix-domain socket pair, writes the input, calls shutdown(SHUT_WR) on its end and keeps that end open until the child exits or the timeout kills it. On macOS, a child blocked in read(2) on that socket is sometimes not woken by the shutdown, although the socket already holds end of file, so the call ends only at its timeout. Node 22, 24 and 26 stall this way on darwin/arm64, and so does the same syscall sequence written in C, without Node or libuv. With a pipe, with a close instead of the shutdown or with a regular file, that reproducer did not stall, and neither did Node on Linux. The suites saw the stall as an occasional spawnSync ETIMEDOUT of one CLI call. - scripts/cli-process.mjs adds spawnWithInputFile(file, args, options). It is spawnSync(file, args, options) except that options.input is written to a temporary file that the child gets as its standard input. The file has mode 0600 in a fresh directory of mode 0700, and the directory is removed as soon as the file is open, before the child starts, or at once when writing or opening the file throws. The removal uses path calls only; if it fails, the input descriptor is closed and the call throws. Status, signal, error, stdout, stderr, timeout, maxBuffer and the other passed options keep spawnSync's meaning, except that writing the input can no longer fail with EPIPE. An unsupported option is refused, and so is any args that is not an array, including the two-argument form spawnWithInputFile(file, options), which spawnSync would read as its options while it drops the third argument. The result also reports stdinUnreadBytes, the input bytes past the file offset that the child shares with the parent when it exits: read(2) and lseek(2) on the inherited descriptor move it; pread(2), mmap(2) and, on Linux, reopening /dev/stdin do not. - All 42 spawnSync calls with input in 29 test files under scripts/ use it. - assertCLITransport accepts a non-zero stdinUnreadBytes under the rule it already applies to EPIPE: only from a child that exited normally with a non-zero status and no signal. EPIPE stays accepted for the asynchronous pipe transport of the schema-completeness test. - The early-refusal test of the browser server asserts that stdinUnreadBytes equals the size of its input, so the refusal read none of it, instead of asserting EPIPE. - scripts/cli-process.test.mjs covers the file transport, the unread-input and EPIPE rules through both transports, the kept and refused options, the removal of the input file before the child runs and after a throw, a failed write included, and the closing of the input descriptor after a failed removal. It adds a lexical source check that refuses an `input` member in the arguments of a direct spawnSync, execFileSync or execSync call under scripts/ and tests/. - proofkit/witness-plan.json: the boundary-contract-check policy declares scripts and tests, which that source check reads. No timeout, retry or assertion is relaxed. Proof on this tree: A local npm run check sequence on darwin/arm64 passed (exit 0) on tree 600c4bf80032 with Node 26.11.1, npm 12.2.0, CPython 3.15.0 and Go 1.27.3, with no Go test verdict replayed from the test cache: 561/561 runtime browser cases in Chromium, Firefox and WebKit with zero retries. One recorded deviation: the host firewall blocks Go-built binaries from the network, so go:vulncheck ran against a vulnerability-database snapshot fetched with curl (modified 2026-10-21T00:27:29Z): no vulnerabilities. --- proofkit/witness-plan.json | 4 +- scripts/admission-records-structure.test.mjs | 6 +- scripts/adoption-envelope-structure.test.mjs | 6 +- scripts/agent-workflow-structure.test.mjs | 4 +- scripts/authority-guard-structure.test.mjs | 6 +- scripts/bootstrap-structure.test.mjs | 6 +- scripts/boundary-structure.test.mjs | 6 +- scripts/browser-server-structure.test.mjs | 12 +- scripts/cli-process.mjs | 132 +++++++- scripts/cli-process.test.mjs | 311 +++++++++++++++++- scripts/compact-proof-structure.test.mjs | 6 +- scripts/consumer-evidence-structure.test.mjs | 6 +- scripts/explicit-input-structure.test.mjs | 6 +- scripts/gradual-adoption-structure.test.mjs | 16 +- scripts/impact-structure.test.mjs | 6 +- scripts/inventory-coverage-structure.test.mjs | 8 +- scripts/materialization-structure.test.mjs | 6 +- scripts/overview-claims-structure.test.mjs | 6 +- scripts/profile-registry-structure.test.mjs | 6 +- scripts/project-status-structure.test.mjs | 6 +- scripts/proof-lookup-structure.test.mjs | 12 +- scripts/proof-routing-structure.test.mjs | 6 +- scripts/public-api-structure.test.mjs | 6 +- scripts/receipt-admission-structure.test.mjs | 6 +- scripts/receipt-structure.test.mjs | 6 +- .../release-classification-structure.test.mjs | 6 +- scripts/report-admission-structure.test.mjs | 4 +- scripts/scaffold-doctor-structure.test.mjs | 10 +- scripts/selective-proof-structure.test.mjs | 10 +- scripts/source-conversion-structure.test.mjs | 6 +- scripts/spec-tree-structure.test.mjs | 4 +- scripts/workspace-structure.test.mjs | 4 +- 32 files changed, 519 insertions(+), 126 deletions(-) diff --git a/proofkit/witness-plan.json b/proofkit/witness-plan.json index 147099ae..b5eedc6b 100644 --- a/proofkit/witness-plan.json +++ b/proofkit/witness-plan.json @@ -2160,6 +2160,7 @@ "proofkit/cli-contract.v2.json", "proofkit/requirement-bindings.json", "proofkit/witness-plan.json", + "scripts", "scripts/admission-records-structure.test.mjs", "scripts/adoption-envelope-structure.test.mjs", "scripts/agent-workflow-structure.test.mjs", @@ -2196,7 +2197,8 @@ "scripts/selective-proof-structure.test.mjs", "scripts/source-conversion-structure.test.mjs", "scripts/stable-json.mjs", - "scripts/workspace-structure.test.mjs" + "scripts/workspace-structure.test.mjs", + "tests" ], "outputSelectors": [], "resourceReads": [ diff --git a/scripts/admission-records-structure.test.mjs b/scripts/admission-records-structure.test.mjs index fd63159b..afc65c6a 100644 --- a/scripts/admission-records-structure.test.mjs +++ b/scripts/admission-records-structure.test.mjs @@ -1,11 +1,11 @@ import assert from 'node:assert/strict'; -import {execFileSync, spawnSync} from 'node:child_process'; +import {execFileSync} from 'node:child_process'; import {mkdtempSync, readFileSync, rmSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {join} from 'node:path'; import test, {after, before} from 'node:test'; import Ajv2020 from 'ajv/dist/2020.js'; -import {assertCLITransport} from './cli-process.mjs'; +import {assertCLITransport, spawnWithInputFile} from './cli-process.mjs'; import {expandSharedDefinitions} from './schema-definitions.mjs'; const root = new URL('../', import.meta.url); @@ -94,7 +94,7 @@ function branchIndex(command, value) { return matches[0]; } function run(command, input, flags = []) { - const result = spawnSync(binary, [command, '--input', '-', ...flags], { + const result = spawnWithInputFile(binary, [command, '--input', '-', ...flags], { input: typeof input === 'string' ? input : JSON.stringify(input), encoding: 'utf8', timeout: 30000, maxBuffer: 32 << 20, }); assertCLITransport(result); diff --git a/scripts/adoption-envelope-structure.test.mjs b/scripts/adoption-envelope-structure.test.mjs index ad1f842a..73677b34 100644 --- a/scripts/adoption-envelope-structure.test.mjs +++ b/scripts/adoption-envelope-structure.test.mjs @@ -1,11 +1,11 @@ import assert from 'node:assert/strict'; -import {execFileSync, spawnSync} from 'node:child_process'; +import {execFileSync} from 'node:child_process'; import {mkdtempSync, readFileSync, rmSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {join} from 'node:path'; import test, {after, before} from 'node:test'; import Ajv2020 from 'ajv/dist/2020.js'; -import {assertCLITransport} from './cli-process.mjs'; +import {assertCLITransport, spawnWithInputFile} from './cli-process.mjs'; import {impactInput} from './impact-fixtures.mjs'; const root = new URL('../', import.meta.url); @@ -234,7 +234,7 @@ const stateOf = value => Array.isArray(value) ? (value.every(row => row.state == : value.state ?? value.report?.state ?? value.sourceReport?.state; function run(command, input, flags = [], env = {}) { - const result = spawnSync(binary, [command, '--input', '-', ...flags], { + const result = spawnWithInputFile(binary, [command, '--input', '-', ...flags], { input: typeof input === 'string' ? input : JSON.stringify(input), encoding: 'utf8', timeout: 20000, maxBuffer: 16 << 20, env: {...process.env, AGENTIC_PROOFKIT_LAUNCHER_PROFILE: 'path', AGENTIC_PROOFKIT_PYTHON_EXECUTABLE: '', ...env}, }); diff --git a/scripts/agent-workflow-structure.test.mjs b/scripts/agent-workflow-structure.test.mjs index bf7061e0..eff76e42 100644 --- a/scripts/agent-workflow-structure.test.mjs +++ b/scripts/agent-workflow-structure.test.mjs @@ -6,7 +6,7 @@ import {tmpdir} from 'node:os'; import {join} from 'node:path'; import test, {after, before} from 'node:test'; import Ajv2020 from 'ajv/dist/2020.js'; -import {assertCLITransport} from './cli-process.mjs'; +import {assertCLITransport, spawnWithInputFile} from './cli-process.mjs'; import {stableJSONStringify} from './stable-json.mjs'; // Independent oracle for agent-route, change-workflow-plan and @@ -64,7 +64,7 @@ after(() => {if (directory) rmSync(directory, {recursive: true, force: true});}) const pathLauncher = {AGENTIC_PROOFKIT_LAUNCHER_PROFILE: 'path', AGENTIC_PROOFKIT_PYTHON_EXECUTABLE: ''}; function run(args, input, {env = pathLauncher, cwd} = {}) { - const result = spawnSync(binary, args, { + const result = spawnWithInputFile(binary, args, { input: input === undefined ? undefined : typeof input === 'string' ? input : JSON.stringify(input), encoding: 'utf8', timeout: 20000, maxBuffer: 16 << 20, env: {...process.env, ...env}, cwd, }); diff --git a/scripts/authority-guard-structure.test.mjs b/scripts/authority-guard-structure.test.mjs index 6f32f5f6..d402935f 100644 --- a/scripts/authority-guard-structure.test.mjs +++ b/scripts/authority-guard-structure.test.mjs @@ -1,12 +1,12 @@ import assert from "node:assert/strict"; import {createHash} from "node:crypto"; -import {execFileSync, spawnSync} from "node:child_process"; +import {execFileSync} from "node:child_process"; import {mkdtempSync, readFileSync, rmSync} from "node:fs"; import {tmpdir} from "node:os"; import {join} from "node:path"; import test, {before, after} from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; -import {assertCLITransport} from "./cli-process.mjs"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; const root = new URL("../", import.meta.url), producer = "producer-policy-self-proof", migration = "migration-parity-admission"; const baseline = JSON.parse(readFileSync(new URL("internal/app/testdata/authority-guard-native-observations.json", root))); @@ -41,7 +41,7 @@ before(() => { after(() => {if (directory) rmSync(directory, {recursive: true, force: true});}); const hash = bytes => createHash("sha256").update(bytes).digest("hex"); function invoke(command, input, argv = [command, "--input", "-"]) { - const result = spawnSync(binary, argv, {input, encoding: "utf8", timeout: 10000, maxBuffer: 2 << 20}); + const result = spawnWithInputFile(binary, argv, {input, encoding: "utf8", timeout: 10000, maxBuffer: 2 << 20}); return assertCLITransport(result); } const utf8Order = (a, b) => Buffer.compare(Buffer.from(a, "utf8"), Buffer.from(b, "utf8")); diff --git a/scripts/bootstrap-structure.test.mjs b/scripts/bootstrap-structure.test.mjs index 9f35c13a..eab77466 100644 --- a/scripts/bootstrap-structure.test.mjs +++ b/scripts/bootstrap-structure.test.mjs @@ -1,12 +1,12 @@ import assert from "node:assert/strict"; import {createHash} from "node:crypto"; -import {execFileSync, spawnSync} from "node:child_process"; +import {execFileSync} from "node:child_process"; import {mkdtempSync, readFileSync, rmSync} from "node:fs"; import {tmpdir} from "node:os"; import {join} from "node:path"; import test, {before, after} from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; -import {assertCLITransport} from "./cli-process.mjs"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; const root = new URL("../", import.meta.url); const contract = JSON.parse(readFileSync(process.env.PROOFKIT_BOOTSTRAP_CONTRACT || new URL("proofkit/cli-contract.v2.json", root), "utf8")); @@ -27,7 +27,7 @@ before(() => { after(() => { if (directory) rmSync(directory, {recursive: true, force: true}); }); const hash = text => createHash("sha256").update(text).digest("hex"); function invoke(inputJSON, args = []) { - const r = spawnSync(binary, ["self-check", "--input", "-", ...args], {input: inputJSON, encoding: "utf8", timeout: 10000, maxBuffer: 1 << 20}); + const r = spawnWithInputFile(binary, ["self-check", "--input", "-", ...args], {input: inputJSON, encoding: "utf8", timeout: 10000, maxBuffer: 1 << 20}); return assertCLITransport(r); } function expected(kind) { diff --git a/scripts/boundary-structure.test.mjs b/scripts/boundary-structure.test.mjs index 9eabe3c2..9aa923c8 100644 --- a/scripts/boundary-structure.test.mjs +++ b/scripts/boundary-structure.test.mjs @@ -1,11 +1,11 @@ import assert from "node:assert/strict"; -import {execFileSync, spawnSync} from "node:child_process"; +import {execFileSync} from "node:child_process"; import {mkdtempSync, readFileSync, rmSync} from "node:fs"; import {tmpdir} from "node:os"; import {join} from "node:path"; import test, {before, after} from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; -import {assertCLITransport} from "./cli-process.mjs"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; const root = new URL("../", import.meta.url); const contract = JSON.parse(readFileSync(process.env.PROOFKIT_BOUNDARY_CONTRACT || new URL("proofkit/cli-contract.v2.json", root), "utf8")); @@ -46,7 +46,7 @@ before(() => { after(() => { if (directory) rmSync(directory, {recursive: true, force: true}); }); function native(family, input, expected, label) { - const result = spawnSync(binary, [family.command, "--input", "-"], { + const result = spawnWithInputFile(binary, [family.command, "--input", "-"], { input: typeof input === "string" ? input : JSON.stringify(input), encoding: "utf8", timeout: 10_000, maxBuffer: 2 << 20, }); assertCLITransport(result, label); diff --git a/scripts/browser-server-structure.test.mjs b/scripts/browser-server-structure.test.mjs index bb5f85be..fa205c88 100644 --- a/scripts/browser-server-structure.test.mjs +++ b/scripts/browser-server-structure.test.mjs @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import {execFileSync, spawn, spawnSync} from 'node:child_process'; +import {execFileSync, spawn} from 'node:child_process'; import {existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; import {get, request as httpRequest} from 'node:http'; import {tmpdir} from 'node:os'; @@ -7,7 +7,7 @@ import {delimiter, join} from 'node:path'; import test, {after, before} from 'node:test'; import {setTimeout as sleep} from 'node:timers/promises'; import Ajv2020 from 'ajv/dist/2020.js'; -import {assertCLITransport} from './cli-process.mjs'; +import {assertCLITransport, spawnWithInputFile} from './cli-process.mjs'; import {bindingInput, resolverInput} from './proof-routing-fixtures.mjs'; import {directComposeInput, inventoryInput, normalizedComposeInput, requirementSource} from './inventory-coverage-fixtures.mjs'; @@ -116,7 +116,7 @@ const flipped = value => typeof value === 'string' ? 7 : typeof value === 'numbe let directory, binary; const fixtures = {}; function spawnCLI(args, input, options = {}) { - return spawnSync(binary, args, {input: input === undefined ? '' : JSON.stringify(input), + return spawnWithInputFile(binary, args, {input: input === undefined ? '' : JSON.stringify(input), encoding: 'utf8', timeout: 30000, maxBuffer: 32 << 20, ...options}); } function run(args, input, options = {}) { @@ -437,11 +437,11 @@ test('serving browse mode emits one non-JSON URL record and terminates on SIGTER }); test('a CLI that exits before reading its stdin is accepted only as a refusal', () => { - // 8 MiB exceeds every pipe buffer; the flag refusal exits before reading stdin, so writing the payload must report EPIPE. + // The flag refusal exits before reading stdin, so the child leaves every byte of the payload unread. const payload = {...clone(fixtures.workspace), padding: 'x'.repeat(8 << 20)}; const args = browserArgs('workspace', ['--serve', '--session-mode', 'one-shot-question']); - assert.equal(refused(args, payload, 'oversized one-shot without --open').error?.code, 'EPIPE'); - // The closed pipe passes the transport check, so a caller that expects success fails on the refusal status. + assert.equal(refused(args, payload, 'oversized one-shot without --open').stdinUnreadBytes, Buffer.byteLength(JSON.stringify(payload))); + // Unread input passes the transport check only with a refusal status, so a caller that expects success fails on that status. assert.throws(() => json(args, payload), error => error.code === 'ERR_ASSERTION' && error.actual === 1 && error.expected === 0); }); diff --git a/scripts/cli-process.mjs b/scripts/cli-process.mjs index d021982f..8a070df0 100644 --- a/scripts/cli-process.mjs +++ b/scripts/cli-process.mjs @@ -1,20 +1,132 @@ -// Transport check for oracles that run the Proofkit CLI through spawnSync. +// Transport for oracles that run the Proofkit CLI, or another child process, +// synchronously. // -// A CLI that refuses its flags exits before it reads stdin, so spawnSync -// can report EPIPE for the input it could not write. As in -// internal/tools/workflowsmoke/process_streams.go, that is an ordinary early -// exit and not a transport failure. EPIPE is accepted only when the child -// exited normally with a non-zero status and no signal. Any other spawn error, -// a timeout, a signal, or EPIPE with status 0 (a success that did not consume -// its input) fails. Callers keep their own status, stdout and stderr checks. +// spawnWithInputFile(file, args, options) is spawnSync(file, args, options), +// except that options.input reaches the child as a standard input opened on +// a temporary regular file instead of through a pipe. For the `input` option, +// Node's spawnSync gives the child one end of a Unix-domain socket pair: it +// writes the input, calls shutdown(SHUT_WR) on its end and keeps that end open +// until the child exits or the timeout kills it. On macOS, a child blocked in +// read(2) on the socket is sometimes not woken by the shutdown although the +// socket already holds end of file, so the call ends only at its timeout. The +// same syscall sequence stalls without Node; a pipe, a close or a regular file +// does not. A regular file ends at its size, so the child always reads end of +// file. +// +// args must be an array. spawnSync(file, options) would take the options from +// the args position and drop the third argument, so the two-argument form +// would feed the input through the socket again; it is refused. +// +// The input is written to a file of mode 0600 in a fresh directory of mode +// 0700 and opened read-only. The directory is removed as soon as the file is +// open, before the child starts: the open descriptor keeps the file, so while +// the child runs no input remains on disk, and a signal that ends this process +// then leaves none behind. A throw while the file is written or opened removes +// the directory too. The removal uses path calls only; if it fails, the input +// descriptor is closed and the call throws. +// +// The result is spawnSync's result (status, signal, error, stdout, stderr, +// output, pid) plus stdinUnreadBytes: the input bytes past the file offset +// that the child shares with this process, read when the child has exited. +// read(2) and lseek(2) on the inherited descriptor move that offset; pread(2), +// mmap(2) and, on Linux, reopening /dev/stdin do not. A child that reads its +// input only those ways reports unread bytes although it read them, and a +// child that seeks past bytes it did not read reports them as read. No current +// caller depends on those readings: its children are Proofkit CLI builds and +// the Node probes of cli-process.test.mjs, which read their standard input +// with read(2) or not at all. An empty or absent input is an empty file. With +// stdio ["ignore", "pipe", "pipe"] the child's standard input is the null +// device, no input is allowed and stdinUnreadBytes is absent. +// +// assertCLITransport checks the transport of a result. A CLI that refuses its +// flags exits before it reads stdin. Through a pipe (an asynchronous spawn +// that writes stdin), the parent can then report EPIPE for the input it could +// not write; through spawnWithInputFile, stdinUnreadBytes is not zero. As in +// internal/tools/workflowsmoke/process_streams.go, either is an ordinary early +// exit and not a transport failure only when the child exited normally with a +// non-zero status and no signal. Any other spawn error, a timeout, a signal, +// and EPIPE or a non-zero stdinUnreadBytes with status 0 (a success that did +// not consume its input) fail. Callers keep their own status, stdout and +// stderr checks. import assert from "node:assert/strict"; +import {spawnSync} from "node:child_process"; +import {closeSync, mkdtempSync, openSync, readSync, rmdirSync, rmSync, writeFileSync} from "node:fs"; +import {tmpdir} from "node:os"; +import {join} from "node:path"; + +const passedOptions = new Set(["cwd", "encoding", "env", "killSignal", "maxBuffer", "timeout"]); +const ignoredStdin = ["ignore", "pipe", "pipe"]; + +export function spawnWithInputFile(file, args, options = {}) { + if (!Array.isArray(args)) { + throw new TypeError("spawnWithInputFile: args must be an array; spawnWithInputFile(file, options) is refused"); + } + const {input, stdio, ...passed} = options; + for (const key of Object.keys(passed)) { + if (!passedOptions.has(key)) throw new TypeError(`spawnWithInputFile: unsupported option ${key}`); + } + if (stdio !== undefined) { + if (!Array.isArray(stdio) || stdio.length !== ignoredStdin.length || stdio.some((value, index) => value !== ignoredStdin[index])) { + throw new TypeError('spawnWithInputFile: stdio must be absent or ["ignore", "pipe", "pipe"]'); + } + if (input !== undefined) throw new TypeError("spawnWithInputFile: input with an ignored standard input"); + return spawnSync(file, args, {...passed, stdio}); + } + const bytes = inputBytes(input, passed.encoding); + const directory = mkdtempSync(join(tmpdir(), "proofkit-cli-stdin-")); + const path = join(directory, "stdin"); + let descriptor; + try { + writeFileSync(path, bytes, {flag: "wx", mode: 0o600}); + descriptor = openSync(path, "r"); + } finally { + // Path calls only: the removal needs no descriptor of its own. If it + // fails, the input descriptor is closed before the throw. + try { + rmSync(path, {force: true}); + rmdirSync(directory); + } catch (error) { + if (descriptor !== undefined) closeSync(descriptor); + throw error; + } + } + try { + // Only the caller's options pass: an explicit undefined maxBuffer would lift spawnSync's default bound. + const result = spawnSync(file, args, {...passed, stdio: [descriptor, "pipe", "pipe"]}); + result.stdinUnreadBytes = unreadBytes(descriptor); + return result; + } finally { + closeSync(descriptor); + } +} + +// As spawnSync does, a falsy input is no input and a string is encoded with +// options.encoding. +function inputBytes(input, encoding) { + if (!input) return Buffer.alloc(0); + if (typeof input === "string") return Buffer.from(input, encoding); + if (ArrayBuffer.isView(input)) return input; + throw new TypeError("spawnWithInputFile: input must be a string, Buffer, TypedArray or DataView"); +} + +// The child inherited this open file description, so its read(2) calls +// advanced the shared offset; reading from that offset counts the bytes past +// it. +function unreadBytes(descriptor) { + const buffer = Buffer.allocUnsafe(1 << 16); + let total = 0; + for (let count; (count = readSync(descriptor, buffer, 0, buffer.length, null)) > 0;) total += count; + return total; +} export function assertCLITransport(result, label = "CLI") { - const observed = `${label}: transport error ${result.error}, status ${result.status}, signal ${result.signal}`; + const observed = `${label}: transport error ${result.error}, status ${result.status}, signal ${result.signal}, unread input bytes ${result.stdinUnreadBytes}`; + const refusal = Number.isInteger(result.status) && result.status !== 0; assert.equal(result.signal, null, observed); if (result.error !== undefined) { assert.equal(result.error.code, "EPIPE", observed); - assert.ok(Number.isInteger(result.status) && result.status !== 0, observed); + assert.ok(refusal, observed); } + if (result.stdinUnreadBytes !== undefined && result.stdinUnreadBytes !== 0) assert.ok(refusal, observed); return result; } diff --git a/scripts/cli-process.test.mjs b/scripts/cli-process.test.mjs index aad33e9c..5d80be47 100644 --- a/scripts/cli-process.test.mjs +++ b/scripts/cli-process.test.mjs @@ -1,13 +1,15 @@ import assert from "node:assert/strict"; -import {execFileSync, spawnSync} from "node:child_process"; -import {mkdtempSync, rmSync} from "node:fs"; +import {execFileSync, spawn} from "node:child_process"; +import fs, {mkdtempSync, readFileSync, readdirSync, rmSync, statSync} from "node:fs"; +import {syncBuiltinESMExports} from "node:module"; import {tmpdir} from "node:os"; -import {join} from "node:path"; +import {dirname, join, sep} from "node:path"; import test, {after, before} from "node:test"; -import {assertCLITransport} from "./cli-process.mjs"; +import {fileURLToPath} from "node:url"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; // 8 MiB exceeds every pipe buffer, so a child that exits without reading stdin -// deterministically leaves spawnSync with an unwritten payload and EPIPE. +// deterministically leaves a pipe writer with an unwritten payload and EPIPE. const payload = "x".repeat(8 << 20); const root = new URL("../", import.meta.url); let directory, binary; @@ -19,33 +21,56 @@ before(() => { }); }); after(() => { if (directory) rmSync(directory, {recursive: true, force: true}); }); -const invoke = (file, args, input, timeout = 20000, maxBuffer = 2 << 20) => spawnSync(file, args, {input, encoding: "utf8", timeout, maxBuffer}); +const invoke = (file, args, input, timeout = 20000, maxBuffer = 2 << 20) => spawnWithInputFile(file, args, {input, encoding: "utf8", timeout, maxBuffer}); const fakeChild = (source, input = payload, maxBuffer) => invoke(process.execPath, ["-e", source], input, 20000, maxBuffer); const rejected = (result, label) => assert.throws(() => assertCLITransport(result, label), {code: "ERR_ASSERTION"}, label); +// The asynchronous pipe transport of scripts/schema-completeness.test.mjs: the +// parent writes stdin through a pipe and reports EPIPE when the child is gone. +const pipeChild = (file, args, input) => new Promise(resolve => { + const child = spawn(file, args); + let failure, stdinError; + child.on("error", error => { failure ??= error; }); + child.stdin.on("error", error => { stdinError ??= error; }); + child.stdout.resume(); child.stderr.resume(); + child.on("close", (status, signal) => resolve({status, signal, error: failure ?? stdinError})); + child.stdin.end(input); +}); -test("a CLI flag refusal that exits before reading stdin is accepted as a closed pipe", () => { +test("a CLI flag refusal that exits before reading stdin is accepted with its input unread", async () => { const args = ["change", "plan", "--input", "-", "--agent-envelope", "--format", "text"]; const refusal = invoke(binary, args, payload); - assert.equal(refusal.error?.code, "EPIPE", "the oversized payload must outlast the refusing child"); + assert.equal(refusal.error, undefined); assert.equal(refusal.stdinUnreadBytes, payload.length, "the refusal must read none of its input"); assert.equal(assertCLITransport(refusal, "early refusal"), refusal); assert.equal(refusal.status, 1); assert.equal(refusal.stdout, ""); assert.notEqual(refusal.stderr, ""); const small = assertCLITransport(invoke(binary, args, "{}"), "small refusal"); - assert.equal(small.status, 1); assert.equal(small.stderr, refusal.stderr); + assert.equal(small.status, 1); assert.equal(small.stderr, refusal.stderr); assert.equal(small.stdinUnreadBytes, 2); + const piped = await pipeChild(binary, args, payload); + assert.equal(piped.error?.code, "EPIPE", "the oversized payload must outlast the refusing child"); + assert.equal(assertCLITransport(piped, "early refusal through a pipe"), piped); assert.equal(piped.status, 1); }); -test("EPIPE from a child that exits 0 without reading its required input is rejected", () => { +test("unread input or EPIPE from a child that exits 0 is rejected", async () => { const result = fakeChild("process.exit(0)"); - assert.equal(result.error?.code, "EPIPE"); assert.equal(result.status, 0); assert.equal(result.signal, null); + assert.equal(result.error, undefined); assert.equal(result.status, 0); assert.equal(result.stdinUnreadBytes, payload.length); rejected(result, "unread input with status 0"); + const partial = fakeChild("require('node:fs').readSync(0, Buffer.alloc(1024)); process.exit(0)"); + assert.equal(partial.status, 0); assert.equal(partial.stdinUnreadBytes, payload.length - 1024); + rejected(partial, "partly read input with status 0"); + const piped = await pipeChild(process.execPath, ["-e", "process.exit(0)"], payload); + assert.equal(piped.error?.code, "EPIPE"); assert.equal(piped.status, 0); assert.equal(piped.signal, null); + rejected(piped, "EPIPE with status 0"); }); -test("a child killed by a signal is rejected with or without EPIPE", () => { +test("a child killed by a signal is rejected with or without unread input", async () => { const result = fakeChild("process.kill(process.pid, 'SIGKILL')"); - assert.equal(result.error?.code, "EPIPE"); assert.equal(result.status, null); assert.equal(result.signal, "SIGKILL"); - rejected(result, "signal with EPIPE"); + assert.equal(result.status, null); assert.equal(result.signal, "SIGKILL"); assert.equal(result.stdinUnreadBytes, payload.length); + rejected(result, "signal with unread input"); const withoutInput = fakeChild("process.kill(process.pid, 'SIGKILL')", ""); - assert.equal(withoutInput.error, undefined); assert.equal(withoutInput.signal, "SIGKILL"); - rejected(withoutInput, "signal without EPIPE"); + assert.equal(withoutInput.error, undefined); assert.equal(withoutInput.signal, "SIGKILL"); assert.equal(withoutInput.stdinUnreadBytes, 0); + rejected(withoutInput, "signal without input"); + const piped = await pipeChild(process.execPath, ["-e", "process.kill(process.pid, 'SIGKILL')"], payload); + assert.equal(piped.error?.code, "EPIPE"); assert.equal(piped.signal, "SIGKILL"); + rejected(piped, "signal with EPIPE"); }); test("spawn errors other than EPIPE and timeouts are rejected", () => { @@ -60,3 +85,257 @@ test("spawn errors other than EPIPE and timeouts are rejected", () => { assert.equal(timeout.error?.code, "ETIMEDOUT"); rejected(timeout, "timeout"); }); + +// The child reports the kind of its standard input and the bytes it read. +const stdinProbe = "const fs = require('node:fs'); const stat = fs.fstatSync(0); const bytes = fs.readFileSync(0);" + + " process.stdout.write(JSON.stringify({file: stat.isFile(), socket: stat.isSocket(), fifo: stat.isFIFO(), device: stat.isCharacterDevice(), hex: bytes.toString('hex')}))"; + +test("input reaches the child as a regular file that it reads to the end", () => { + const cases = [ + ["string", "caf\u00e9 \u{1F600}\n", Buffer.from("caf\u00e9 \u{1F600}\n", "utf8")], + ["Buffer", Buffer.from([0, 255, 10, 13]), Buffer.from([0, 255, 10, 13])], + ["Uint8Array", new Uint8Array([1, 2, 3]), Buffer.from([1, 2, 3])], + ["empty string", "", Buffer.alloc(0)], + ["absent", undefined, Buffer.alloc(0)], + ]; + for (const [label, input, expected] of cases) { + const result = assertCLITransport(invoke(process.execPath, ["-e", stdinProbe], input), label); + assert.equal(result.status, 0, `${label}: ${result.stderr}`); assert.equal(result.stdinUnreadBytes, 0, label); + assert.deepEqual(JSON.parse(result.stdout), {file: true, socket: false, fifo: false, device: false, hex: expected.toString("hex")}, label); + } + const latin1 = spawnWithInputFile(process.execPath, ["-e", stdinProbe], {input: "\u00e9", encoding: "latin1"}); + assert.equal(JSON.parse(latin1.stdout).hex, "e9", "a string input is encoded with options.encoding, as spawnSync does"); + const ignored = spawnWithInputFile(process.execPath, ["-e", stdinProbe], {stdio: ["ignore", "pipe", "pipe"], encoding: "utf8"}); + assert.equal(ignored.status, 0, ignored.stderr); assert.equal(ignored.stdinUnreadBytes, undefined); + assert.deepEqual(JSON.parse(ignored.stdout), {file: false, socket: false, fifo: false, device: true, hex: ""}); +}); + +test("the helper keeps spawnSync's options and refuses what it cannot keep", () => { + // Without maxBuffer the child still has spawnSync's default bound. + const bounded = spawnWithInputFile(process.execPath, ["-e", "process.stdout.write('x'.repeat(2 << 20))"], {input: "{}", encoding: "utf8"}); + assert.equal(bounded.error?.code, "ENOBUFS"); + const options = spawnWithInputFile(process.execPath, ["-e", "process.stdout.write(process.cwd() + ' ' + process.env.PROOFKIT_PROBE)"], + {cwd: directory, env: {...process.env, PROOFKIT_PROBE: "kept"}, encoding: "utf8", timeout: 20000, killSignal: "SIGKILL", maxBuffer: 1 << 20}); + assert.equal(options.status, 0, options.stderr); assert.match(options.stdout, / kept$/); + assert.throws(() => spawnWithInputFile(process.execPath, [], {shell: true}), /unsupported option shell/); + assert.throws(() => spawnWithInputFile(process.execPath, [], {stdio: "pipe"}), /stdio must be absent/); + assert.throws(() => spawnWithInputFile(process.execPath, [], {stdio: ["pipe", "pipe", "pipe"]}), /stdio must be absent/); + assert.throws(() => spawnWithInputFile(process.execPath, [], {stdio: ["ignore", "pipe", "pipe"], input: "{}"}), /ignored standard input/); + assert.throws(() => spawnWithInputFile(process.execPath, [], {input: 7}), /input must be/); + // spawnSync(file, options) would take {input} as its options and drop the third argument. + assert.throws(() => spawnWithInputFile(process.execPath, {input: "{}"}), /args must be an array/, "the two-argument form"); + assert.throws(() => spawnWithInputFile(process.execPath, undefined, {input: "{}"}), /args must be an array/); + assert.throws(() => spawnWithInputFile(process.execPath, "-e", {input: "{}"}), /args must be an array/); +}); + +// The helper's temporary root is a fresh TMPDIR, so the test sees every entry +// the helper leaves there. A wrapper of fs.openSync records the modes of the +// input file and of its directory at the read-only open (writeFileSync's own +// open for writing passes through), and can make that open fail; the child +// reports what it sees while it runs. Wrappers of fs.writeFileSync, fs.rmSync +// and fs.rmdirSync can make the write fail after it created the file, or the +// first removal call fail; the open descriptors of this process are the +// entries of /dev/fd. +test("the input file leaves the disk before the child runs, and no file remains after a call or a throw", () => { + const root = mkdtempSync(join(tmpdir(), "proofkit-cli-process-tmp-")); + const savedTMPDIR = process.env.TMPDIR, opened = []; + const original = {openSync: fs.openSync, writeFileSync: fs.writeFileSync, rmSync: fs.rmSync, rmdirSync: fs.rmdirSync}; + const inRoot = path => typeof path === "string" && path.startsWith(root + sep); + const simulated = (message, code) => Object.assign(new Error(message), {code}); + let failOpen = false, failWrite = false, failRemoval = false; + fs.openSync = (path, ...rest) => { + if (inRoot(path) && rest[0] === "r") { + opened.push({directory: statSync(dirname(path)).mode & 0o777, file: statSync(path).mode & 0o777}); + if (failOpen) throw simulated("simulated open failure", "EMFILE"); + } + return original.openSync(path, ...rest); + }; + fs.writeFileSync = (path, data, ...rest) => { + if (!inRoot(path) || !failWrite) return original.writeFileSync(path, data, ...rest); + original.writeFileSync(path, "", ...rest); + throw simulated("simulated write failure", "ENOSPC"); + }; + const removal = name => (path, ...rest) => { + if (inRoot(path) && failRemoval) { failRemoval = false; throw simulated(`simulated ${name} failure`, "EBUSY"); } + return original[name](path, ...rest); + }; + fs.rmSync = removal("rmSync"); fs.rmdirSync = removal("rmdirSync"); + syncBuiltinESMExports(); + process.env.TMPDIR = root; + try { + const probe = "const fs = require('node:fs'); const stat = fs.fstatSync(0);" + + " process.stdout.write(JSON.stringify({mode: stat.mode & 0o777, links: stat.nlink, entries: fs.readdirSync(process.env.TMPDIR), bytes: fs.readFileSync(0).length}))"; + const result = assertCLITransport(spawnWithInputFile(process.execPath, ["-e", probe], {input: '{"a":1}', encoding: "utf8", timeout: 20000}), "residue probe"); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(result.stdout), {mode: 0o600, links: 0, entries: [], bytes: 7}, "the child must read an unlinked 0600 file, with no input left on disk"); + assert.deepEqual(opened, [{directory: 0o700, file: 0o600}], "the directory is 0700 and the file 0600 up to the open"); + assert.deepEqual(readdirSync(root), [], "no file may remain after a call"); + assert.throws(() => spawnWithInputFile(process.execPath, ["-e", ""], {input: "{}", timeout: -1}), {code: "ERR_OUT_OF_RANGE"}); + assert.deepEqual(readdirSync(root), [], "no file may remain after spawnSync throws"); + failOpen = true; + assert.throws(() => spawnWithInputFile(process.execPath, ["-e", ""], {input: "{}"}), {code: "EMFILE"}); + assert.deepEqual(readdirSync(root), [], "no file may remain after the open throws"); + assert.equal(opened.length, 3); + failOpen = false; failWrite = true; + assert.throws(() => spawnWithInputFile(process.execPath, ["-e", ""], {input: "{}"}), {code: "ENOSPC"}); + assert.deepEqual(readdirSync(root), [], "no file may remain after the write throws"); + failWrite = false; failRemoval = true; + const descriptors = readdirSync("/dev/fd").length; + assert.throws(() => spawnWithInputFile(process.execPath, ["-e", ""], {input: "{}"}), {code: "EBUSY"}); + assert.equal(readdirSync("/dev/fd").length, descriptors, "a failed removal must close the input descriptor"); + } finally { + Object.assign(fs, original); + syncBuiltinESMExports(); + if (savedTMPDIR === undefined) delete process.env.TMPDIR; else process.env.TMPDIR = savedTMPDIR; + rmSync(root, {recursive: true, force: true}); + } +}); + +// Source check: a lexical scan of the .js, .cjs, .mjs, .ts, .cts and .mts files +// under scripts/ and tests/ (node_modules excluded) that refuses an `input` +// member in the arguments of a direct spawnSync, execFileSync or execSync +// call; spawnWithInputFile carries input instead. It skips comments, strings, +// template text and regular expressions, finds each name of the three +// functions followed by "(", also as a member (x.spawnSync(...)), and reports +// every `input` member of an object literal anywhere in that call's +// arguments: a plain or quoted key, a string-literal computed key +// (["input"]), a shorthand, or a method with a plain or quoted name. It does +// not see options assembled outside the call, a synchronous spawn function +// under another name, calls through .call, .apply, Reflect.apply or an +// optional call ?.(), a bracketed callee (cp["spawnSync"](...)), a call with +// type arguments (spawnSync(...)), getter, setter, async or generator +// members, methods with a computed key (["input"]() {...}), computed keys +// other than string literals, or escaped key names. Its lexer chooses between +// a division and a regular expression from the previous token alone, so it +// misreads some code: for example a / after ++, --, a template literal, an +// object literal, a property named like a keyword that a regular expression +// can follow (x.in / y) or a name with a non-ASCII letter, a regular +// expression after ) or export default (export default /'/), and a hashbang +// line with an unbalanced quote. A misreading can hide the code after it. +const synchronousSpawns = new Set(["spawnSync", "execFileSync", "execSync"]); +const regexAfter = new Set(["await", "case", "delete", "do", "else", "in", "instanceof", "new", "of", "return", "throw", "typeof", "void", "yield"]); + +function tokenize(source) { + const tokens = [], frames = []; + let index = 0, line = 1; + const push = (type, value) => tokens.push({type, value, line}); + const take = end => { line += source.slice(index, end).split("\n").length - 1; const text = source.slice(index, end); index = end; return text; }; + const quoted = quote => { + let end = index + 1; + while (end < source.length && source[end] !== quote) end += source[end] === "\\" ? 2 : 1; + return take(end + 1).slice(1, -1); + }; + const templateText = () => { + let end = index; + while (end < source.length && source[end] !== "`" && !(source[end] === "$" && source[end + 1] === "{")) end += source[end] === "\\" ? 2 : 1; + take(end); + if (source[index] === "`") { take(index + 1); return; } + take(index + 2); frames.push(0); + }; + const regexAllowed = () => { + const previous = tokens.at(-1); + if (previous === undefined) return true; + if (previous.type === "name") return regexAfter.has(previous.value); + return previous.type === "punct" && previous.value !== ")" && previous.value !== "]"; + }; + while (index < source.length) { + const char = source[index], next = source[index + 1]; + if (/\s/.test(char)) { take(index + 1); continue; } + if (char === "/" && next === "/") { const end = source.indexOf("\n", index); take(end < 0 ? source.length : end); continue; } + if (char === "/" && next === "*") { const end = source.indexOf("*/", index + 2); take(end < 0 ? source.length : end + 2); continue; } + if (char === "'" || char === "\"") { push("string", quoted(char)); continue; } + if (char === "`") { take(index + 1); templateText(); continue; } + if (char === "}" && frames.length > 0 && frames.at(-1) === 0) { frames.pop(); take(index + 1); templateText(); continue; } + if (char === "/" && regexAllowed()) { + let end = index + 1, inClass = false; + while (end < source.length && (inClass || source[end] !== "/")) { + if (source[end] === "\\") end++; + else if (source[end] === "[") inClass = true; + else if (source[end] === "]") inClass = false; + end++; + } + end++; + while (/[a-z]/i.test(source[end] ?? "")) end++; + push("regex", take(end)); + continue; + } + const name = /^[A-Za-z_$][\w$]*/.exec(source.slice(index, index + 256)); + if (name) { push("name", take(index + name[0].length)); continue; } + const number = /^[0-9][\w.]*/.exec(source.slice(index, index + 64)); + if (number) { push("number", take(index + number[0].length)); continue; } + const punct = source.startsWith("...", index) ? "..." : char; + if (frames.length > 0 && punct === "{") frames[frames.length - 1]++; + if (frames.length > 0 && punct === "}") frames[frames.length - 1]--; + push("punct", take(index + punct.length)); + } + return tokens; +} + +function inputPipeViolations(source) { + const tokens = tokenize(source), violations = []; + const opens = {"(": ")", "[": "]", "{": "}"}; + for (let start = 0; start < tokens.length - 1; start++) { + const callee = tokens[start]; + if (callee.type !== "name" || !synchronousSpawns.has(callee.value) || tokens[start + 1].value !== "(") continue; + if (tokens[start - 1]?.value === "function") continue; + const stack = []; + for (let at = start + 1; at < tokens.length; at++) { + const token = tokens[at]; + if (token.type === "punct" && Object.hasOwn(opens, token.value)) { stack.push(token.value); continue; } + if (token.type === "punct" && Object.values(opens).includes(token.value)) { + stack.pop(); + if (stack.length === 0) break; + continue; + } + if (token.value !== "input" || (token.type !== "name" && token.type !== "string")) continue; + const before = tokens[at - 1].value, after = tokens[at + 1]?.value; + const member = stack.at(-1) === "{" && (before === "{" || before === ",") && [":", ",", "}", "("].includes(after); + const computed = token.type === "string" && before === "[" && after === "]" && tokens[at + 2]?.value === ":"; + if (member || computed) violations.push(`line ${token.line}: ${callee.value} passes input`); + } + } + return violations; +} + +test("the source check finds the input option in synchronous spawn calls", () => { + const found = [ + "spawnSync(binary, args, {input: text, encoding: 'utf8'});", + "spawnSync(binary, args, {\n encoding: 'utf8',\n input,\n});", + "child_process.spawnSync(binary, args, {'input': text});", + "execFileSync(binary, args, {\"input\": text});", + "execFileSync(binary, {input: text});", + "execSync(`cat ${file}`, {[\"input\"]: text});", + "spawnSync(file, args, {...(argvOnly ? {stdio: ['ignore', 'pipe', 'pipe']} : {input}), timeout: 1});", + "const re = /\\)/; spawnSync(file, [`${x}`, re.source], {env: {...process.env}, input: x});", + "spawnSync(binary, args, {input() { return text; }});", + ]; + for (const source of found) assert.equal(inputPipeViolations(source).length, 1, source); + const clean = [ + "spawnSync(binary, ['--input', '-'], {encoding: 'utf8'});", + "spawnWithInputFile(binary, args, {input: text});", + "spawnSync(binary, args, {env: {...process.env, X: input}});", + "spawnSync(make(a, input), args, {timeout: input ? 1 : 2});", + "// spawnSync(binary, args, {input: text});\nspawnSync(binary, args);", + "const text = 'spawnSync(binary, args, {input: text})'; const t = `spawnSync(b, a, {input: ${'x'}})`;", + "const help = {input: 1}; spawnSync(binary, args, {stdio: 'ignore'});", + "function spawnSync(file, args, {input} = {}) {}", + ]; + for (const source of clean) assert.deepEqual(inputPipeViolations(source), [], source); +}); + +test("no direct synchronous spawn call under scripts or tests passes an input member", () => { + const rootPath = fileURLToPath(root), violations = []; + let scanned = 0; + for (const top of ["scripts", "tests"]) { + for (const entry of readdirSync(join(rootPath, top), {recursive: true, withFileTypes: true})) { + if (!entry.isFile() || !/\.[cm]?[jt]s$/.test(entry.name)) continue; + const path = join(entry.parentPath, entry.name); + if (path.split(/[\\/]/).includes("node_modules")) continue; + const source = readFileSync(path, "utf8"); + scanned += tokenize(source).filter(token => token.type === "name" && synchronousSpawns.has(token.value)).length; + for (const violation of inputPipeViolations(source)) violations.push(`${path.slice(rootPath.length)} ${violation}`); + } + } + assert.ok(scanned > 0, "the scan must see the synchronous spawn calls of the build steps"); + assert.deepEqual(violations, [], "use spawnWithInputFile from scripts/cli-process.mjs for stdin input"); +}); diff --git a/scripts/compact-proof-structure.test.mjs b/scripts/compact-proof-structure.test.mjs index 474e8c53..a4d5c514 100644 --- a/scripts/compact-proof-structure.test.mjs +++ b/scripts/compact-proof-structure.test.mjs @@ -1,11 +1,11 @@ import assert from "node:assert/strict"; -import {execFileSync, spawnSync} from "node:child_process"; +import {execFileSync} from "node:child_process"; import {mkdtempSync, readFileSync, rmSync} from "node:fs"; import {tmpdir} from "node:os"; import {join} from "node:path"; import test, {after, before} from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; -import {assertCLITransport} from "./cli-process.mjs"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; const root = new URL("../", import.meta.url); const contract = JSON.parse(readFileSync(new URL("proofkit/cli-contract.v2.json", root), "utf8")); @@ -34,7 +34,7 @@ before(() => { after(() => { if (directory) rmSync(directory, {recursive: true, force: true}); }); function native(input, accepted, label) { - const result = spawnSync(binary, ["requirement-proof-resolver", "--input", "-", "--local-environment-class", "local-go"], { + const result = spawnWithInputFile(binary, ["requirement-proof-resolver", "--input", "-", "--local-environment-class", "local-go"], { input: typeof input === "string" ? input : JSON.stringify(input), encoding: "utf8", timeout: 10_000, maxBuffer: 2 << 20, }); diff --git a/scripts/consumer-evidence-structure.test.mjs b/scripts/consumer-evidence-structure.test.mjs index 32022772..f609a110 100644 --- a/scripts/consumer-evidence-structure.test.mjs +++ b/scripts/consumer-evidence-structure.test.mjs @@ -1,12 +1,12 @@ import assert from 'node:assert/strict'; import {createHash} from 'node:crypto'; -import {execFileSync, spawnSync} from 'node:child_process'; +import {execFileSync} from 'node:child_process'; import {mkdtempSync, readFileSync, rmSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {join} from 'node:path'; import test, {before, after} from 'node:test'; import Ajv2020 from 'ajv/dist/2020.js'; -import {assertCLITransport} from './cli-process.mjs'; +import {assertCLITransport, spawnWithInputFile} from './cli-process.mjs'; const root = new URL('../', import.meta.url); const external = 'external-consumer', registry = 'registry-consumer', composer = 'registry-consumer-proof-input-compose'; @@ -65,7 +65,7 @@ const conservesPredecessor = (row, stdout) => { return predecessor !== null && hash(predecessor) === row.stdoutSHA256; }; function invoke(command, input, argv = [command, '--input', '-']) { - const result = spawnSync(binary, argv, {input: typeof input === 'string' ? input : JSON.stringify(input), encoding: 'utf8', timeout: 10000, maxBuffer: 2 << 20}); + const result = spawnWithInputFile(binary, argv, {input: typeof input === 'string' ? input : JSON.stringify(input), encoding: 'utf8', timeout: 10000, maxBuffer: 2 << 20}); return assertCLITransport(result); } const seed = command => JSON.parse(baseline.observations.find(x => x.command === command && x.name === 'valid').input); diff --git a/scripts/explicit-input-structure.test.mjs b/scripts/explicit-input-structure.test.mjs index 554db122..da1a3808 100644 --- a/scripts/explicit-input-structure.test.mjs +++ b/scripts/explicit-input-structure.test.mjs @@ -1,12 +1,12 @@ import assert from "node:assert/strict"; import {createHash} from "node:crypto"; -import {execFileSync, spawnSync} from "node:child_process"; +import {execFileSync} from "node:child_process"; import {mkdtempSync, readFileSync, rmSync} from "node:fs"; import {tmpdir} from "node:os"; import {join} from "node:path"; import test, {before, after} from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; -import {assertCLITransport} from "./cli-process.mjs"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; const root = new URL("../", import.meta.url); const contract = JSON.parse(readFileSync(process.env.PROOFKIT_EXPLICIT_INPUT_CONTRACT || new URL("proofkit/cli-contract.v2.json", root), "utf8")); @@ -40,7 +40,7 @@ function containers(value, path = []) { return [path, ...Object.entries(value).flatMap(([key, child]) => containers(child, [...path, key]))]; } function invoke(command, input, args = []) { - const r = spawnSync(binary, [command, "--input", "-", ...args], {input: typeof input === "string" ? input : JSON.stringify(input), encoding: "utf8", timeout: 10000, maxBuffer: 4 << 20}); + const r = spawnWithInputFile(binary, [command, "--input", "-", ...args], {input: typeof input === "string" ? input : JSON.stringify(input), encoding: "utf8", timeout: 10000, maxBuffer: 4 << 20}); assertCLITransport(r, command); return r; } diff --git a/scripts/gradual-adoption-structure.test.mjs b/scripts/gradual-adoption-structure.test.mjs index ae7d8872..568f3f73 100644 --- a/scripts/gradual-adoption-structure.test.mjs +++ b/scripts/gradual-adoption-structure.test.mjs @@ -1,11 +1,11 @@ import assert from 'node:assert/strict'; -import {execFileSync, spawnSync} from 'node:child_process'; +import {execFileSync} from 'node:child_process'; import {mkdtempSync, readFileSync, rmSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {join} from 'node:path'; import test, {after, before} from 'node:test'; import Ajv2020 from 'ajv/dist/2020.js'; -import {assertCLITransport} from './cli-process.mjs'; +import {assertCLITransport, spawnWithInputFile} from './cli-process.mjs'; import {expandSharedDefinitions} from './schema-definitions.mjs'; const root = new URL('../', import.meta.url); @@ -113,7 +113,7 @@ function schema(command, direction, index, value, expected = true, label = '') { const pathLauncher = {AGENTIC_PROOFKIT_LAUNCHER_PROFILE: 'path', AGENTIC_PROOFKIT_PYTHON_EXECUTABLE: ''}; function invoke(command, input, phase = 'passed', flags = [], inputIndex = 0, launcher = pathLauncher) { if (typeof input !== 'string' && phase !== 'refused' && phase !== 'repair') schema(command, 'input', inputIndex, input); - const result = spawnSync(binary, [command, '--input', '-', ...flags], { + const result = spawnWithInputFile(binary, [command, '--input', '-', ...flags], { input: typeof input === 'string' ? input : JSON.stringify(input), encoding: 'utf8', timeout: 10000, maxBuffer: 8 << 20, env: {...process.env, ...launcher}, }); @@ -602,7 +602,7 @@ test('populated, blocked, failed-reference and normalized payload output domains const value = invoke('gradual-adoption-bootstrap', input, phase); carrierPredicates('gradual-adoption-bootstrap', 0, value, 'payload'); invoke('gradual-adoption', value.payloads.adoptionProfile, phase); - const guidanceResult = spawnSync(binary, ['gradual-adoption-guidance', '--input', '-'], {input: JSON.stringify(value.payloads.adoptionGuidance), encoding: 'utf8', timeout: 10000}); + const guidanceResult = spawnWithInputFile(binary, ['gradual-adoption-guidance', '--input', '-'], {input: JSON.stringify(value.payloads.adoptionGuidance), encoding: 'utf8', timeout: 10000}); assertCLITransport(guidanceResult); assert.equal(guidanceResult.stderr, ''); schema('gradual-adoption-guidance', 'output', 0, JSON.parse(guidanceResult.stdout)); const manifest = invoke('gradual-adoption-bootstrap', input, phase, ['--materialization-manifest']); @@ -795,12 +795,12 @@ test('post-redaction identity and all starter consumers preserve native wire dom schema('gradual-adoption-guidance', 'output', 1, {...envelope, envelopeId: 'foreign-fixture-value'}, false); const execute = (command, input, pointer, exit) => { const flags = pointer ? ['--input-pointer', pointer] : []; - const result = spawnSync(binary, [command, '--input', '-', ...flags], {input, encoding: 'utf8', timeout: 10000, maxBuffer: 8 << 20}); + const result = spawnWithInputFile(binary, [command, '--input', '-', ...flags], {input, encoding: 'utf8', timeout: 10000, maxBuffer: 8 << 20}); assertCLITransport(result); assert.equal(result.status, exit, result.stderr); assert.equal(result.stderr, ''); return result; }; - const normal = spawnSync(binary, ['gradual-adoption-bootstrap', '--input', '-'], {input: JSON.stringify(bootstrap()), encoding: 'utf8', timeout: 10000}); + const normal = spawnWithInputFile(binary, ['gradual-adoption-bootstrap', '--input', '-'], {input: JSON.stringify(bootstrap()), encoding: 'utf8', timeout: 10000}); assertCLITransport(normal); assert.equal(normal.status, 0); assert.equal(normal.stderr, ''); const payloads = JSON.parse(normal.stdout); assert.equal(payloads.nextCommands[2], 'agentic-proofkit witness-plan --input proofkit/witness.json'); @@ -809,14 +809,14 @@ test('post-redaction identity and all starter consumers preserve native wire dom for (const token of ['9007199254740993', '9223372036854775807']) { const input = JSON.stringify(bootstrap()).replace('"profileLines":42', '"profileLines":' + token); assert(input.includes('"profileLines":' + token)); - const producer = spawnSync(binary, ['gradual-adoption-bootstrap', '--input', '-'], {input, encoding: 'utf8', timeout: 10000, maxBuffer: 8 << 20}); + const producer = spawnWithInputFile(binary, ['gradual-adoption-bootstrap', '--input', '-'], {input, encoding: 'utf8', timeout: 10000, maxBuffer: 8 << 20}); assertCLITransport(producer); assert.equal(producer.status, 1); assert.equal(producer.stderr, ''); assert(new RegExp('"profileLines"\\s*:\\s*' + token + '(?=[,}\\s])').test(producer.stdout)); const profile = execute('gradual-adoption', producer.stdout, '/payloads/adoptionProfile', 1); assert(new RegExp('"profileLineCount"\\s*:\\s*' + token + '(?=[,}\\s])').test(profile.stdout)); execute('gradual-adoption-guidance', producer.stdout, '/payloads/adoptionGuidance', 0); execute('witness-plan', producer.stdout, '/payloads/witnessPlanInput', 0); - const manifest = spawnSync(binary, ['gradual-adoption-bootstrap', '--input', '-', '--materialization-manifest'], {input, encoding: 'utf8', timeout: 10000, maxBuffer: 8 << 20}); + const manifest = spawnWithInputFile(binary, ['gradual-adoption-bootstrap', '--input', '-', '--materialization-manifest'], {input, encoding: 'utf8', timeout: 10000, maxBuffer: 8 << 20}); assertCLITransport(manifest); assert.equal(manifest.status, 1); assert.equal(manifest.stderr, ''); const content = JSON.parse(manifest.stdout).files.find(file => file.payloadKey === 'adoptionProfile').content; assert(new RegExp('"profileLines"\\s*:\\s*' + token + '(?=[,}\\s])').test(content)); diff --git a/scripts/impact-structure.test.mjs b/scripts/impact-structure.test.mjs index f8374890..0c968009 100644 --- a/scripts/impact-structure.test.mjs +++ b/scripts/impact-structure.test.mjs @@ -1,12 +1,12 @@ import assert from "node:assert/strict"; -import {execFileSync, spawnSync} from "node:child_process"; +import {execFileSync} from "node:child_process"; import {createHash} from "node:crypto"; import {mkdtempSync, readFileSync, rmSync, writeFileSync} from "node:fs"; import {tmpdir} from "node:os"; import {join} from "node:path"; import test, {before, after} from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; -import {assertCLITransport} from "./cli-process.mjs"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; import {compileVariantUnion} from "./schema-definitions.mjs"; import {composeInput, impactInput, obligation} from "./impact-fixtures.mjs"; @@ -58,7 +58,7 @@ function native(name, value, {exit = 0, report = true, carrier = "stdin", extra const path = join(directory, "input.json"); writeFileSync(path, JSON.stringify({payload: value})); args = [name, "--input", path, "--input-pointer", "/payload", ...extra]; input = ""; } else if (carrier === "compact") args.unshift("--json-layout", "compact"); - const invoke = file => spawnSync(file, args, {input, encoding: "utf8", timeout: 10000, maxBuffer: 4 << 20}); + const invoke = file => spawnWithInputFile(file, args, {input, encoding: "utf8", timeout: 10000, maxBuffer: 4 << 20}); const result = invoke(binary); assertCLITransport(result); assert.equal(result.status, exit, `${name}: ${result.stderr}`); diff --git a/scripts/inventory-coverage-structure.test.mjs b/scripts/inventory-coverage-structure.test.mjs index de352ca0..54f34b4c 100644 --- a/scripts/inventory-coverage-structure.test.mjs +++ b/scripts/inventory-coverage-structure.test.mjs @@ -1,11 +1,11 @@ import assert from "node:assert/strict"; -import {execFileSync, spawnSync} from "node:child_process"; +import {execFileSync} from "node:child_process"; import {mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from "node:fs"; import {tmpdir} from "node:os"; import {join} from "node:path"; import test, {before, after} from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; -import {assertCLITransport} from "./cli-process.mjs"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; import {compileVariantUnion} from "./schema-definitions.mjs"; import {annotatedInventoryInput, directComposeInput, discoveryInput, inventoryInput, normalizedComposeInput, ownerInvariantRegistry, proofInventoryInput, requirementSource, sourceSetInput, twoSourceSetInput, wrappedInventory} from "./inventory-coverage-fixtures.mjs"; @@ -43,7 +43,7 @@ function native(command, input, {flags = [], exit = 0, report = true, carrier = writeFileSync(path, JSON.stringify({payload: input})); args = [command, "--input", path, "--input-pointer", "/payload", ...flags]; bytes = ""; } else if (carrier === "compact") args.unshift("--json-layout", "compact"); - const invoke = path => spawnSync(path, args, {input: bytes, encoding: "utf8", timeout: 10000, maxBuffer: 4 << 20}); + const invoke = path => spawnWithInputFile(path, args, {input: bytes, encoding: "utf8", timeout: 10000, maxBuffer: 4 << 20}); const result = invoke(binary); assertCLITransport(result); for (const value of absent) assert.equal((result.stdout + result.stderr).includes(value), false, "caller value disclosed"); @@ -126,7 +126,7 @@ test("context compose output requires the proof-binding selection members its pr writeFileSync(join(repository, "proofkit/bindings.json"), JSON.stringify(binding)); const catalog = {schemaVersion: 2, catalogId: "catalog.selection", specTree: {path: "specs/tree.json"}, requirementSources: [{path: "specs/requirements.v2.json", nodeId: "spec.root"}], proofBinding: {path: "proofkit/bindings.json"}}; - const result = assertCLITransport(spawnSync(binary, [contextComposer, "--repo-root", repository, "--input", "-"], + const result = assertCLITransport(spawnWithInputFile(binary, [contextComposer, "--repo-root", repository, "--input", "-"], {input: JSON.stringify(catalog), encoding: "utf8", timeout: 10000, maxBuffer: 4 << 20})); assert.equal(result.status, 0, result.stderr); assert.equal(result.stderr, ""); const output = JSON.parse(result.stdout), slice = context => ({schemaVersion: 2, sliceId: "slice.one", context, query: {profile: "proof"}}); diff --git a/scripts/materialization-structure.test.mjs b/scripts/materialization-structure.test.mjs index ff213243..a3212126 100644 --- a/scripts/materialization-structure.test.mjs +++ b/scripts/materialization-structure.test.mjs @@ -1,11 +1,11 @@ import assert from 'node:assert/strict'; -import {execFileSync, spawnSync} from 'node:child_process'; +import {execFileSync} from 'node:child_process'; import {mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {join} from 'node:path'; import test, {after, before} from 'node:test'; import Ajv2020 from 'ajv/dist/2020.js'; -import {assertCLITransport} from './cli-process.mjs'; +import {assertCLITransport, spawnWithInputFile} from './cli-process.mjs'; import {compileVariantUnion} from './schema-definitions.mjs'; const root = new URL('../', import.meta.url); @@ -38,7 +38,7 @@ function native(command, repo, value, flags = [], executable = binary) { const descriptor = contract.commands.find(row => row.command === command); const argv = [...(descriptor.route ?? [command]), '--repo-root', repo, ...flags]; if (value !== undefined) argv.push('--input', '-'); - const result = spawnSync(executable, argv, {input: value === undefined ? undefined : JSON.stringify(value), + const result = spawnWithInputFile(executable, argv, {input: value === undefined ? undefined : JSON.stringify(value), encoding: 'utf8', timeout: 10000, maxBuffer: 2 << 20}); assertCLITransport(result); return result; diff --git a/scripts/overview-claims-structure.test.mjs b/scripts/overview-claims-structure.test.mjs index 99e8a7c9..6e98b851 100644 --- a/scripts/overview-claims-structure.test.mjs +++ b/scripts/overview-claims-structure.test.mjs @@ -1,12 +1,12 @@ import assert from "node:assert/strict"; import {createHash} from "node:crypto"; -import {execFileSync, spawnSync} from "node:child_process"; +import {execFileSync} from "node:child_process"; import {mkdtempSync, readFileSync, rmSync} from "node:fs"; import {tmpdir} from "node:os"; import {join} from "node:path"; import test, {before, after} from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; -import {assertCLITransport} from "./cli-process.mjs"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; const root = new URL("../", import.meta.url), command = "spec-overview-claims"; const contract = JSON.parse(readFileSync(process.env.PROOFKIT_OVERVIEW_CONTRACT || new URL("proofkit/cli-contract.v2.json", root), "utf8")); @@ -44,7 +44,7 @@ before(() => { after(() => {if (directory) rmSync(directory, {recursive: true, force: true});}); const hash = bytes => createHash("sha256").update(bytes).digest("hex"); function invoke(input, argv = [command, "--input", "-"]) { - const result = spawnSync(binary, argv, {input, encoding: "utf8", timeout: 10000, maxBuffer: 2 << 20}); + const result = spawnWithInputFile(binary, argv, {input, encoding: "utf8", timeout: 10000, maxBuffer: 2 << 20}); return assertCLITransport(result); } function reportSemantics(input, output) { diff --git a/scripts/profile-registry-structure.test.mjs b/scripts/profile-registry-structure.test.mjs index d41969a2..d46b816c 100644 --- a/scripts/profile-registry-structure.test.mjs +++ b/scripts/profile-registry-structure.test.mjs @@ -1,11 +1,11 @@ import assert from 'node:assert/strict'; -import {execFileSync, spawnSync} from 'node:child_process'; +import {execFileSync} from 'node:child_process'; import {mkdtempSync, readFileSync, rmSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {join} from 'node:path'; import test, {after, before} from 'node:test'; import Ajv2020 from 'ajv/dist/2020.js'; -import {assertCLITransport} from './cli-process.mjs'; +import {assertCLITransport, spawnWithInputFile} from './cli-process.mjs'; import {expandSharedDefinitions} from './schema-definitions.mjs'; // Independent oracle: expected inventories, vocabularies and fixtures are @@ -63,7 +63,7 @@ function schema(command, direction, index, value, expected = true, label = '') { assert.equal(check(value), expected, `${command}/${direction}/${index}/${label}: ${JSON.stringify(check.errors?.slice(0, 2))}`); } function run(command, input, flags = []) { - const result = spawnSync(binary, [command, '--input', '-', ...flags], { + const result = spawnWithInputFile(binary, [command, '--input', '-', ...flags], { input: typeof input === 'string' ? input : JSON.stringify(input), encoding: 'utf8', timeout: 10000, maxBuffer: 8 << 20, }); assertCLITransport(result); diff --git a/scripts/project-status-structure.test.mjs b/scripts/project-status-structure.test.mjs index bace1f45..ad038fba 100644 --- a/scripts/project-status-structure.test.mjs +++ b/scripts/project-status-structure.test.mjs @@ -1,12 +1,12 @@ import assert from 'node:assert/strict'; import {createHash} from 'node:crypto'; -import {execFileSync, spawnSync} from 'node:child_process'; +import {execFileSync} from 'node:child_process'; import {mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, lstatSync, writeFileSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {join} from 'node:path'; import test from 'node:test'; import Ajv2020 from 'ajv/dist/2020.js'; -import {assertCLITransport} from './cli-process.mjs'; +import {assertCLITransport, spawnWithInputFile} from './cli-process.mjs'; import {compileVariantUnion} from './schema-definitions.mjs'; const root = new URL('../', import.meta.url); @@ -134,7 +134,7 @@ test('actual navigation CLI preserves read-only states and its status-owned next const descriptor = contract.commands.find(row => row.command === command); const args = [...(descriptor.route ?? [command]), '--repo-root', repo, ...flags]; if (input !== undefined) args.push('--input', '-'); - const result = spawnSync(binary, args, {input: input === undefined ? undefined : JSON.stringify(input), encoding: 'utf8', timeout: 10000, maxBuffer: 2 << 20}); + const result = spawnWithInputFile(binary, args, {input: input === undefined ? undefined : JSON.stringify(input), encoding: 'utf8', timeout: 10000, maxBuffer: 2 << 20}); assertCLITransport(result); assert.equal(result.status, expected, result.stderr); return result; } diff --git a/scripts/proof-lookup-structure.test.mjs b/scripts/proof-lookup-structure.test.mjs index 65c51963..317d2dac 100644 --- a/scripts/proof-lookup-structure.test.mjs +++ b/scripts/proof-lookup-structure.test.mjs @@ -1,12 +1,12 @@ import assert from 'node:assert/strict'; import {createHash} from 'node:crypto'; -import {execFileSync, spawnSync} from 'node:child_process'; +import {execFileSync} from 'node:child_process'; import {mkdtempSync, readFileSync, rmSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {join} from 'node:path'; import test, {before, after} from 'node:test'; import Ajv2020 from 'ajv/dist/2020.js'; -import {assertCLITransport} from './cli-process.mjs'; +import {assertCLITransport, spawnWithInputFile} from './cli-process.mjs'; const root = new URL('../', import.meta.url); const source = 'requirement-proof-source-set', view = 'requirement-proof-view'; @@ -62,7 +62,7 @@ before(() => { }); after(() => {if (directory) rmSync(directory, {recursive: true, force: true});}); function invoke(command, value, flags = []) { - const result = spawnSync(binary, [command, '--input', '-', ...flags], { + const result = spawnWithInputFile(binary, [command, '--input', '-', ...flags], { input: JSON.stringify(value), encoding: 'utf8', timeout: 10000, maxBuffer: 2 << 20}); return assertCLITransport(result); } @@ -76,7 +76,7 @@ function output(command, input, flags = []) { test('lookup contracts conserve73actual predecessor process and rendered streams', () => { for (const row of baseline.observations) { - const result = spawnSync(binary, row.argv, {input: row.carrier === 'stdin' ? row.input : undefined, + const result = spawnWithInputFile(binary, row.argv, {input: row.carrier === 'stdin' ? row.input : undefined, stdio: row.carrier === 'argv-only' ? ['ignore', 'pipe', 'pipe'] : undefined, encoding: 'utf8', timeout: 10000, maxBuffer: 2 << 20}); assertCLITransport(result, row.name); @@ -96,7 +96,7 @@ test('the KERNEL-02 stream delta is exact, positioned and limited to its rows', const rows = baseline.observations.filter(row => row.format === 'html'); assert.deepEqual(rows.map(row => row.name), [...delta.rows]); const row = rows[0]; - const current = assertCLITransport(spawnSync(binary, row.argv, {input: row.input, encoding: 'utf8', timeout: 10000, + const current = assertCLITransport(spawnWithInputFile(binary, row.argv, {input: row.input, encoding: 'utf8', timeout: 10000, maxBuffer: 2 << 20}), row.name).stdout; assert.ok(conservesPredecessor(row, current), 'current stdout must carry exactly the declared delta'); const rule = delta.insertion; @@ -108,7 +108,7 @@ test('the KERNEL-02 stream delta is exact, positioned and limited to its rows', assert.equal(conservesPredecessor(row, mutated), false, name); } const unlisted = baseline.observations.find(item => item.format === 'json' && item.exitCode === 0); - const unlistedCurrent = assertCLITransport(spawnSync(binary, unlisted.argv, {input: unlisted.input, encoding: 'utf8', + const unlistedCurrent = assertCLITransport(spawnWithInputFile(binary, unlisted.argv, {input: unlisted.input, encoding: 'utf8', timeout: 10000, maxBuffer: 2 << 20}), unlisted.name).stdout; assert.ok(conservesPredecessor(unlisted, unlistedCurrent), 'unlisted rows stay byte-exact'); assert.equal(conservesPredecessor(unlisted, unlistedCurrent + delta.anchor + rule), false, 'delta applied to an unlisted row'); diff --git a/scripts/proof-routing-structure.test.mjs b/scripts/proof-routing-structure.test.mjs index 9b417d0d..ae424e97 100644 --- a/scripts/proof-routing-structure.test.mjs +++ b/scripts/proof-routing-structure.test.mjs @@ -1,12 +1,12 @@ import assert from "node:assert/strict"; -import {execFileSync, spawnSync} from "node:child_process"; +import {execFileSync} from "node:child_process"; import {createHash} from "node:crypto"; import {mkdtempSync, readFileSync, rmSync, writeFileSync} from "node:fs"; import {tmpdir} from "node:os"; import {join} from "node:path"; import test, {before, after} from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; -import {assertCLITransport} from "./cli-process.mjs"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; import {compileVariantUnion, expandSharedDefinitions} from "./schema-definitions.mjs"; import {bindingInput, witnessInput, projectedWitnessInput, schedulerInput, populatedSchedulerInput, resolverInput} from "./proof-routing-fixtures.mjs"; @@ -62,7 +62,7 @@ function native(command, input, {exit = 0, report = true, carrier = "stdin", ext args = [command, "--input", path, "--input-pointer", "/payload", ...extra]; bytes = ""; } else if (carrier === "compact") args.unshift("--json-layout", "compact"); - const invoke = executable => spawnSync(executable, args, {input: bytes, encoding: "utf8", timeout: 10000, maxBuffer: 4 << 20}); + const invoke = executable => spawnWithInputFile(executable, args, {input: bytes, encoding: "utf8", timeout: 10000, maxBuffer: 4 << 20}); const result = invoke(binary); assertCLITransport(result); assert.equal(result.status, exit, `${command}: ${result.stderr}`); diff --git a/scripts/public-api-structure.test.mjs b/scripts/public-api-structure.test.mjs index 2b36cd31..3d434242 100644 --- a/scripts/public-api-structure.test.mjs +++ b/scripts/public-api-structure.test.mjs @@ -1,11 +1,11 @@ import assert from "node:assert/strict"; -import {execFileSync, spawnSync} from "node:child_process"; +import {execFileSync} from "node:child_process"; import {mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync} from "node:fs"; import {tmpdir} from "node:os"; import {dirname, join} from "node:path"; import test, {before, after} from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; -import {assertCLITransport} from "./cli-process.mjs"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; import {compileVariantUnion} from "./schema-definitions.mjs"; const root = new URL("../", import.meta.url); @@ -51,7 +51,7 @@ function native(input, {files = corpus.observations[0].files, carrier = "stdin"} bytes = ""; } if (carrier === "compact") argv = ["--json-layout", "compact", ...argv]; - const result = spawnSync(binary, argv, {input: bytes, encoding: "utf8", timeout: 10_000, maxBuffer: 4 << 20}); + const result = spawnWithInputFile(binary, argv, {input: bytes, encoding: "utf8", timeout: 10_000, maxBuffer: 4 << 20}); assertCLITransport(result, "native"); assert.ok(result.status === 0 || result.status === 1); if (result.stdout) { diff --git a/scripts/receipt-admission-structure.test.mjs b/scripts/receipt-admission-structure.test.mjs index 780f5f9d..f8d7c49d 100644 --- a/scripts/receipt-admission-structure.test.mjs +++ b/scripts/receipt-admission-structure.test.mjs @@ -1,11 +1,11 @@ import assert from "node:assert/strict"; -import {execFileSync, spawnSync} from "node:child_process"; +import {execFileSync} from "node:child_process"; import {mkdtempSync, readFileSync, rmSync} from "node:fs"; import {tmpdir} from "node:os"; import {join} from "node:path"; import test, {before, after} from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; -import {assertCLITransport} from "./cli-process.mjs"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; const root = new URL("../", import.meta.url); const contract = JSON.parse(readFileSync(process.env.PROOFKIT_RECEIPT_ADMISSION_CONTRACT || new URL("proofkit/cli-contract.v2.json", root), "utf8")); @@ -42,7 +42,7 @@ before(() => { after(() => { if (directory) rmSync(directory, {recursive: true, force: true}); }); function native(family, input, expected, label) { - const result = spawnSync(binary, [family.command, "--input", "-"], { + const result = spawnWithInputFile(binary, [family.command, "--input", "-"], { input: typeof input === "string" ? input : JSON.stringify(input), encoding: "utf8", timeout: 10_000, maxBuffer: 2 << 20, }); assertCLITransport(result, label); diff --git a/scripts/receipt-structure.test.mjs b/scripts/receipt-structure.test.mjs index 0e1bcbcc..f684aef1 100644 --- a/scripts/receipt-structure.test.mjs +++ b/scripts/receipt-structure.test.mjs @@ -1,11 +1,11 @@ import assert from "node:assert/strict"; -import {execFileSync, spawnSync} from "node:child_process"; +import {execFileSync} from "node:child_process"; import {mkdtempSync, readFileSync, rmSync} from "node:fs"; import {tmpdir} from "node:os"; import {join} from "node:path"; import test, {after, before} from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; -import {assertCLITransport} from "./cli-process.mjs"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; const root = new URL("../", import.meta.url); const contract = JSON.parse(readFileSync(new URL("proofkit/cli-contract.v2.json", root), "utf8")); @@ -37,7 +37,7 @@ before(() => { after(() => { if (directory) rmSync(directory, {recursive: true, force: true}); }); function native(family, input, outcome, label) { - const result = spawnSync(binary, [family.command, "--input", "-"], { + const result = spawnWithInputFile(binary, [family.command, "--input", "-"], { input: typeof input === "string" ? input : JSON.stringify(input), encoding: "utf8", timeout: 10_000, maxBuffer: 2 << 20, }); diff --git a/scripts/release-classification-structure.test.mjs b/scripts/release-classification-structure.test.mjs index b7b8222e..094b2bfb 100644 --- a/scripts/release-classification-structure.test.mjs +++ b/scripts/release-classification-structure.test.mjs @@ -1,12 +1,12 @@ import assert from "node:assert/strict"; import {createHash} from "node:crypto"; -import {execFileSync, spawnSync} from "node:child_process"; +import {execFileSync} from "node:child_process"; import {mkdtempSync, readFileSync, rmSync} from "node:fs"; import {tmpdir} from "node:os"; import {join} from "node:path"; import test, {before, after} from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; -import {assertCLITransport} from "./cli-process.mjs"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; import {compileVariantUnion} from "./schema-definitions.mjs"; const root = new URL("../", import.meta.url), release = "release-authority", readiness = "readiness-closeout"; @@ -42,7 +42,7 @@ const order = (a, b) => Buffer.compare(Buffer.from(a, "utf8"), Buffer.from(b, "u const trim = text => text.replace(/^\p{White_Space}+|\p{White_Space}+$/gu, ""); const mergedClaims = (command, values) => [...new Set([...builtin[command], ...values.map(trim)])].sort(order); function invoke(row) { - const result = spawnSync(binary, row.argv, {input: row.input, encoding: "utf8", timeout: 10000, maxBuffer: 2 << 20}); + const result = spawnWithInputFile(binary, row.argv, {input: row.input, encoding: "utf8", timeout: 10000, maxBuffer: 2 << 20}); return assertCLITransport(result); } function expectedHelp(row) { diff --git a/scripts/report-admission-structure.test.mjs b/scripts/report-admission-structure.test.mjs index 387d672c..0bdc547c 100644 --- a/scripts/report-admission-structure.test.mjs +++ b/scripts/report-admission-structure.test.mjs @@ -6,7 +6,7 @@ import {tmpdir} from "node:os"; import {join} from "node:path"; import test, {before, after} from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; -import {assertCLITransport} from "./cli-process.mjs"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; const root = new URL("../", import.meta.url); const contract = JSON.parse(readFileSync(process.env.PROOFKIT_REPORT_ADMISSION_CONTRACT || new URL("proofkit/cli-contract.v2.json", root), "utf8")); @@ -44,7 +44,7 @@ const objects = value => paths(value, value => !Array.isArray(value)); const arrays = value => paths(value, Array.isArray); const wrongType = value => typeof value === "string" ? 0 : typeof value === "number" || typeof value === "boolean" ? "wrong-type" : value === null ? false : Array.isArray(value) ? {} : []; function native(family, input, expected, label, extra = []) { - const result = spawnSync(binary, [family.command, "--input", "-", ...extra], { + const result = spawnWithInputFile(binary, [family.command, "--input", "-", ...extra], { input: typeof input === "string" ? input : JSON.stringify(input), encoding: "utf8", timeout: 10_000, maxBuffer: 8 << 20, }); assertCLITransport(result, label); diff --git a/scripts/scaffold-doctor-structure.test.mjs b/scripts/scaffold-doctor-structure.test.mjs index e1ab942a..37378460 100644 --- a/scripts/scaffold-doctor-structure.test.mjs +++ b/scripts/scaffold-doctor-structure.test.mjs @@ -1,11 +1,11 @@ import assert from 'node:assert/strict'; -import {execFileSync, spawnSync} from 'node:child_process'; +import {execFileSync} from 'node:child_process'; import {mkdtempSync, readFileSync, rmSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {join} from 'node:path'; import test, {after, before} from 'node:test'; import Ajv2020 from 'ajv/dist/2020.js'; -import {assertCLITransport} from './cli-process.mjs'; +import {assertCLITransport, spawnWithInputFile} from './cli-process.mjs'; import {expandSharedDefinitions} from './schema-definitions.mjs'; // Independently authored expectations for adoption-doctor, scaffold-project-structure @@ -194,7 +194,7 @@ function selected(command, value) { } const pathLauncher = {AGENTIC_PROOFKIT_LAUNCHER_PROFILE: 'path', AGENTIC_PROOFKIT_PYTHON_EXECUTABLE: ''}; function run(command, input, flags = [], global = []) { - const result = spawnSync(binary, [...global, command, '--input', '-', ...flags], { + const result = spawnWithInputFile(binary, [...global, command, '--input', '-', ...flags], { input: typeof input === 'string' ? input : JSON.stringify(input), encoding: 'utf8', timeout: 20000, maxBuffer: 16 << 20, env: {...process.env, ...pathLauncher}, }); @@ -735,7 +735,7 @@ test('whole chains reuse the producing commands and their published structures', const bootstrapRun = run('gradual-adoption-bootstrap', input.bootstrap); assert.equal(bootstrapRun.status, 0); assert.equal(bootstrapRun.stderr, ''); assert.deepEqual(value.bootstrapReport, JSON.parse(bootstrapRun.stdout)); - const workflow = spawnSync(binary, ['adoption-workflow-plan', '--input', '-', '--input-pointer', '/adoptionWorkflowInput'], { + const workflow = spawnWithInputFile(binary, ['adoption-workflow-plan', '--input', '-', '--input-pointer', '/adoptionWorkflowInput'], { input: JSON.stringify(value), encoding: 'utf8', timeout: 20000, env: {...process.env, ...pathLauncher}}); assert.equal(workflow.status, 0, workflow.stderr); assert.equal(workflow.stderr, ''); assert.deepEqual(JSON.parse(workflow.stdout), value.adoptionWorkflowPlan); @@ -745,7 +745,7 @@ test('whole chains reuse the producing commands and their published structures', assert.deepEqual(JSON.parse(files[input.paths.workflowInputPath].content), value.adoptionWorkflowInput); assert.deepEqual(JSON.parse(files[input.repoProfileScaffold.repository.profilePath].content), value.repoProfileScaffoldPlan.repoProfileDraft); for (const path of [input.paths.repoProfileScaffoldInputPath, input.paths.bootstrapInputPath]) { - const replay = spawnSync(binary, [path === input.paths.bootstrapInputPath ? 'gradual-adoption-bootstrap' : 'scaffold-profile-plan', '--input', '-'], + const replay = spawnWithInputFile(binary, [path === input.paths.bootstrapInputPath ? 'gradual-adoption-bootstrap' : 'scaffold-profile-plan', '--input', '-'], {input: files[path].content, encoding: 'utf8', timeout: 20000, env: {...process.env, ...pathLauncher}}); assert.equal(replay.status, 0, replay.stderr); } diff --git a/scripts/selective-proof-structure.test.mjs b/scripts/selective-proof-structure.test.mjs index 8143353b..dece4677 100644 --- a/scripts/selective-proof-structure.test.mjs +++ b/scripts/selective-proof-structure.test.mjs @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import {execFileSync, spawnSync} from "node:child_process"; +import {execFileSync} from "node:child_process"; import {createHash} from "node:crypto"; import {mkdtempSync, readFileSync, rmSync, writeFileSync} from "node:fs"; import {tmpdir} from "node:os"; @@ -7,7 +7,7 @@ import {join, relative, isAbsolute} from "node:path"; import {fileURLToPath} from "node:url"; import test, {before, after} from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; -import {assertCLITransport} from "./cli-process.mjs"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; import {compileVariantUnion} from "./schema-definitions.mjs"; import {planInput, populatedPlanInput, evidenceInput, projectionInput, boundProjectionInput, decisionInput, decisionStates} from "./selective-proof-fixtures.mjs"; @@ -59,7 +59,7 @@ function native(name, value, {exit = 0, report = true, extra = [], carrier = "st const path = join(directory, "input.json"); writeFileSync(path, JSON.stringify({payload: value})); args = [name, "--input", path, "--input-pointer", "/payload", ...extra]; input = ""; } else if (carrier === "compact") args.unshift("--json-layout", "compact"); - const invoke = file => spawnSync(file, args, {...(carrier === "argv-only" ? {stdio: ["ignore", "pipe", "pipe"]} : {input}), + const invoke = file => spawnWithInputFile(file, args, {...(carrier === "argv-only" ? {stdio: ["ignore", "pipe", "pipe"]} : {input}), encoding: "utf8", timeout: 10000, maxBuffer: 4 << 20}); const result = invoke(binary); assertCLITransport(result, name); assert.equal(result.status, exit, `${name}: ${result.stderr}`); @@ -326,7 +326,7 @@ test("coverage sourcePath is required nullable across receipt and command-key re test("delimiter-bearing duplicate keys retain command text and only repair the declared legacy fields", () => { const invoke = (file, input) => { - const result = spawnSync(file, [names[1], "--input", "-"], {input: JSON.stringify(input), encoding: "utf8", timeout: 10000, maxBuffer: 4 << 20}); + const result = spawnWithInputFile(file, [names[1], "--input", "-"], {input: JSON.stringify(input), encoding: "utf8", timeout: 10000, maxBuffer: 4 << 20}); assertCLITransport(result); assert.equal(result.status, 1); assert.equal(result.stderr, ""); return result; @@ -386,7 +386,7 @@ test("repaired report hashes bind both envelope references across layouts and om const sourceRefId = "proofkit.agent.context.selective-evidence.001"; const invoke = (file, input, layout) => { const args = layout ? ["--json-layout", layout, names[1], "--input", "-", "--agent-envelope"] : [names[1], "--input", "-"]; - const result = spawnSync(file, args, + const result = spawnWithInputFile(file, args, {input: JSON.stringify(input), encoding: "utf8", timeout: 10000, maxBuffer: 4 << 20}); assertCLITransport(result); assert.equal(result.status, 1); assert.equal(result.stderr, ""); diff --git a/scripts/source-conversion-structure.test.mjs b/scripts/source-conversion-structure.test.mjs index c4c4bea5..ca176d03 100644 --- a/scripts/source-conversion-structure.test.mjs +++ b/scripts/source-conversion-structure.test.mjs @@ -1,12 +1,12 @@ import assert from 'node:assert/strict'; import {createHash} from 'node:crypto'; -import {execFileSync, spawnSync} from 'node:child_process'; +import {execFileSync} from 'node:child_process'; import {mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync} from 'node:fs'; import {tmpdir} from 'node:os'; import {join} from 'node:path'; import test, {after, before} from 'node:test'; import Ajv2020 from 'ajv/dist/2020.js'; -import {assertCLITransport} from './cli-process.mjs'; +import {assertCLITransport, spawnWithInputFile} from './cli-process.mjs'; const root = new URL('../', import.meta.url); const contract = JSON.parse(readFileSync(process.env.PROOFKIT_SOURCE_CONVERSION_CONTRACT || new URL('proofkit/cli-contract.v2.json', root))); @@ -46,7 +46,7 @@ function valid(direction, target, value, expected = true, label = '') { } function run(input, args, options = {}) { - const result = spawnSync(binary, [command, '--input', '-', ...args], { + const result = spawnWithInputFile(binary, [command, '--input', '-', ...args], { input, encoding: 'utf8', timeout: 20000, maxBuffer: 16 << 20, ...options, }); return assertCLITransport(result); diff --git a/scripts/spec-tree-structure.test.mjs b/scripts/spec-tree-structure.test.mjs index 7068e202..8ce0e043 100644 --- a/scripts/spec-tree-structure.test.mjs +++ b/scripts/spec-tree-structure.test.mjs @@ -5,7 +5,7 @@ import path from "node:path"; import {spawnSync} from "node:child_process"; import test from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; -import {assertCLITransport} from "./cli-process.mjs"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; const contract = JSON.parse(fs.readFileSync(new URL("../proofkit/cli-contract.v2.json", import.meta.url), "utf8")); const observations = JSON.parse(fs.readFileSync(new URL("../internal/app/testdata/spec-tree-native-observations.json", import.meta.url), "utf8")); @@ -201,7 +201,7 @@ test("spec tree schemas accept native failed reports visiting undeclared child I input.nodes = input.nodes.filter(node => node.nodeId === input.rootNodeId); input.overlays = []; input.edges = Array.from({length: count}, (_, i) => ({parentNodeId: input.rootNodeId, childNodeId: `missing${String(i).padStart(5, "0")}`})); - const run = spawnSync(binary, ["requirement-spec-tree", "--input", "-"], {env, input: JSON.stringify(input), encoding: "utf8", timeout: 60000, maxBuffer: 32 << 20}); + const run = spawnWithInputFile(binary, ["requirement-spec-tree", "--input", "-"], {env, input: JSON.stringify(input), encoding: "utf8", timeout: 60000, maxBuffer: 32 << 20}); assertCLITransport(run); assert.equal(run.status, count === 0 ? 0 : 1); assert.equal(run.stderr, ""); diff --git a/scripts/workspace-structure.test.mjs b/scripts/workspace-structure.test.mjs index c2f6c60a..033c14cf 100644 --- a/scripts/workspace-structure.test.mjs +++ b/scripts/workspace-structure.test.mjs @@ -6,7 +6,7 @@ import {tmpdir} from "node:os"; import {join} from "node:path"; import test, {before, after} from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; -import {assertCLITransport} from "./cli-process.mjs"; +import {assertCLITransport, spawnWithInputFile} from "./cli-process.mjs"; import {compileVariantUnion, expandSharedDefinitions} from "./schema-definitions.mjs"; const root = new URL("../", import.meta.url); @@ -55,7 +55,7 @@ function native(command, input, extra = [], carrier = "stdin") { bytes = ""; } if (carrier === "compact") argv = ["--json-layout", "compact", ...argv]; - const result = spawnSync(binary, argv, {input: bytes, encoding: "utf8", timeout: 10_000, maxBuffer: 4 << 20}); + const result = spawnWithInputFile(binary, argv, {input: bytes, encoding: "utf8", timeout: 10_000, maxBuffer: 4 << 20}); assertCLITransport(result, command); assert.ok(result.status === 0 || result.status === 1, `${command}/exit`); result.output = undefined; From ad690be01737c5957ce0afd1261410acea5b44bf Mon Sep 17 00:00:00 2001 From: iperev Date: Sun, 25 Oct 2026 21:00:21 +0100 Subject: [PATCH 6/6] fix: retain schema security prerequisite in stdin witnesses --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3b847083..7770ca24 100644 --- a/package-lock.json +++ b/package-lock.json @@ -417,9 +417,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.8", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", - "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.9.tgz", + "integrity": "sha512-hS/0Wt3Ya043oKXs0sIxrMpKQITXhMLIrVAKKXi4gozGC9v7YRWVnXIZzjNPTO02M9BdxLCWqK1yRXgNziVyJQ==", "dev": true, "funding": [ {