From b117998eee281b80b29c20b1b319de58313b9d81 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 17:46:13 +0000 Subject: [PATCH 1/4] ci(ui): pass the per-file timings reporter in Test UI and port its weekly dataset refresh The test-ui job's whole-suite leg now passes ui/scripts/test-file-timings.mjs beside the default and github-actions reporters, and uploads each shard's report as test-file-timings-N on the full battery's events (continue-on-error, a missing report is a warning). The new ui-test-timings-refresh.yml selects the newest eligible runs, downloads their reports, merges them into ui/scripts/test-shard-timings.json and opens the PR as the fleet App, dry-run on pull_request, mirroring shard-timings-refresh.yml. The selector's shard job name follows the root job (Test UI (N/M)), the generator's generated_by names the workflow that now exists, and the pins the objectui ci.yml carried come back against the root ci.yml and the refresh. Report-only: no threshold, no new failing check. Claude-Session: https://claude.ai/code/session_01JQwxbXf3xRiPQrLhrgsFzw Co-Authored-By: Claude --- .github/workflows/ci.yml | 39 +- .github/workflows/ui-test-timings-refresh.yml | 360 ++++++++++++++++++ .../__tests__/test-file-timings.test.ts | 95 ++++- ui/scripts/test-file-timings.mjs | 38 +- 4 files changed, 494 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/ui-test-timings-refresh.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95f27514282..85d2f5fde0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3885,23 +3885,50 @@ jobs: # paths relative to ui/, the vitest root, passed as positionals with no # `--`, the only spelling ui/scripts/vitest-invocation-guard.mjs admits. # - # objectui's full-battery runs also pass its per-file timings reporter - # and upload the result for the weekly dataset refresh. Both belong to - # that refresh workflow, which is in the governance-workflow audit - # (sub-issue 4 of the epic); until it is ported the committed dataset - # is what the sequencer reads. + # THE PER-FILE TIMINGS (objectstack#29517). The whole suite also passes + # ui/scripts/test-file-timings.mjs, the per-file timings reporter, beside + # the two reporters it would otherwise displace. It writes this shard's + # report to $OBJECTUI_TEST_TIMINGS_OUT, and the step after this one + # uploads it on the full battery's events as `test-file-timings-N`. + # .github/workflows/ui-test-timings-refresh.yml merges those reports + # weekly into ui/scripts/test-shard-timings.json, the dataset the shard + # sequencer splits on. Report-only, by ruling: the reporter never throws + # into vitest (a failure is one stderr line and no report), and nothing + # here or there grades a duration. The affected speed passes no reporter: + # a slice of the suite is not a measurement of it. + # ui/scripts/__tests__/test-file-timings.test.ts holds this step, the + # upload and the refresh's `--shards` to the names that script spells. - name: Run the ui tests (shard ${{ matrix.shard }}/16) env: MODE: ${{ steps.select.outputs.mode }} + OBJECTUI_TEST_TIMINGS_OUT: ${{ runner.temp }}/test-file-timings/shard-${{ matrix.shard }}.json run: | if [ "$MODE" = 'affected' ]; then mapfile -t filters < "$RUNNER_TEMP/affected-tests.txt" echo "Affected speed: ${#filters[@]} vitest filter(s), shard ${{ matrix.shard }} of 16." pnpm --dir ui exec vitest run --shard=${{ matrix.shard }}/16 --reporter=default --reporter=github-actions "${filters[@]}" else - pnpm --dir ui run test --shard=${{ matrix.shard }}/16 --reporter=default --reporter=github-actions + pnpm --dir ui run test --shard=${{ matrix.shard }}/16 --reporter=default --reporter=github-actions --reporter=./scripts/test-file-timings.mjs fi + # Diagnostics only, so it can never red a shard: `continue-on-error`, and + # a missing report is a warning, not an error (the refresh's selector + # then skips this run by name). Uploaded on the full battery's events + # only, the deny-list of THE TWO SPEEDS above `jobs:`; a pull request that + # failed open to the whole suite writes a report and uploads nothing. The + # refresh reads the newest fifty completed runs of this workflow on + # `main`, so a push run's report needs two days and a scheduled run's a + # week, which also covers a quiet weekend of scheduled runs alone. + - name: Publish this shard's per-file test timings + if: ${{ always() && github.event_name != 'pull_request' && github.event_name != 'merge_group' }} + continue-on-error: true + uses: actions/upload-artifact@v7 + with: + name: test-file-timings-${{ matrix.shard }} + path: ${{ runner.temp }}/test-file-timings/ + if-no-files-found: warn + retention-days: ${{ github.event_name == 'schedule' && 7 || 2 }} + test-ui-gate: # Stable required-check name for the sharded Test UI matrix (#26460, ruling # A, comment 6435318516), in test-gate's shape and for test-gate's reason diff --git a/.github/workflows/ui-test-timings-refresh.yml b/.github/workflows/ui-test-timings-refresh.yml new file mode 100644 index 00000000000..cc7aaa0b267 --- /dev/null +++ b/.github/workflows/ui-test-timings-refresh.yml @@ -0,0 +1,360 @@ +# Regenerate ui/scripts/test-shard-timings.json on a timer, and open the PR. +# +# ══════════════════════════════════════════════════════════════════════════════ +# WHAT THIS IS (objectstack#29517, ruling 6583487846 on objectstack#29469) +# ══════════════════════════════════════════════════════════════════════════════ +# +# ui/scripts/test-shard-timings.json is the per-file duration dataset the +# Test UI split is balanced on: vitest's `sequence.sequencer` +# (ui/scripts/test-shard-sequencer.mjs) reads it to place every test file of +# the `--shard=N/16` legs. It is GENERATED: the `test-ui` job of ci.yml passes +# the per-file timings reporter on the whole suite and uploads one report per +# shard (`test-file-timings-N`), and this workflow merges the reports of the +# newest eligible runs into the dataset. objectui ran the same loop as its +# `test-shard-timings-refresh.yml`; the import of ui/ into this repository did +# not carry the reporter or the refresh, so the dataset stood still from +# 2026-10-19 while the suite moved under it. This file is that refresh, ported +# beside the Test Core one (shard-timings-refresh.yml), which it mirrors in +# shape: select, regenerate, compare, and open a PR as the fleet App. +# +# ALL THREE PARTS LIVE IN ONE SCRIPT: ui/scripts/test-file-timings.mjs +# --------------------------------------------------------------------- +# the writer (the reporter ci.yml's `test-ui` job passes), +# `select` the newest completed `push` / `schedule` runs of ci.yml on +# `main` whose sixteen `Test UI (N/16)` shards ALL concluded +# `success` and whose sixteen `test-file-timings-N` artifacts +# are all still there (a red or cancelled shard may have +# stopped part-way, and its report would weigh what it reached +# against a run that never reached the rest), +# `merge` per run a file's durations summed, across runs the MEDIAN, +# keyed by `pathKey()` and never by path. +# This file only drives them. Their header is the authority on the numbers. +# +# WHAT THIS WORKFLOW IS NOT ALLOWED TO DO +# -------------------------------------- +# ⛔ It never hand-edits the dataset. Every byte it commits came out of +# `merge`; a hand-touched number under a generated file's `generated_by` +# is worse than a stale one, whose staleness at least shows in +# `measured.at`. +# ⛔ It grades nothing. A reporter and a dataset only, by ruling: no +# threshold, no new failing check. Its red is only its own breakage (a +# generator that refuses, a push that fails), never a slow file. +# ⛔ It never touches the shard matrix, `timeout-minutes` or any bound. +# ⛔ It never opens a PR when the regenerated file is byte-identical. +# +# WHY IT RUNS HERE AND NOT IN A SEAT +# ---------------------------------- +# An artifact download redirects to the Actions blob host, which agent +# containers cannot reach (shard-timings-refresh.yml's header has the +# measurements). A GitHub-hosted runner reaches it natively, so the +# regeneration happens where the data already is. +# +# THE PULL REQUEST LEG +# -------------------- +# A pull request that edits this file or a script it loads runs the selection, +# the downloads and the regeneration for real and skips only the push, the PR +# and the label write, so an edit that breaks the refresh is red before it +# merges. With no eligible run on `main` (the state right after the reporter +# first lands, until a full-battery run has uploaded sixteen reports) it reads +# NOT MEASURED and stays green: nothing was regenerated and nothing failed. +name: UI Test Timings Refresh + +on: + # Mondays, after the Test Core refresh (05:30) and off the :00 minute + # everyone else's cron queues on. + schedule: + - cron: '47 5 * * 1' + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/ui-test-timings-refresh.yml' + - 'ui/scripts/test-file-timings.mjs' + - 'ui/scripts/test-shard-partition.mjs' + - 'ui/scripts/invoked-as.mjs' + +permissions: + contents: read + +concurrency: + group: ui-test-timings-refresh-${{ github.ref }} + cancel-in-progress: true + +jobs: + refresh: + # The name objectui's lane carried, which ui/scripts/dependabot-merge-gate.mjs + # still lists among its optional contexts. + name: Regenerate the per-file test timings + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read # the checkout + actions: read # list runs, read jobs, download the timing artifacts + env: + # The `test-ui` matrix width. test-file-timings.test.ts holds it equal to + # ci.yml's `shard:` list, so a resize that forgets this line reds there. + UI_TEST_SHARDS: '16' + RUN_LIMIT: '6' + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version-file: .nvmrc + + - name: Choose the newest eligible full-battery runs + id: select + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + SELECT_EXIT=0 + node ui/scripts/test-file-timings.mjs select --shards "$UI_TEST_SHARDS" --limit "$RUN_LIMIT" \ + > "$RUNNER_TEMP/runs.json" 2> "$RUNNER_TEMP/select-stderr.txt" || SELECT_EXIT=$? + cat "$RUNNER_TEMP/select-stderr.txt" + if [ "$SELECT_EXIT" -eq 3 ]; then + echo "not_measured=true" >> "$GITHUB_OUTPUT" + echo "::warning::UI test timings NOT MEASURED: no completed full-battery run of ci.yml on main has all $UI_TEST_SHARDS Test UI shards green and all $UI_TEST_SHARDS timing artifacts. The dataset was left untouched." + { + echo "### UI test timings: NOT MEASURED (exit 3), nothing was regenerated and nothing failed" + echo + echo "\`test-file-timings.mjs select\` found no eligible run, so" + echo "\`ui/scripts/test-shard-timings.json\` is untouched. Expected until a full-battery run" + echo "(a \`push\` that starts Test UI, or the 4-hourly \`schedule\`) has uploaded all" + echo "$UI_TEST_SHARDS \`test-file-timings-N\` reports after the reporter landed." + echo + echo "⛔ A NOT MEASURED that persists past a day of scheduled runs is a defect, not a" + echo "steady state: the reporter or its upload is not running in \`test-ui\`. File a card." + echo + echo "The selector's log, verbatim:" + echo + echo '```' + sed 's/> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + if [ "$SELECT_EXIT" -ne 0 ]; then + echo "::error::test-file-timings.mjs select exited $SELECT_EXIT. Nothing was regenerated." + exit "$SELECT_EXIT" + fi + echo "Eligible runs, newest first:" + node -e ' + const runs = JSON.parse(require("fs").readFileSync(process.env.RUNNER_TEMP + "/runs.json", "utf8")); + for (const r of runs) console.log(` ${r.id} ${r.event} ${r.created_at} ${String(r.head_sha).slice(0, 10)}`); + ' + + # One directory per run, named by the run id (the generator reads the + # id off the directory's name), one subdirectory per artifact. A run + # whose download fails or comes back short is dropped by name, and the + # runs file the generator reads is rewritten to the runs it keeps, so + # `measured.at` and `measured.runs` describe exactly what was merged. + - name: Download each run's timing reports + id: download + if: steps.select.outputs.not_measured != 'true' + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + WORK="$RUNNER_TEMP/timings" + mkdir -p "$WORK" + KEPT=() + for RUN_ID in $(node -e ' + for (const r of JSON.parse(require("fs").readFileSync(process.env.RUNNER_TEMP + "/runs.json", "utf8"))) console.log(r.id); + '); do + if ! gh run download "$RUN_ID" -R "$GITHUB_REPOSITORY" --pattern 'test-file-timings-*' --dir "$WORK/$RUN_ID"; then + echo "::warning::Could not download run $RUN_ID's timing reports; dropping that run." + rm -rf "${WORK:?}/$RUN_ID" + continue + fi + REPORTS=$(find "$WORK/$RUN_ID" -name '*.json' -type f | wc -l) + if [ "$REPORTS" -ne "$UI_TEST_SHARDS" ]; then + echo "::warning::Run $RUN_ID carried $REPORTS timing report(s), not $UI_TEST_SHARDS; dropping that run." + rm -rf "${WORK:?}/$RUN_ID" + continue + fi + echo "Run $RUN_ID: $REPORTS timing reports." + KEPT+=("$RUN_ID") + done + if [ "${#KEPT[@]}" -eq 0 ]; then + echo "::error::No selected run's timing reports could be read. Nothing was regenerated." + exit 1 + fi + KEPT_IDS="${KEPT[*]}" node -e ' + const fs = require("fs"); + const keep = new Set(process.env.KEPT_IDS.split(" ")); + const runs = JSON.parse(fs.readFileSync(process.env.RUNNER_TEMP + "/runs.json", "utf8")); + fs.writeFileSync(process.env.RUNNER_TEMP + "/kept-runs.json", JSON.stringify(runs.filter((r) => keep.has(String(r.id))), null, 2)); + ' + echo "runs=${KEPT[*]}" >> "$GITHUB_OUTPUT" + echo "newest=${KEPT[0]}" >> "$GITHUB_OUTPUT" + + - name: Regenerate the dataset + id: generate + if: steps.select.outputs.not_measured != 'true' + env: + RUNS: ${{ steps.download.outputs.runs }} + run: | + set -euo pipefail + DIRS=() + for R in $RUNS; do DIRS+=("$RUNNER_TEMP/timings/$R"); done + node ui/scripts/test-file-timings.mjs merge \ + --runs-file "$RUNNER_TEMP/kept-runs.json" \ + --out ui/scripts/test-shard-timings.json \ + --summary "$RUNNER_TEMP/summary.md" \ + "${DIRS[@]}" + if git diff --quiet -- ui/scripts/test-shard-timings.json; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "The regenerated dataset is BYTE-IDENTICAL to the committed one; no PR." | tee -a "$GITHUB_STEP_SUMMARY" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Compose the pull request body + if: steps.generate.outputs.changed == 'true' + env: + RUNS: ${{ steps.download.outputs.runs }} + run: | + set -euo pipefail + { + echo "Refreshes \`ui/scripts/test-shard-timings.json\`, the per-file dataset the Test UI" + echo "shard split is balanced on. Opened automatically by" + echo "\`.github/workflows/ui-test-timings-refresh.yml\`. Every byte came out of" + echo "\`ui/scripts/test-file-timings.mjs merge\`; nothing here was hand-edited, and no bound," + echo "timeout or matrix entry was touched." + echo + cat "$RUNNER_TEMP/summary.md" + echo + echo "## Source" + echo + echo "Every run below had all $UI_TEST_SHARDS \`Test UI (N/$UI_TEST_SHARDS)\` shards conclude" + echo "\`success\` and all $UI_TEST_SHARDS \`test-file-timings-N\` reports retained; the log of" + echo "the run that opened this PR names every run it skipped and why." + echo + for R in $RUNS; do + echo "- https://github.com/$GITHUB_REPOSITORY/actions/runs/$R" + done + echo + echo "Pushed and opened as the fleet App (\`objectstack-fleet[bot]\`), so this PR's own" + echo "events start CI: no push or reopen is needed to get its checks." + } > "$RUNNER_TEMP/pr-body.md" + echo "Composed a $(wc -l < "$RUNNER_TEMP/pr-body.md")-line pull request body." + + - name: Check the fleet App credential is visible to this repository + id: credential + if: steps.generate.outputs.changed == 'true' && github.event_name != 'pull_request' + env: + HAS_APP_ID: ${{ vars.OS_FLEET_APP_ID != '' }} + HAS_PRIVATE_KEY: ${{ secrets.OS_FLEET_PRIVATE_KEY != '' }} + run: | + if [ "$HAS_APP_ID" != 'true' ] || [ "$HAS_PRIVATE_KEY" != 'true' ]; then + echo "::error title=UI test timings refresh::The fleet App credential is not visible to this repository (OS_FLEET_APP_ID set: $HAS_APP_ID; OS_FLEET_PRIVATE_KEY set: $HAS_PRIVATE_KEY). Nothing was pushed." + exit 1 + fi + echo "repository=${GITHUB_REPOSITORY#*/}" >> "$GITHUB_OUTPUT" + + - name: Mint the fleet App token, narrowed to this repository + id: fleet + if: steps.generate.outputs.changed == 'true' && github.event_name != 'pull_request' + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ vars.OS_FLEET_APP_ID }} + private-key: ${{ secrets.OS_FLEET_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ steps.credential.outputs.repository }} + permission-contents: write # push the bot branch + permission-pull-requests: write # open the PR and add its label + + # On `pull_request` every outward act is printed, not run, and the local + # commit still happens, so the write step's own script is exercised up to + # the push. Same shape as shard-timings-refresh.yml's write step. + - name: Push the refresh branch and open the pull request (dry run on pull_request) + if: steps.generate.outputs.changed == 'true' + env: + DRY_RUN: ${{ github.event_name == 'pull_request' }} + GH_TOKEN: ${{ github.event_name == 'pull_request' && github.token || steps.fleet.outputs.token }} + RUNS: ${{ steps.download.outputs.runs }} + NEWEST: ${{ steps.download.outputs.newest }} + run: | + set -euo pipefail + case "$DRY_RUN" in + true) DRY_TAG='(dry run, not executed) ' ;; + false) DRY_TAG='' ;; + *) + echo "::error::DRY_RUN must be 'true' or 'false', got '$DRY_RUN'. Nothing was pushed." + exit 1 + ;; + esac + if [ "$GITHUB_EVENT_NAME" = 'pull_request' ] && [ "$DRY_RUN" != 'true' ]; then + echo "::error::DRY_RUN is '$DRY_RUN' on a pull_request run. A pull_request run of this lane never pushes a branch, opens a PR or writes a label. Nothing was pushed." + exit 1 + fi + : "${GH_TOKEN:?is not set; gh would run unauthenticated. Nothing was pushed.}" + outward() { + local stand_in='' + if [ "$1" = '--stand-in' ]; then + stand_in="$2" + shift 2 + fi + if [ "$DRY_RUN" = 'true' ]; then + { printf 'DRY RUN, not executed:'; printf ' %q' "$@"; printf '\n'; } >&2 + if [ -n "$stand_in" ]; then printf '%s\n' "$stand_in"; fi + return 0 + fi + "$@" + } + BRANCH="claude/ui-test-timings-refresh-$NEWEST" + outward gh auth setup-git + git config user.name 'objectstack-fleet[bot]' + git config user.email '332303061+objectstack-fleet[bot]@users.noreply.github.com' + git switch -c "$BRANCH" + git add ui/scripts/test-shard-timings.json + git commit \ + -m "chore(ci): refresh the Test UI per-file timings dataset" \ + -m "Regenerated by .github/workflows/ui-test-timings-refresh.yml from the test-file-timings reports of run(s) $RUNS, newest $NEWEST. Generated, never hand-edited." + outward git push origin "$BRANCH" + PR_URL=$(outward --stand-in "https://github.com/$GITHUB_REPOSITORY/pull/0" \ + gh pr create --base main --head "$BRANCH" \ + --title "chore(ci): refresh the Test UI per-file timings dataset" \ + --body-file "$RUNNER_TEMP/pr-body.md") + echo "${DRY_TAG}Opened $PR_URL" | tee -a "$GITHUB_STEP_SUMMARY" + PR_NUMBER=$(outward --stand-in 0 gh pr view "$PR_URL" --json number --jq .number) + outward gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/labels" \ + -f "labels[]=skip-changeset" > /dev/null + if outward --stand-in skip-changeset gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/labels" --jq '.[].name' \ + | grep -qxF 'skip-changeset'; then + echo "${DRY_TAG}skip-changeset confirmed on PR #$PR_NUMBER." + else + echo "::warning::skip-changeset did not survive the write on PR #$PR_NUMBER (a concurrent whole-set label PUT strips it). Re-applying once." + outward gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/labels" \ + -f "labels[]=skip-changeset" > /dev/null + outward gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/labels" --jq '.[].name' \ + | grep -qxF 'skip-changeset' \ + || echo "::error::skip-changeset is still absent from PR #$PR_NUMBER; the changeset gate will demand a changeset this PR legitimately has none of. Apply the label by hand." + fi + if [ "$DRY_RUN" = 'true' ]; then + { + echo "### UI test timings: dry run (no branch pushed, no PR opened, no label written)" + echo + echo "This is a \`pull_request\` run of the refresh lane itself. The selection, the" + echo "downloads, the regeneration and the write step's own script, up to its local" + echo "commit, all ran for real; only the push, the PR and the label write were skipped." + echo "The body below is what a scheduled run would have posted." + echo + echo "---" + echo + cat "$RUNNER_TEMP/pr-body.md" + } >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Say what happened when nothing changed + if: steps.generate.outputs.changed == 'false' + run: | + { + echo "### UI test timings: byte-identical, no PR" + echo + echo "The regenerated \`ui/scripts/test-shard-timings.json\` is byte-for-byte the committed" + echo "file, so there is nothing to open a pull request about." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/ui/scripts/__tests__/test-file-timings.test.ts b/ui/scripts/__tests__/test-file-timings.test.ts index 162342a2734..2d436145ea8 100644 --- a/ui/scripts/__tests__/test-file-timings.test.ts +++ b/ui/scripts/__tests__/test-file-timings.test.ts @@ -20,13 +20,16 @@ import FileTimingsReporter, { shardJobName, } from '../test-file-timings.mjs'; import { TIMINGS_SCHEMA, datasetWeights, pathKey } from '../test-shard-partition.mjs'; +import { parse as parseYaml } from 'yaml'; +import { shardTotal } from '../../../scripts/ci-shard-matrix.mjs'; +import { UI_JOBS, rootCi, rootWorkflow, rootWorkflowsDir } from './helpers/root-ci'; /** * The per-file timing pipeline (objectui#12208): the reporter that measures on * the full battery, the selector that picks which runs feed the dataset, and - * the generator that writes `scripts/test-shard-timings.json`. The pins that - * held objectui's `ci.yml` to the names and flags this file spells retired - * with it (see the end of this file). + * the generator that writes `scripts/test-shard-timings.json`. The last + * describe holds the monorepo root's `ci.yml` (`test-ui`) and its weekly + * refresh workflow to the names and flags this file spells. */ /** A TestModule-shaped object, as vitest 4 hands one to `onTestRunEnd`. */ @@ -181,14 +184,78 @@ describe('the selector', () => { }); }); -// `ci.yml spells what this file spells (objectui#12208)` retired with -// objectstack#25712. It held objectui ci.yml's `test` job to this file's names -// and flags: the `Test (shard N/M)` job name, the timing reporter beside the -// two it would displace, the per-shard `test-file-timings-N` upload, and the -// weekly refresh's `--shards`. That ci.yml runs nowhere under the monorepo, and -// the root `test-ui` job that replaced it carries none of the four on purpose: -// its comment defers the reporter and the upload to the refresh workflow, row -// 28 `test-shard-timings-refresh.yml` of the objectstack#25697 workflow roster -// (→ sub-issue 4, "when ported, restore Test UI's timings reporter and -// upload"), and row 7 `Test (shard n/16)` records the same wait. The port -// brings these pins back against `test-ui` (`helpers/root-ci.ts`). +/** The weekly refresh that drives `select` and `merge`, at the monorepo root. */ +const REFRESH_WORKFLOW = 'ui-test-timings-refresh.yml'; + +type Step = { name?: string; run?: string; uses?: string; if?: string; env?: Record; with?: Record; 'continue-on-error'?: boolean }; +type Job = { name?: string; env?: Record; steps: Step[] }; + +/** `${{ matrix.shard }}` rendered as shard `n`, the way the runner names one leg. */ +const asShard = (text: string, n: number) => text.replaceAll('${{ matrix.shard }}', String(n)); + +/** + * The root `ci.yml` and the refresh spell what this file spells + * (objectstack#29517, which brought back the pins objectui#12208 held against + * objectui's own `ci.yml` and which retired with objectstack#25712). Each pin + * reads the workflow GitHub runs, parsed, so prose in a comment cannot satisfy + * it; a renamed job or step throws instead of matching nothing. + */ +describe('the root ci.yml and the refresh spell what this file spells (objectstack#29517)', () => { + const ci = rootCi(); + const SHARDS = shardTotal(ci, UI_JOBS.test); + const job = (parseYaml(ci) as { jobs: Record }).jobs[UI_JOBS.test]; + const step = (prefix: string): Step => { + const found = job.steps.filter((s) => typeof s.name === 'string' && s.name.startsWith(prefix)); + if (found.length !== 1) throw new Error(`${UI_JOBS.test} has ${found.length} steps named ${JSON.stringify(prefix)}…`); + return found[0]; + }; + const runStep = step('Run the ui tests'); + const uploadStep = step("Publish this shard's per-file test timings"); + + it('names every Test UI shard the way the selector looks it up', () => { + expect(SHARDS).toBeGreaterThan(1); + for (let n = 1; n <= SHARDS; n += 1) expect(asShard(String(job.name), n)).toBe(shardJobName(n, SHARDS)); + }); + + it('passes the reporter beside the two it would displace on the whole suite, and on the affected speed not at all', () => { + const lines = String(runStep.run).split('\n').map((line) => line.trim()); + const whole = lines.filter((line) => line.startsWith('pnpm --dir ui run test ')); + const affected = lines.filter((line) => line.startsWith('pnpm --dir ui exec vitest run ')); + expect(whole).toHaveLength(1); + expect(affected).toHaveLength(1); + const argv = whole[0].split(/\s+/); + for (const reporter of ['--reporter=default', '--reporter=github-actions', '--reporter=./scripts/test-file-timings.mjs']) { + expect(argv).toContain(reporter); + } + expect(affected[0]).not.toContain('test-file-timings'); + expect(runStep.env?.[OUT_ENV]).toBeTruthy(); + }); + + it("uploads each shard's report, from where the reporter writes it, as the artifact the selector requires, never failing the shard", () => { + const out = String(runStep.env?.[OUT_ENV]); + expect(uploadStep.uses).toMatch(/^actions\/upload-artifact@/); + expect(String(uploadStep.with?.path).replace(/\/$/, '')).toBe(path.posix.dirname(out)); + for (let n = 1; n <= SHARDS; n += 1) expect(asShard(String(uploadStep.with?.name), n)).toBe(artifactName(n)); + // Report-only, by ruling: the upload cannot red a shard, and a missing report is not an error. + expect(uploadStep['continue-on-error']).toBe(true); + expect(uploadStep.with?.['if-no-files-found']).not.toBe('error'); + // The full battery's events only, the deny-list of the two per-PR events. + expect(uploadStep.if).toContain("github.event_name != 'pull_request'"); + expect(uploadStep.if).toContain("github.event_name != 'merge_group'"); + }); + + it('runs the refresh with the matrix width, downloads the artifacts by their name, and is the workflow the dataset names', () => { + expect(fs.existsSync(path.join(rootWorkflowsDir, REFRESH_WORKFLOW))).toBe(true); + const refresh = parseYaml(rootWorkflow(REFRESH_WORKFLOW)) as { jobs: Record }; + const jobs = Object.values(refresh.jobs); + expect(jobs).toHaveLength(1); + expect(jobs[0].env?.UI_TEST_SHARDS).toBe(String(SHARDS)); + const runs = jobs[0].steps.map((s) => s.run ?? '').join('\n'); + expect(runs).toContain('node ui/scripts/test-file-timings.mjs select --shards "$UI_TEST_SHARDS"'); + expect(runs).toContain('node ui/scripts/test-file-timings.mjs merge'); + expect(runs).toContain('--out ui/scripts/test-shard-timings.json'); + expect(runs).toContain(`--pattern '${artifactName(1).replace(/1$/, '*')}'`); + const dataset = buildDataset({ runs: [{ meta: { id: 1 }, durations: new Map([['a.test.ts', 10]]) }], measuredAt: 'x' }); + expect(dataset.generated_by).toContain(`.github/workflows/${REFRESH_WORKFLOW}`); + }); +}); diff --git a/ui/scripts/test-file-timings.mjs b/ui/scripts/test-file-timings.mjs index f08b7d0d1ec..35e966f8985 100644 --- a/ui/scripts/test-file-timings.mjs +++ b/ui/scripts/test-file-timings.mjs @@ -9,31 +9,33 @@ * * ONE file, three parts, so the report format has exactly one spelling: * - * THE WRITER, this module's default export: a vitest reporter. `ci.yml`'s - * `test` job adds it beside the reporters vitest would have picked on its - * own, on the full battery's events, and uploads what it writes as the - * `test-file-timings-N` artifact of shard N: + * THE WRITER, this module's default export: a vitest reporter. The + * `test-ui` job of the monorepo root's `.github/workflows/ci.yml` adds it + * beside the reporters vitest would have picked on its own whenever it runs + * the whole suite, and on the full battery's events uploads what it writes + * as the `test-file-timings-N` artifact of shard N: * - * pnpm test --shard=N/M --reporter=default --reporter=github-actions \ - * --reporter=./scripts/test-file-timings.mjs + * pnpm --dir ui run test --shard=N/M --reporter=default \ + * --reporter=github-actions --reporter=./scripts/test-file-timings.mjs * -> $OBJECTUI_TEST_TIMINGS_OUT (no variable, no file: one stderr line) * * THE SELECTOR, `select`: the newest completed full-battery `CI` runs on - * `main` whose M test shards ALL reported success and whose M timing - * artifacts are all still there. + * `main` whose M `Test UI (N/M)` shards ALL reported success and whose M + * timing artifacts are all still there. * - * node scripts/test-file-timings.mjs select --shards 16 --limit 6 > runs.json + * node ui/scripts/test-file-timings.mjs select --shards 16 --limit 6 > runs.json * * THE GENERATOR, `merge`: one directory per run (the run id is the * directory's name), every report under it read; per run a file's * durations are summed, across runs the MEDIAN is kept, and the dataset is * keyed by `pathKey()` of each file, never by its path. * - * node scripts/test-file-timings.mjs merge --runs-file runs.json \ - * --out scripts/test-shard-timings.json [--summary body.md] DIR... + * node ui/scripts/test-file-timings.mjs merge --runs-file runs.json \ + * --out ui/scripts/test-shard-timings.json [--summary body.md] DIR... * - * `.github/workflows/test-shard-timings-refresh.yml` drives the last two on a - * timer and opens the pull request; ⛔ nothing writes the dataset by hand. + * The root's `.github/workflows/ui-test-timings-refresh.yml` drives the last + * two on a timer and opens the pull request; ⛔ nothing writes the dataset by + * hand. * * ## Why not vitest's built-in `json` reporter * @@ -79,10 +81,10 @@ export const REPORT_SCHEMA = 'objectui.test-file-timings/1'; export const OUT_ENV = 'OBJECTUI_TEST_TIMINGS_OUT'; /** The dataset rounds every file's median to this many ms, so noise below it does not churn the file. */ export const ROUND_MS = 10; -/** The artifact name of shard N's report, as `ci.yml` uploads it. */ +/** The artifact name of shard N's report, as the root `ci.yml`'s `test-ui` job uploads it. */ export const artifactName = (/** @type {number} */ n) => `test-file-timings-${n}`; -/** The check-run name of shard N of M, as `ci.yml`'s `test` job spells it. */ -export const shardJobName = (/** @type {number} */ n, /** @type {number} */ m) => `Test (shard ${n}/${m})`; +/** The check-run name of shard N of M, as the root `ci.yml`'s `test-ui` job spells it. */ +export const shardJobName = (/** @type {number} */ n, /** @type {number} */ m) => `Test UI (${n}/${m})`; export const EXIT_OK = 0; export const EXIT_USAGE = 2; @@ -277,7 +279,7 @@ export function buildDataset({ runs, measuredAt }) { generated_by: 'scripts/test-file-timings.mjs merge, from the per-file timing reports of the runs under measured.runs; ' + 'each key is pathKey() of a test file (scripts/test-shard-partition.mjs says why it is not the path). ' + - 'Never hand-edited: .github/workflows/test-shard-timings-refresh.yml regenerates it weekly from main (objectui#12208).', + 'Never hand-edited: .github/workflows/ui-test-timings-refresh.yml at the monorepo root regenerates it weekly from main (objectui#12208).', // Sorted by run id, so the order the directories were named in cannot move a byte. measured: { at: measuredAt, @@ -392,7 +394,7 @@ export async function selectRuns({ let examined = 0; for (const run of listing?.workflow_runs ?? []) { if (chosen.length >= limit || examined >= limit * 5) break; - // The full battery: `push` and the hourly `schedule` (objectui#12214). + // The full battery: a `push` that starts Test UI, and the 4-hourly `schedule`. if (run.event !== 'push' && run.event !== 'schedule') continue; examined += 1; const jobs = (await apiGet(`/repos/${repository}/actions/runs/${run.id}/jobs?per_page=100&filter=latest`, io))?.jobs ?? []; From 736df2ec66b6cf4f88679a05bdeda4e81a9f1d36 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 18:18:47 +0000 Subject: [PATCH 2/4] ci(ui): spell the timings reporter from the workspace root and declare the refresh's no-check-families check-self-test-wired keys a `./scripts/` path in a workflow to the root scripts/ directory, so the ui-relative reporter spelling minted a key with no file behind it. The refresh workflow is path-filtered and runs no named check, so it carries the dispatch-gates no-check-families declaration. Claude-Session: https://claude.ai/code/session_01JQwxbXf3xRiPQrLhrgsFzw Co-Authored-By: Claude --- .github/workflows/ci.yml | 5 ++++- .github/workflows/ui-test-timings-refresh.yml | 6 ++++++ ui/scripts/__tests__/test-file-timings.test.ts | 3 ++- ui/scripts/test-file-timings.mjs | 3 ++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85d2f5fde0e..9a0caec7dc0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3898,6 +3898,9 @@ jobs: # a slice of the suite is not a measurement of it. # ui/scripts/__tests__/test-file-timings.test.ts holds this step, the # upload and the refresh's `--shards` to the names that script spells. + # The reporter is spelled from the workspace root, never `./scripts/…` + # relative to ui/: the root's workflow readers (check-self-test-wired) + # key a `./scripts/` path to the ROOT `scripts/`, where it is not. - name: Run the ui tests (shard ${{ matrix.shard }}/16) env: MODE: ${{ steps.select.outputs.mode }} @@ -3908,7 +3911,7 @@ jobs: echo "Affected speed: ${#filters[@]} vitest filter(s), shard ${{ matrix.shard }} of 16." pnpm --dir ui exec vitest run --shard=${{ matrix.shard }}/16 --reporter=default --reporter=github-actions "${filters[@]}" else - pnpm --dir ui run test --shard=${{ matrix.shard }}/16 --reporter=default --reporter=github-actions --reporter=./scripts/test-file-timings.mjs + pnpm --dir ui run test --shard=${{ matrix.shard }}/16 --reporter=default --reporter=github-actions --reporter="${GITHUB_WORKSPACE}/ui/scripts/test-file-timings.mjs" fi # Diagnostics only, so it can never red a shard: `continue-on-error`, and diff --git a/.github/workflows/ui-test-timings-refresh.yml b/.github/workflows/ui-test-timings-refresh.yml index cc7aaa0b267..15cf6fb6aa0 100644 --- a/.github/workflows/ui-test-timings-refresh.yml +++ b/.github/workflows/ui-test-timings-refresh.yml @@ -57,6 +57,12 @@ # merges. With no eligible run on `main` (the state right after the reporter # first lands, until a full-battery run has uploaded sixteen reports) it reads # NOT MEASURED and stays green: nothing was regenerated and nothing failed. +# +# Its steps are a generator's select and merge, not a named local check: the +# scripts it loads are pinned by ui/scripts/__tests__/test-file-timings.test.ts, +# which Test UI runs on any change to them. +# dispatch-gates: no-check-families -- a weekly regeneration (select, download, merge, open a PR), not a verification; the scripts it loads are pinned by Test UI's test-file-timings.test.ts + name: UI Test Timings Refresh on: diff --git a/ui/scripts/__tests__/test-file-timings.test.ts b/ui/scripts/__tests__/test-file-timings.test.ts index 2d436145ea8..452a4afcab9 100644 --- a/ui/scripts/__tests__/test-file-timings.test.ts +++ b/ui/scripts/__tests__/test-file-timings.test.ts @@ -224,7 +224,8 @@ describe('the root ci.yml and the refresh spell what this file spells (objectsta expect(whole).toHaveLength(1); expect(affected).toHaveLength(1); const argv = whole[0].split(/\s+/); - for (const reporter of ['--reporter=default', '--reporter=github-actions', '--reporter=./scripts/test-file-timings.mjs']) { + // From the workspace root: a `./scripts/` spelling is read as the ROOT `scripts/` by the root's workflow readers. + for (const reporter of ['--reporter=default', '--reporter=github-actions', '--reporter="${GITHUB_WORKSPACE}/ui/scripts/test-file-timings.mjs"']) { expect(argv).toContain(reporter); } expect(affected[0]).not.toContain('test-file-timings'); diff --git a/ui/scripts/test-file-timings.mjs b/ui/scripts/test-file-timings.mjs index 35e966f8985..b8f8aa76095 100644 --- a/ui/scripts/test-file-timings.mjs +++ b/ui/scripts/test-file-timings.mjs @@ -16,7 +16,8 @@ * as the `test-file-timings-N` artifact of shard N: * * pnpm --dir ui run test --shard=N/M --reporter=default \ - * --reporter=github-actions --reporter=./scripts/test-file-timings.mjs + * --reporter=github-actions \ + * --reporter="${GITHUB_WORKSPACE}/ui/scripts/test-file-timings.mjs" * -> $OBJECTUI_TEST_TIMINGS_OUT (no variable, no file: one stderr line) * * THE SELECTOR, `select`: the newest completed full-battery `CI` runs on From f23342d25417d7c3447e6fba129e791abc182684 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 18:46:36 +0000 Subject: [PATCH 3/4] test(pm): census the refresh workflow's no-check-families declaration dispatch-gates' self-test names every live population marker; the new ui-test-timings-refresh.yml declaration is the tenth row. Claude-Session: https://claude.ai/code/session_01JQwxbXf3xRiPQrLhrgsFzw Co-Authored-By: Claude --- scripts/pm/dispatch-gates.self-test.mjs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/pm/dispatch-gates.self-test.mjs b/scripts/pm/dispatch-gates.self-test.mjs index 4ed0a5e9c21..123c013f6c5 100644 --- a/scripts/pm/dispatch-gates.self-test.mjs +++ b/scripts/pm/dispatch-gates.self-test.mjs @@ -7544,7 +7544,7 @@ export function selfTest({ tier = 'full' } = {}) { } const censusRows = liveMarkerCensus.map((r) => `${r.file}:${r.line} ${r.key}`).sort(); t( - `the live tree carries the nine declarations measured for this census, and no others (${censusRows.join(' · ') || 'none'})`, + `the live tree carries the ten declarations measured for this census, and no others (${censusRows.join(' · ') || 'none'})`, censusRows.join(' · ') === [ // Seventh row, added with the declaration it names: `checklist-status.yml` // is paths-filtered (its `pull_request:` trigger is filtered to itself) and @@ -7557,6 +7557,10 @@ export function selfTest({ tier = 'full' } = {}) { '.github/workflows/merged-branch-reaper.yml:212 no-check-families', '.github/workflows/os-create-smoke.yml:48 no-check-families', '.github/workflows/scaffold-e2e.yml:23 no-check-families', + // Tenth row, added with the declaration it names: `ui-test-timings-refresh.yml` + // is paths-filtered (its own file and the three ui scripts it loads) and + // runs a generator's select and merge, so it discovers no `check:` family. + '.github/workflows/ui-test-timings-refresh.yml:64 no-check-families', // Eighth row, added with the declaration it names (#20278): the checker's // bare diff-scoped run declares the two step-`env:` values it does not need. 'scripts/check-issue-citations.mjs:204 local-env', @@ -7577,7 +7581,7 @@ export function selfTest({ tier = 'full' } = {}) { const censusCut = liveMarkerCensus.filter((r) => !r.whole).map((r) => `${r.file}:${r.line} ${r.key}`); t( `every live reason on those markers ENDS on its own marker line (cut: ${censusCut.join(', ') || 'none'})`, - censusCut.length === 0 && liveMarkerCensus.length === 9, + censusCut.length === 0 && liveMarkerCensus.length === 10, ); t( 'and every one of them carries a non-empty reason — whole is not the same claim as present, and both are owed', From 5f0c784fe69e44879a3fe73e2384e64c4f6a66f4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 20:34:06 +0000 Subject: [PATCH 4/4] ci(ui): prove the timings reporter loads by absolute path, overwrite on re-run, main-only refresh dispatch - test-file-timings.test.ts spawns the vitest CLI with the reporter by absolute path over one fixture test and asserts exit 0 and a schema-tagged report (a PR runs Test UI at the affected speed, so nothing else proves it). - The per-shard timings upload sets overwrite: true, so a re-run shard replaces its report; its comment now says retention is over-provisioned against the selector's window rather than measuring it. - ui-test-timings-refresh.yml refuses a workflow_dispatch on any ref but main before anything else runs. Claude-Session: https://claude.ai/code/session_01JQwxbXf3xRiPQrLhrgsFzw Co-Authored-By: Claude --- .github/workflows/ci.yml | 11 +++-- .github/workflows/ui-test-timings-refresh.yml | 11 +++++ .../__tests__/test-file-timings.test.ts | 42 +++++++++++++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a0caec7dc0..f9880d07562 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3918,10 +3918,12 @@ jobs: # a missing report is a warning, not an error (the refresh's selector # then skips this run by name). Uploaded on the full battery's events # only, the deny-list of THE TWO SPEEDS above `jobs:`; a pull request that - # failed open to the whole suite writes a report and uploads nothing. The - # refresh reads the newest fifty completed runs of this workflow on - # `main`, so a push run's report needs two days and a scheduled run's a - # week, which also covers a quiet weekend of scheduled runs alone. + # failed open to the whole suite writes a report and uploads nothing. + # `overwrite: true` so a re-run shard replaces its report, as the + # shard-attestation uploads do. The refresh reads only the newest + # completed `push` / `schedule` runs of this workflow on `main`, as many + # as its selector's own cap examines, so the retention below is + # deliberately more than that window needs; it is not a measurement of it. - name: Publish this shard's per-file test timings if: ${{ always() && github.event_name != 'pull_request' && github.event_name != 'merge_group' }} continue-on-error: true @@ -3931,6 +3933,7 @@ jobs: path: ${{ runner.temp }}/test-file-timings/ if-no-files-found: warn retention-days: ${{ github.event_name == 'schedule' && 7 || 2 }} + overwrite: true test-ui-gate: # Stable required-check name for the sharded Test UI matrix (#26460, ruling diff --git a/.github/workflows/ui-test-timings-refresh.yml b/.github/workflows/ui-test-timings-refresh.yml index 15cf6fb6aa0..3c8add7eaca 100644 --- a/.github/workflows/ui-test-timings-refresh.yml +++ b/.github/workflows/ui-test-timings-refresh.yml @@ -101,6 +101,17 @@ jobs: UI_TEST_SHARDS: '16' RUN_LIMIT: '6' steps: + # A dispatch from another branch would regenerate from that branch's + # scripts and open a PR against `main` from them; only `main` runs the + # write path. Refused before anything else runs. + - name: Refuse a dispatch on any ref but main + if: github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/main' + env: + REF: ${{ github.ref }} + run: | + echo "::error title=UI test timings refresh::workflow_dispatch runs only on refs/heads/main, not '$REF'. Nothing was regenerated or pushed." + exit 1 + - name: Checkout repository uses: actions/checkout@v7 with: diff --git a/ui/scripts/__tests__/test-file-timings.test.ts b/ui/scripts/__tests__/test-file-timings.test.ts index 452a4afcab9..b01990985e7 100644 --- a/ui/scripts/__tests__/test-file-timings.test.ts +++ b/ui/scripts/__tests__/test-file-timings.test.ts @@ -1,7 +1,10 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { spawnSync } from 'node:child_process'; import fs from 'node:fs'; +import { createRequire } from 'node:module'; import os from 'node:os'; import path from 'node:path'; +import { fileURLToPath } from 'node:url'; // Plain-JS CI helpers; their types are INFERRED from the .mjs source by // `tsconfig.scripts.json` (`allowJs`), so no `@ts-expect-error` here. @@ -23,6 +26,7 @@ import { TIMINGS_SCHEMA, datasetWeights, pathKey } from '../test-shard-partition import { parse as parseYaml } from 'yaml'; import { shardTotal } from '../../../scripts/ci-shard-matrix.mjs'; import { UI_JOBS, rootCi, rootWorkflow, rootWorkflowsDir } from './helpers/root-ci'; +import { childVitestEnv } from './helpers/child-vitest-env'; /** * The per-file timing pipeline (objectui#12208): the reporter that measures on @@ -94,6 +98,44 @@ describe('the writer', () => { }); }); +/** + * The writer as the real vitest CLI loads it: by ABSOLUTE path, the spelling + * the root `ci.yml` passes. A pull request runs Test UI at the affected speed, + * which passes no reporter, so nothing else on a PR proves the load; a load + * failure would red every full-battery Test UI shard on `main`. One fixture + * test in a scratch root with its own empty config (never this tree's, whose + * projects and invocation guard are not what is under test). + */ +describe('the writer, loaded by the vitest CLI from an absolute path', () => { + const reporterPath = fileURLToPath(new URL('../test-file-timings.mjs', import.meta.url)); + const vitestCli = (() => { + const require = createRequire(import.meta.url); + const pkgPath = require.resolve('vitest/package.json'); + const bin = (JSON.parse(fs.readFileSync(pkgPath, 'utf8')) as { bin: { vitest: string } }).bin.vitest; + return path.resolve(path.dirname(pkgPath), bin); + })(); + let dir: string; + beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'file-timings-load-')); + }); + afterEach(() => fs.rmSync(dir, { recursive: true, force: true })); + + it('exits 0 and writes a schema-tagged report naming the fixture file', () => { + fs.writeFileSync(path.join(dir, 'vitest.config.mjs'), 'export default {};\n'); + fs.writeFileSync(path.join(dir, 'smoke.test.mjs'), "test('runs', () => { expect(1).toBe(1); });\n"); + const out = path.join(dir, 'report', 'shard-1.json'); + const child = spawnSync( + process.execPath, + [vitestCli, 'run', '--root', dir, '--config', path.join(dir, 'vitest.config.mjs'), '--globals', '--reporter=default', `--reporter=${reporterPath}`], + { cwd: dir, encoding: 'utf8', env: childVitestEnv({ [OUT_ENV]: out }), timeout: 120_000 }, + ); + expect(child.status, `${child.stdout ?? ''}${child.stderr ?? ''}`).toBe(0); + const report = JSON.parse(fs.readFileSync(out, 'utf8')); + expect(report.schema).toBe(REPORT_SCHEMA); + expect(report.files.map((f: { file: string }) => f.file)).toEqual(['smoke.test.mjs']); + }, 180_000); +}); + describe('the generator', () => { let dir: string; beforeEach(() => {