From eeab330f8ee33ff25d1bc7132877ab8d289db0ea Mon Sep 17 00:00:00 2001 From: mintaka Date: Fri, 16 Oct 2026 00:16:42 -0400 Subject: [PATCH 1/2] feat(ci): route a red full sweep to a ci-backstop issue (RIG-5491) A backstop job after rollup on push and schedule opens, comments on or closes the one ci-backstop issue. A run older than the newest finished sweep does nothing, so a stale red run never reopens a closed issue. Co-authored-by: Matt Wilkinson --- .github/workflows/ci.yml | 39 +++++ .moon/workspace.yml | 2 + bun.lock | 10 ++ tools/ci-backstop/README.md | 16 ++ tools/ci-backstop/biome.json | 5 + tools/ci-backstop/core.test.ts | 92 +++++++++++ tools/ci-backstop/core.ts | 40 +++++ tools/ci-backstop/index.ts | 281 ++++++++++++++++++++++++++++++++ tools/ci-backstop/moon.yml | 19 +++ tools/ci-backstop/package.json | 14 ++ tools/ci-backstop/tsconfig.json | 11 ++ 11 files changed, 529 insertions(+) create mode 100644 tools/ci-backstop/README.md create mode 100644 tools/ci-backstop/biome.json create mode 100644 tools/ci-backstop/core.test.ts create mode 100644 tools/ci-backstop/core.ts create mode 100755 tools/ci-backstop/index.ts create mode 100644 tools/ci-backstop/moon.yml create mode 100644 tools/ci-backstop/package.json create mode 100644 tools/ci-backstop/tsconfig.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a17ff8d95..3a02e1fa5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2888,6 +2888,45 @@ jobs: fi echo "all required work jobs succeeded (or correctly skipped per their setup flag)" + + backstop: + name: backstop + needs: rollup + if: ${{ !cancelled() && contains(fromJSON('["push","schedule"]'), github.event_name) }} + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + concurrency: + group: ci-backstop + cancel-in-progress: false + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31 + with: + extra_nix_config: | + experimental-features = nix-command flakes + extra-substituters = https://devenv.cachix.org https://cachix.cachix.org + extra-trusted-public-keys = devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw= cachix.cachix.org-1:eWNHQldwUO7G2VkjpnjDbWwy4KQ/HNxht7H4SSoMckM= + - name: Put the pinned bun toolchain on PATH + # bun comes from the nix-pinned toolchain, never setup-bun, so it is the + # same derivation as the dev shell's. `jq -r` renders a missing `.store` + # as the literal `null`, so guard for it explicitly. + run: | + set -euo pipefail + store=$(nix eval --json -f tools/toolchain/gate-tools.nix langs.bun | jq -r '.store') + if [ -z "$store" ] || [ "$store" = null ]; then + echo "::error::gate-tools.nix langs.bun produced no store path" >&2 + exit 1 + fi + nix build --no-link "$store" + echo "$store/bin" >>"$GITHUB_PATH" + - name: Route failed full sweep + env: + ROLLUP_RESULT: ${{ needs.rollup.result }} + GH_TOKEN: ${{ github.token }} + run: bun tools/ci-backstop/index.ts + regen-forge-fixtures: name: regen-fixtures runs-on: ubuntu-latest diff --git a/.moon/workspace.yml b/.moon/workspace.yml index 6cb17628c..fed160ee2 100644 --- a/.moon/workspace.yml +++ b/.moon/workspace.yml @@ -216,6 +216,8 @@ projects: # The Bun advisory check is a dedicated, always-run workflow job; this project # keeps its typecheck and tests in the affected Moon matrix. bun-audit-gate: 'tools/bun-audit-gate' + # The full-sweep failure backstop: a pure issue decision plus its tests. + ci-backstop: 'tools/ci-backstop' # File hashing uses moon's in-process native walker (the v2.5+ default), NOT the # git shell-out. This is load-bearing: the git hasher (`git ls-files` then `git diff --git a/bun.lock b/bun.lock index f6c35e308..33eefb3bd 100644 --- a/bun.lock +++ b/bun.lock @@ -147,6 +147,16 @@ "typescript": "catalog:", }, }, + "tools/ci-backstop": { + "name": "@compass/ci-backstop", + "bin": { + "ci-backstop": "./index.ts", + }, + "devDependencies": { + "@types/bun": "catalog:", + "typescript": "catalog:", + }, + }, "tools/compass-app-dev": { "name": "@compass/compass-app-dev", "bin": { diff --git a/tools/ci-backstop/README.md b/tools/ci-backstop/README.md new file mode 100644 index 000000000..b4d7d7b49 --- /dev/null +++ b/tools/ci-backstop/README.md @@ -0,0 +1,16 @@ +# ci-backstop + +Routes failed full CI sweeps to one `ci-backstop` issue. A green full sweep +closes that issue. `core.ts` holds the pure action decision; `index.ts` reads +GitHub Actions data and applies it through `gh`. + +## Run + +```sh +bun tools/ci-backstop/index.ts [--dry-run] +moon run ci-backstop:ci +``` + +The `backstop` job in `.github/workflows/ci.yml` runs on push and schedule +results after `rollup`. `GH_TOKEN` must have issue write access. `--dry-run` +prints the action and does not create the label or change an issue. diff --git a/tools/ci-backstop/biome.json b/tools/ci-backstop/biome.json new file mode 100644 index 000000000..9cad0143f --- /dev/null +++ b/tools/ci-backstop/biome.json @@ -0,0 +1,5 @@ +{ + "root": false, + "extends": "//", + "linter": { "rules": { "suspicious": { "noConsole": "off" } } } +} diff --git a/tools/ci-backstop/core.test.ts b/tools/ci-backstop/core.test.ts new file mode 100644 index 000000000..df4a8e063 --- /dev/null +++ b/tools/ci-backstop/core.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, test } from "bun:test"; +import { backstopAction } from "./core.ts"; + +const runUrl = "https://github.com/RigelBuild/compass/actions/runs/123"; +const failedJobs = ["moon (bun)", "pgtest"]; + +const cases = [ + { + name: "failure without an issue opens one", + result: "failure", + openIssue: null, + newerSweep: false, + kind: "open", + }, + { + name: "failure with an issue comments", + result: "failure", + openIssue: 42, + newerSweep: false, + kind: "comment", + }, + { + name: "success without an issue does nothing", + result: "success", + openIssue: null, + newerSweep: false, + kind: "none", + }, + { + name: "success with an issue closes it", + result: "success", + openIssue: 42, + newerSweep: false, + kind: "close", + }, + { + name: "newer sweep supersedes failure without an issue", + result: "failure", + openIssue: null, + newerSweep: true, + kind: "none", + }, + { + name: "newer sweep supersedes failure with an issue", + result: "failure", + openIssue: 42, + newerSweep: true, + kind: "none", + }, + { + name: "newer sweep supersedes success without an issue", + result: "success", + openIssue: null, + newerSweep: true, + kind: "none", + }, + { + name: "newer sweep supersedes success with an issue", + result: "success", + openIssue: 42, + newerSweep: true, + kind: "none", + }, +] as const; + +describe("backstopAction", () => { + for (const scenario of cases) { + test(scenario.name, () => { + const action = backstopAction({ + result: scenario.result, + event: "push", + runUrl, + failedJobs, + openIssue: scenario.openIssue, + runStartedAt: "2026-10-15T10:00:00Z", + newestSweepStartedAt: scenario.newerSweep + ? "2026-10-15T10:00:01Z" + : "2026-10-15T10:00:00Z", + }); + + expect(action.kind).toBe(scenario.kind); + if (action.kind !== "none") { + expect(action.body).toContain(runUrl); + expect(action.body).toContain("moon (bun), pgtest"); + expect(action.body.split("\n")).toHaveLength(2); + } + if (action.kind === "open") { + expect(action.title).toBe("CI push full sweep failed"); + } + }); + } +}); diff --git a/tools/ci-backstop/core.ts b/tools/ci-backstop/core.ts new file mode 100644 index 000000000..dfe4ff337 --- /dev/null +++ b/tools/ci-backstop/core.ts @@ -0,0 +1,40 @@ +export type BackstopAction = + | { kind: "open"; title: string; body: string } + | { kind: "comment" | "close"; issue: number; body: string } + | { kind: "none" }; + +export function backstopAction(input: { + result: "success" | "failure"; + event: "push" | "schedule"; + runUrl: string; + failedJobs: readonly string[]; + openIssue: number | null; + runStartedAt: string; + newestSweepStartedAt: string; +}): BackstopAction { + if (Date.parse(input.newestSweepStartedAt) > Date.parse(input.runStartedAt)) { + return { kind: "none" }; + } + + const failedJobs = + input.failedJobs.length > 0 ? input.failedJobs.join(", ") : "none"; + const body = [ + `CI ${input.event} full sweep: ${input.result}. Run: ${input.runUrl}`, + `Failed jobs: ${failedJobs}`, + ].join("\n"); + + if (input.result === "failure") { + if (input.openIssue !== null) { + return { kind: "comment", issue: input.openIssue, body }; + } + return { + kind: "open", + title: `CI ${input.event} full sweep failed`, + body, + }; + } + + return input.openIssue === null + ? { kind: "none" } + : { kind: "close", issue: input.openIssue, body }; +} diff --git a/tools/ci-backstop/index.ts b/tools/ci-backstop/index.ts new file mode 100755 index 000000000..7b038a4c9 --- /dev/null +++ b/tools/ci-backstop/index.ts @@ -0,0 +1,281 @@ +#!/usr/bin/env bun +import { backstopAction } from "./core.ts"; + +const LABEL = "ci-backstop"; + +type RunEnvironment = { + result: "success" | "failure"; + event: "push" | "schedule"; + runId: string; + repository: string; + serverUrl: string; +}; + +function requiredEnv(name: string): string { + const value = process.env[name]; + if (value === undefined || value.length === 0) { + throw new Error(`${name} is required`); + } + return value; +} + +function readEnvironment(): RunEnvironment { + const result = requiredEnv("ROLLUP_RESULT"); + if (result !== "success" && result !== "failure") { + throw new Error(`unexpected ROLLUP_RESULT: ${result}`); + } + + const event = requiredEnv("GITHUB_EVENT_NAME"); + if (event !== "push" && event !== "schedule") { + throw new Error(`unexpected GITHUB_EVENT_NAME: ${event}`); + } + + const runId = requiredEnv("GITHUB_RUN_ID"); + if (!/^\d+$/.test(runId)) { + throw new Error(`GITHUB_RUN_ID must be a positive integer: ${runId}`); + } + + const repository = requiredEnv("GITHUB_REPOSITORY"); + if (!/^[^/]+\/[^/]+$/.test(repository)) { + throw new Error( + `GITHUB_REPOSITORY must be owner/repository: ${repository}`, + ); + } + + const serverUrl = requiredEnv("GITHUB_SERVER_URL").replace(/\/$/, ""); + const parsedServerUrl = new URL(serverUrl); + if ( + parsedServerUrl.protocol !== "https:" && + parsedServerUrl.protocol !== "http:" + ) { + throw new Error("GITHUB_SERVER_URL must use HTTP or HTTPS"); + } + return { + result, + event, + runId, + repository, + serverUrl, + }; +} + +async function gh(args: readonly string[]): Promise { + const child = Bun.spawn(["gh", ...args], { + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + if (exitCode !== 0) { + throw new Error( + `gh ${args.join(" ")} failed with exit code ${exitCode}: ${stderr.trim()}`, + ); + } + return stdout.trim(); +} + +function parseJson(value: string, command: string): unknown { + try { + return JSON.parse(value); + } catch (error) { + throw new Error( + `gh ${command} returned invalid JSON: ${error instanceof Error ? error.message : String(error)}`, + ); + } +} + +function record(value: unknown): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("gh returned an unexpected JSON object"); + } + return Object.fromEntries(Object.entries(value)); +} + +function runStartedAt(value: unknown, source: string): string { + const data = record(value); + const startedAt = data.run_started_at; + if (typeof startedAt !== "string" || Number.isNaN(Date.parse(startedAt))) { + throw new Error(`${source} has no valid run_started_at`); + } + return startedAt; +} + +function failedJobNames(value: unknown): string[] { + const response = record(value); + if (!Array.isArray(response.jobs)) { + throw new Error("gh jobs response has no jobs array"); + } + return response.jobs.flatMap((job) => { + const data = record(job); + if (data.conclusion !== "failure") return []; + if (typeof data.name !== "string") { + throw new Error("gh returned a failed job without a name"); + } + return [data.name]; + }); +} + +function openIssueNumber(value: unknown): number | null { + if (!Array.isArray(value)) { + throw new Error("gh issue list response is not an array"); + } + if (value.length > 1) { + throw new Error(`more than one open issue has the ${LABEL} label`); + } + const issue = value[0]; + if (issue === undefined) return null; + const data = record(issue); + if (typeof data.number !== "number" || !Number.isSafeInteger(data.number)) { + throw new Error("gh returned an issue without a valid number"); + } + return data.number; +} + +async function startedAtForLatestCompletedSweep( + repository: string, + event: "push" | "schedule", +): Promise { + const endpoint = `repos/${repository}/actions/workflows/ci.yml/runs?event=${event}&status=completed&per_page=1`; + const response = parseJson( + await gh([ + "api", + "--jq", + ".workflow_runs | map({id, run_started_at, status, event})", + endpoint, + ]), + `api ${endpoint}`, + ); + if (!Array.isArray(response)) { + throw new Error(`gh api ${endpoint} returned no workflow_runs array`); + } + const latestRun = response[0]; + if (latestRun === undefined) return null; + const runData = record(latestRun); + if ( + runData.status !== "completed" || + runData.event !== event || + typeof runData.id !== "number" + ) { + throw new Error(`gh api ${endpoint} returned an invalid completed run`); + } + return runStartedAt(runData, "completed workflow run"); +} + +async function main(): Promise { + const dryRun = process.argv.slice(2); + if (dryRun.length > 1 || (dryRun.length === 1 && dryRun[0] !== "--dry-run")) { + throw new Error("usage: bun tools/ci-backstop/index.ts [--dry-run]"); + } + const environment = readEnvironment(); + const { repository, runId } = environment; + + const runEndpoint = `repos/${repository}/actions/runs/${runId}`; + const [runResponse, jobsResponse] = await Promise.all([ + gh(["api", runEndpoint]), + gh(["api", `${runEndpoint}/jobs?per_page=100`]), + ]); + const currentRunStartedAt = runStartedAt( + parseJson(runResponse, `api ${runEndpoint}`), + "current workflow run", + ); + const failedJobs = failedJobNames( + parseJson(jobsResponse, `api ${runEndpoint}/jobs`), + ); + + const [latestPush, latestSchedule] = await Promise.all([ + startedAtForLatestCompletedSweep(repository, "push"), + startedAtForLatestCompletedSweep(repository, "schedule"), + ]); + const newestSweepStartedAt = + [latestPush, latestSchedule] + .filter((startedAt): startedAt is string => startedAt !== null) + .sort((left, right) => Date.parse(left) - Date.parse(right)) + .at(-1) ?? currentRunStartedAt; + + if (dryRun[0] !== "--dry-run") { + await gh(["label", "create", LABEL, "--force", "--repo", repository]); + } + const issueResponse = await gh([ + "issue", + "list", + "--repo", + repository, + "--state", + "open", + "--label", + LABEL, + "--json", + "number", + "--limit", + "2", + ]); + const openIssue = openIssueNumber(parseJson(issueResponse, "issue list")); + const action = backstopAction({ + result: environment.result, + event: environment.event, + runUrl: `${environment.serverUrl}/${repository}/actions/runs/${runId}`, + failedJobs, + openIssue, + runStartedAt: currentRunStartedAt, + newestSweepStartedAt, + }); + + if (dryRun[0] === "--dry-run") { + console.log(JSON.stringify(action, null, 2)); + return; + } + + switch (action.kind) { + case "open": + await gh([ + "issue", + "create", + "--repo", + repository, + "--title", + action.title, + "--body", + action.body, + "--label", + LABEL, + ]); + break; + case "comment": + await gh([ + "issue", + "comment", + String(action.issue), + "--repo", + repository, + "--body", + action.body, + ]); + break; + case "close": + await gh([ + "issue", + "close", + String(action.issue), + "--repo", + repository, + "--comment", + action.body, + ]); + break; + case "none": + console.log("No backstop issue action needed."); + break; + } +} + +if (import.meta.main) { + try { + await main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/tools/ci-backstop/moon.yml b/tools/ci-backstop/moon.yml new file mode 100644 index 000000000..7fb4dafea --- /dev/null +++ b/tools/ci-backstop/moon.yml @@ -0,0 +1,19 @@ +# yaml-language-server: $schema=https://moonrepo.dev/schemas/project.json +# +# A full-sweep failure backstop. Its pure action decision and tests ride the +# bun CI group; the workflow job routes push and schedule results to one issue. +layer: 'tool' +language: 'typescript' +tags: ['bun', 'oss', 'ci-group.bun'] + +tasks: + typecheck: + command: 'bunx tsc --noEmit' + deps: ['install'] + inputs: ['*.ts', 'tsconfig.json', '/tsconfig.base.json', 'package.json', '/bun.lock'] + test: + inputs: ['*.ts', 'tsconfig.json', '/tsconfig.base.json', 'package.json', '/bun.lock'] + ci: + deps: ['typecheck', 'test'] + options: + cache: false diff --git a/tools/ci-backstop/package.json b/tools/ci-backstop/package.json new file mode 100644 index 000000000..f751ff529 --- /dev/null +++ b/tools/ci-backstop/package.json @@ -0,0 +1,14 @@ +{ + "name": "@compass/ci-backstop", + "private": true, + "type": "module", + "description": "Routes failed full CI sweeps to one GitHub issue.", + "module": "index.ts", + "bin": { + "ci-backstop": "./index.ts" + }, + "devDependencies": { + "@types/bun": "catalog:", + "typescript": "catalog:" + } +} diff --git a/tools/ci-backstop/tsconfig.json b/tools/ci-backstop/tsconfig.json new file mode 100644 index 000000000..d40cc9e50 --- /dev/null +++ b/tools/ci-backstop/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022"], + "moduleDetection": "force", + "allowJs": true, + "allowImportingTsExtensions": true, + "noUncheckedIndexedAccess": true, + "types": ["bun"] + } +} From 4f2b3c917372ec0ea1e835799c0e04a540852425 Mon Sep 17 00:00:00 2001 From: mintaka Date: Fri, 16 Oct 2026 00:56:41 -0400 Subject: [PATCH 2/2] fix(ci): let the backstop read runs and name timed-out jobs (RIG-5491) Co-authored-by: Matt Wilkinson --- .github/workflows/ci.yml | 1 + tools/ci-backstop/index.test.ts | 20 ++++++++++++++++++++ tools/ci-backstop/index.ts | 8 ++++++-- 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 tools/ci-backstop/index.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a02e1fa5..c52f9b08c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2895,6 +2895,7 @@ jobs: if: ${{ !cancelled() && contains(fromJSON('["push","schedule"]'), github.event_name) }} runs-on: ubuntu-latest permissions: + actions: read contents: read issues: write concurrency: diff --git a/tools/ci-backstop/index.test.ts b/tools/ci-backstop/index.test.ts new file mode 100644 index 000000000..3742254b8 --- /dev/null +++ b/tools/ci-backstop/index.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, test } from "bun:test"; +import { failedJobNames } from "./index.ts"; + +describe("failedJobNames", () => { + test("names failed, timed-out and cancelled jobs only", () => { + const jobs = [ + { name: "moon (go)", conclusion: "failure" }, + { name: "pgtest", conclusion: "timed_out" }, + { name: "e2e", conclusion: "cancelled" }, + { name: "setup", conclusion: "success" }, + { name: "microvm", conclusion: "skipped" }, + { name: "backstop", conclusion: null }, + ]; + expect(failedJobNames({ jobs })).toEqual(["moon (go)", "pgtest", "e2e"]); + }); + + test("rejects a response without a jobs array", () => { + expect(() => failedJobNames({})).toThrow("no jobs array"); + }); +}); diff --git a/tools/ci-backstop/index.ts b/tools/ci-backstop/index.ts index 7b038a4c9..edfafb078 100755 --- a/tools/ci-backstop/index.ts +++ b/tools/ci-backstop/index.ts @@ -103,14 +103,18 @@ function runStartedAt(value: unknown, source: string): string { return startedAt; } -function failedJobNames(value: unknown): string[] { +// A rollup also fails on a timed-out or cancelled need, so name those jobs too. +const UNSUCCESSFUL = new Set(["failure", "timed_out", "cancelled"]); + +export function failedJobNames(value: unknown): string[] { const response = record(value); if (!Array.isArray(response.jobs)) { throw new Error("gh jobs response has no jobs array"); } return response.jobs.flatMap((job) => { const data = record(job); - if (data.conclusion !== "failure") return []; + if (typeof data.conclusion !== "string") return []; + if (!UNSUCCESSFUL.has(data.conclusion)) return []; if (typeof data.name !== "string") { throw new Error("gh returned a failed job without a name"); }