diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aaaf3cd42..d69fdfb71 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2909,6 +2909,46 @@ jobs: fi echo "all required work jobs succeeded (or correctly skipped per their setup flag)" + + backstop: + name: backstop + needs: rollup + if: ${{ !cancelled() && contains(fromJSON('["push","schedule"]'), github.event_name) }} + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + issues: write + concurrency: + group: ci-backstop + cancel-in-progress: false + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31 + with: + extra_nix_config: | + experimental-features = nix-command flakes + extra-substituters = https://devenv.cachix.org https://cachix.cachix.org + extra-trusted-public-keys = devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw= cachix.cachix.org-1:eWNHQldwUO7G2VkjpnjDbWwy4KQ/HNxht7H4SSoMckM= + - name: Put the pinned bun toolchain on PATH + # bun comes from the nix-pinned toolchain, never setup-bun, so it is the + # same derivation as the dev shell's. `jq -r` renders a missing `.store` + # as the literal `null`, so guard for it explicitly. + run: | + set -euo pipefail + store=$(nix eval --json -f tools/toolchain/gate-tools.nix langs.bun | jq -r '.store') + if [ -z "$store" ] || [ "$store" = null ]; then + echo "::error::gate-tools.nix langs.bun produced no store path" >&2 + exit 1 + fi + nix build --no-link "$store" + echo "$store/bin" >>"$GITHUB_PATH" + - name: Route failed full sweep + env: + ROLLUP_RESULT: ${{ needs.rollup.result }} + GH_TOKEN: ${{ github.token }} + run: bun tools/ci-backstop/index.ts + regen-forge-fixtures: name: regen-fixtures runs-on: ubuntu-latest diff --git a/.moon/workspace.yml b/.moon/workspace.yml index bbc8038a2..89ee30be7 100644 --- a/.moon/workspace.yml +++ b/.moon/workspace.yml @@ -219,6 +219,8 @@ projects: # The Bun advisory check is a dedicated, always-run workflow job; this project # keeps its typecheck and tests in the affected Moon matrix. bun-audit-gate: 'tools/bun-audit-gate' + # The full-sweep failure backstop: a pure issue decision plus its tests. + ci-backstop: 'tools/ci-backstop' # File hashing uses moon's in-process native walker (the v2.5+ default), NOT the # git shell-out. This is load-bearing: the git hasher (`git ls-files` then `git diff --git a/bun.lock b/bun.lock index 7207ef4d9..ac4d50821 100644 --- a/bun.lock +++ b/bun.lock @@ -137,6 +137,16 @@ "typescript": "catalog:", }, }, + "tools/ci-backstop": { + "name": "@compass/ci-backstop", + "bin": { + "ci-backstop": "./index.ts", + }, + "devDependencies": { + "@types/bun": "catalog:", + "typescript": "catalog:", + }, + }, "tools/ci-matrix": { "name": "@compass/ci-matrix", "bin": { @@ -581,6 +591,8 @@ "@compass/bun-audit-gate": ["@compass/bun-audit-gate@workspace:tools/bun-audit-gate"], + "@compass/ci-backstop": ["@compass/ci-backstop@workspace:tools/ci-backstop"], + "@compass/ci-matrix": ["@compass/ci-matrix@workspace:tools/ci-matrix"], "@compass/ci-timing": ["@compass/ci-timing@workspace:tools/ci-timing"], diff --git a/tools/ci-backstop/README.md b/tools/ci-backstop/README.md new file mode 100644 index 000000000..b4d7d7b49 --- /dev/null +++ b/tools/ci-backstop/README.md @@ -0,0 +1,16 @@ +# ci-backstop + +Routes failed full CI sweeps to one `ci-backstop` issue. A green full sweep +closes that issue. `core.ts` holds the pure action decision; `index.ts` reads +GitHub Actions data and applies it through `gh`. + +## Run + +```sh +bun tools/ci-backstop/index.ts [--dry-run] +moon run ci-backstop:ci +``` + +The `backstop` job in `.github/workflows/ci.yml` runs on push and schedule +results after `rollup`. `GH_TOKEN` must have issue write access. `--dry-run` +prints the action and does not create the label or change an issue. diff --git a/tools/ci-backstop/biome.json b/tools/ci-backstop/biome.json new file mode 100644 index 000000000..9cad0143f --- /dev/null +++ b/tools/ci-backstop/biome.json @@ -0,0 +1,5 @@ +{ + "root": false, + "extends": "//", + "linter": { "rules": { "suspicious": { "noConsole": "off" } } } +} diff --git a/tools/ci-backstop/core.test.ts b/tools/ci-backstop/core.test.ts new file mode 100644 index 000000000..df4a8e063 --- /dev/null +++ b/tools/ci-backstop/core.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, test } from "bun:test"; +import { backstopAction } from "./core.ts"; + +const runUrl = "https://github.com/RigelBuild/compass/actions/runs/123"; +const failedJobs = ["moon (bun)", "pgtest"]; + +const cases = [ + { + name: "failure without an issue opens one", + result: "failure", + openIssue: null, + newerSweep: false, + kind: "open", + }, + { + name: "failure with an issue comments", + result: "failure", + openIssue: 42, + newerSweep: false, + kind: "comment", + }, + { + name: "success without an issue does nothing", + result: "success", + openIssue: null, + newerSweep: false, + kind: "none", + }, + { + name: "success with an issue closes it", + result: "success", + openIssue: 42, + newerSweep: false, + kind: "close", + }, + { + name: "newer sweep supersedes failure without an issue", + result: "failure", + openIssue: null, + newerSweep: true, + kind: "none", + }, + { + name: "newer sweep supersedes failure with an issue", + result: "failure", + openIssue: 42, + newerSweep: true, + kind: "none", + }, + { + name: "newer sweep supersedes success without an issue", + result: "success", + openIssue: null, + newerSweep: true, + kind: "none", + }, + { + name: "newer sweep supersedes success with an issue", + result: "success", + openIssue: 42, + newerSweep: true, + kind: "none", + }, +] as const; + +describe("backstopAction", () => { + for (const scenario of cases) { + test(scenario.name, () => { + const action = backstopAction({ + result: scenario.result, + event: "push", + runUrl, + failedJobs, + openIssue: scenario.openIssue, + runStartedAt: "2026-10-15T10:00:00Z", + newestSweepStartedAt: scenario.newerSweep + ? "2026-10-15T10:00:01Z" + : "2026-10-15T10:00:00Z", + }); + + expect(action.kind).toBe(scenario.kind); + if (action.kind !== "none") { + expect(action.body).toContain(runUrl); + expect(action.body).toContain("moon (bun), pgtest"); + expect(action.body.split("\n")).toHaveLength(2); + } + if (action.kind === "open") { + expect(action.title).toBe("CI push full sweep failed"); + } + }); + } +}); diff --git a/tools/ci-backstop/core.ts b/tools/ci-backstop/core.ts new file mode 100644 index 000000000..dfe4ff337 --- /dev/null +++ b/tools/ci-backstop/core.ts @@ -0,0 +1,40 @@ +export type BackstopAction = + | { kind: "open"; title: string; body: string } + | { kind: "comment" | "close"; issue: number; body: string } + | { kind: "none" }; + +export function backstopAction(input: { + result: "success" | "failure"; + event: "push" | "schedule"; + runUrl: string; + failedJobs: readonly string[]; + openIssue: number | null; + runStartedAt: string; + newestSweepStartedAt: string; +}): BackstopAction { + if (Date.parse(input.newestSweepStartedAt) > Date.parse(input.runStartedAt)) { + return { kind: "none" }; + } + + const failedJobs = + input.failedJobs.length > 0 ? input.failedJobs.join(", ") : "none"; + const body = [ + `CI ${input.event} full sweep: ${input.result}. Run: ${input.runUrl}`, + `Failed jobs: ${failedJobs}`, + ].join("\n"); + + if (input.result === "failure") { + if (input.openIssue !== null) { + return { kind: "comment", issue: input.openIssue, body }; + } + return { + kind: "open", + title: `CI ${input.event} full sweep failed`, + body, + }; + } + + return input.openIssue === null + ? { kind: "none" } + : { kind: "close", issue: input.openIssue, body }; +} diff --git a/tools/ci-backstop/index.test.ts b/tools/ci-backstop/index.test.ts new file mode 100644 index 000000000..f285b60d4 --- /dev/null +++ b/tools/ci-backstop/index.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, test } from "bun:test"; +import { failedJobNames, newestVerdictStartedAt } from "./index.ts"; + +describe("failedJobNames", () => { + test("names failed, timed-out and cancelled jobs only", () => { + const jobs = [ + { name: "moon (go)", conclusion: "failure" }, + { name: "pgtest", conclusion: "timed_out" }, + { name: "e2e", conclusion: "cancelled" }, + { name: "setup", conclusion: "success" }, + { name: "microvm", conclusion: "skipped" }, + { name: "backstop", conclusion: null }, + ]; + expect(failedJobNames({ jobs })).toEqual(["moon (go)", "pgtest", "e2e"]); + }); + + test("rejects a response without a jobs array", () => { + expect(() => failedJobNames({})).toThrow("no jobs array"); + }); +}); + +describe("newestVerdictStartedAt", () => { + const run = (conclusion: string, at: string) => ({ + status: "completed", + conclusion, + run_started_at: at, + }); + + test("a newer cancelled run does not supersede an older verdict", () => { + const runs = [ + run("cancelled", "2026-10-16T12:00:00Z"), + run("failure", "2026-10-16T11:00:00Z"), + ]; + expect(newestVerdictStartedAt(runs)).toBe("2026-10-16T11:00:00Z"); + }); + + test("the newest success or failure wins", () => { + const runs = [ + run("success", "2026-10-16T12:00:00Z"), + run("failure", "2026-10-16T11:00:00Z"), + ]; + expect(newestVerdictStartedAt(runs)).toBe("2026-10-16T12:00:00Z"); + }); + + test("no verdict among the runs reads as none", () => { + expect( + newestVerdictStartedAt([run("cancelled", "2026-10-16T12:00:00Z")]), + ).toBeNull(); + }); +}); diff --git a/tools/ci-backstop/index.ts b/tools/ci-backstop/index.ts new file mode 100755 index 000000000..c49ad3889 --- /dev/null +++ b/tools/ci-backstop/index.ts @@ -0,0 +1,305 @@ +#!/usr/bin/env bun +import { backstopAction } from "./core.ts"; + +const LABEL = "ci-backstop"; + +type RunEnvironment = { + result: "success" | "failure"; + event: "push" | "schedule"; + runId: string; + repository: string; + serverUrl: string; +}; + +function requiredEnv(name: string): string { + const value = process.env[name]; + if (value === undefined || value.length === 0) { + throw new Error(`${name} is required`); + } + return value; +} + +function readEnvironment(): RunEnvironment { + const result = requiredEnv("ROLLUP_RESULT"); + if (result !== "success" && result !== "failure") { + throw new Error(`unexpected ROLLUP_RESULT: ${result}`); + } + + const event = requiredEnv("GITHUB_EVENT_NAME"); + if (event !== "push" && event !== "schedule") { + throw new Error(`unexpected GITHUB_EVENT_NAME: ${event}`); + } + + const runId = requiredEnv("GITHUB_RUN_ID"); + if (!/^\d+$/.test(runId)) { + throw new Error(`GITHUB_RUN_ID must be a positive integer: ${runId}`); + } + + const repository = requiredEnv("GITHUB_REPOSITORY"); + if (!/^[^/]+\/[^/]+$/.test(repository)) { + throw new Error( + `GITHUB_REPOSITORY must be owner/repository: ${repository}`, + ); + } + + const serverUrl = requiredEnv("GITHUB_SERVER_URL").replace(/\/$/, ""); + const parsedServerUrl = new URL(serverUrl); + if ( + parsedServerUrl.protocol !== "https:" && + parsedServerUrl.protocol !== "http:" + ) { + throw new Error("GITHUB_SERVER_URL must use HTTP or HTTPS"); + } + return { + result, + event, + runId, + repository, + serverUrl, + }; +} + +async function gh(args: readonly string[]): Promise { + const child = Bun.spawn(["gh", ...args], { + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + if (exitCode !== 0) { + throw new Error( + `gh ${args.join(" ")} failed with exit code ${exitCode}: ${stderr.trim()}`, + ); + } + return stdout.trim(); +} + +function parseJson(value: string, command: string): unknown { + try { + return JSON.parse(value); + } catch (error) { + throw new Error( + `gh ${command} returned invalid JSON: ${error instanceof Error ? error.message : String(error)}`, + ); + } +} + +function record(value: unknown): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("gh returned an unexpected JSON object"); + } + return Object.fromEntries(Object.entries(value)); +} + +function runStartedAt(value: unknown, source: string): string { + const data = record(value); + const startedAt = data.run_started_at; + if (typeof startedAt !== "string" || Number.isNaN(Date.parse(startedAt))) { + throw new Error(`${source} has no valid run_started_at`); + } + return startedAt; +} + +// A rollup also fails on a timed-out or cancelled need, so name those jobs too. +const UNSUCCESSFUL = new Set(["failure", "timed_out", "cancelled"]); + +export function failedJobNames(value: unknown): string[] { + const response = record(value); + if (!Array.isArray(response.jobs)) { + throw new Error("gh jobs response has no jobs array"); + } + return response.jobs.flatMap((job) => { + const data = record(job); + if (typeof data.conclusion !== "string") return []; + if (!UNSUCCESSFUL.has(data.conclusion)) return []; + if (typeof data.name !== "string") { + throw new Error("gh returned a failed job without a name"); + } + return [data.name]; + }); +} + +function openIssueNumber(value: unknown): number | null { + if (!Array.isArray(value)) { + throw new Error("gh issue list response is not an array"); + } + if (value.length > 1) { + throw new Error(`more than one open issue has the ${LABEL} label`); + } + const issue = value[0]; + if (issue === undefined) return null; + const data = record(issue); + if (typeof data.number !== "number" || !Number.isSafeInteger(data.number)) { + throw new Error("gh returned an issue without a valid number"); + } + return data.number; +} + +// A cancelled run never produced a rollup verdict, so it must not supersede this one. +const VERDICTS = ["success", "failure"] as const; + +export function newestVerdictStartedAt(runs: unknown): string | null { + if (!Array.isArray(runs)) { + throw new Error("gh returned no workflow_runs array"); + } + let newest: string | null = null; + for (const run of runs) { + const data = record(run); + if (data.status !== "completed") { + throw new Error("gh returned a run that is not completed"); + } + if (data.conclusion !== "success" && data.conclusion !== "failure") { + continue; + } + const startedAt = runStartedAt(data, "completed workflow run"); + if (newest === null || Date.parse(startedAt) > Date.parse(newest)) { + newest = startedAt; + } + } + return newest; +} + +async function startedAtForLatestVerdict( + repository: string, + event: "push" | "schedule", +): Promise { + // The status filter takes a conclusion, so each query returns that verdict's newest run. + const pages = await Promise.all( + VERDICTS.map(async (verdict) => { + const endpoint = `repos/${repository}/actions/workflows/ci.yml/runs?event=${event}&status=${verdict}&per_page=1`; + const runs = parseJson( + await gh([ + "api", + "--jq", + ".workflow_runs | map({run_started_at, status, conclusion})", + endpoint, + ]), + `api ${endpoint}`, + ); + if (!Array.isArray(runs)) { + throw new Error(`gh api ${endpoint} returned no workflow_runs array`); + } + return runs; + }), + ); + return newestVerdictStartedAt(pages.flat()); +} + +async function main(): Promise { + const dryRun = process.argv.slice(2); + if (dryRun.length > 1 || (dryRun.length === 1 && dryRun[0] !== "--dry-run")) { + throw new Error("usage: bun tools/ci-backstop/index.ts [--dry-run]"); + } + const environment = readEnvironment(); + const { repository, runId } = environment; + + const runEndpoint = `repos/${repository}/actions/runs/${runId}`; + const [runResponse, jobsResponse] = await Promise.all([ + gh(["api", runEndpoint]), + gh(["api", `${runEndpoint}/jobs?per_page=100`]), + ]); + const currentRunStartedAt = runStartedAt( + parseJson(runResponse, `api ${runEndpoint}`), + "current workflow run", + ); + const failedJobs = failedJobNames( + parseJson(jobsResponse, `api ${runEndpoint}/jobs`), + ); + + const [latestPush, latestSchedule] = await Promise.all([ + startedAtForLatestVerdict(repository, "push"), + startedAtForLatestVerdict(repository, "schedule"), + ]); + const newestSweepStartedAt = + [latestPush, latestSchedule] + .filter((startedAt): startedAt is string => startedAt !== null) + .sort((left, right) => Date.parse(left) - Date.parse(right)) + .at(-1) ?? currentRunStartedAt; + + if (dryRun[0] !== "--dry-run") { + await gh(["label", "create", LABEL, "--force", "--repo", repository]); + } + const issueResponse = await gh([ + "issue", + "list", + "--repo", + repository, + "--state", + "open", + "--label", + LABEL, + "--json", + "number", + "--limit", + "2", + ]); + const openIssue = openIssueNumber(parseJson(issueResponse, "issue list")); + const action = backstopAction({ + result: environment.result, + event: environment.event, + runUrl: `${environment.serverUrl}/${repository}/actions/runs/${runId}`, + failedJobs, + openIssue, + runStartedAt: currentRunStartedAt, + newestSweepStartedAt, + }); + + if (dryRun[0] === "--dry-run") { + console.log(JSON.stringify(action, null, 2)); + return; + } + + switch (action.kind) { + case "open": + await gh([ + "issue", + "create", + "--repo", + repository, + "--title", + action.title, + "--body", + action.body, + "--label", + LABEL, + ]); + break; + case "comment": + await gh([ + "issue", + "comment", + String(action.issue), + "--repo", + repository, + "--body", + action.body, + ]); + break; + case "close": + await gh([ + "issue", + "close", + String(action.issue), + "--repo", + repository, + "--comment", + action.body, + ]); + break; + case "none": + console.log("No backstop issue action needed."); + break; + } +} + +if (import.meta.main) { + try { + await main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/tools/ci-backstop/moon.yml b/tools/ci-backstop/moon.yml new file mode 100644 index 000000000..7fb4dafea --- /dev/null +++ b/tools/ci-backstop/moon.yml @@ -0,0 +1,19 @@ +# yaml-language-server: $schema=https://moonrepo.dev/schemas/project.json +# +# A full-sweep failure backstop. Its pure action decision and tests ride the +# bun CI group; the workflow job routes push and schedule results to one issue. +layer: 'tool' +language: 'typescript' +tags: ['bun', 'oss', 'ci-group.bun'] + +tasks: + typecheck: + command: 'bunx tsc --noEmit' + deps: ['install'] + inputs: ['*.ts', 'tsconfig.json', '/tsconfig.base.json', 'package.json', '/bun.lock'] + test: + inputs: ['*.ts', 'tsconfig.json', '/tsconfig.base.json', 'package.json', '/bun.lock'] + ci: + deps: ['typecheck', 'test'] + options: + cache: false diff --git a/tools/ci-backstop/package.json b/tools/ci-backstop/package.json new file mode 100644 index 000000000..f751ff529 --- /dev/null +++ b/tools/ci-backstop/package.json @@ -0,0 +1,14 @@ +{ + "name": "@compass/ci-backstop", + "private": true, + "type": "module", + "description": "Routes failed full CI sweeps to one GitHub issue.", + "module": "index.ts", + "bin": { + "ci-backstop": "./index.ts" + }, + "devDependencies": { + "@types/bun": "catalog:", + "typescript": "catalog:" + } +} diff --git a/tools/ci-backstop/tsconfig.json b/tools/ci-backstop/tsconfig.json new file mode 100644 index 000000000..d40cc9e50 --- /dev/null +++ b/tools/ci-backstop/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022"], + "moduleDetection": "force", + "allowJs": true, + "allowImportingTsExtensions": true, + "noUncheckedIndexedAccess": true, + "types": ["bun"] + } +}