diff --git a/.moon/workspace.yml b/.moon/workspace.yml index 6cb17628c..bbc8038a2 100644 --- a/.moon/workspace.yml +++ b/.moon/workspace.yml @@ -177,6 +177,9 @@ projects: # query projects`, or the generator's zero-untagged assertion never covers the # very tool that enforces it. ci-matrix: 'tools/ci-matrix' + # CI Actions timing measurements; registering the project keeps its pure metric + # tests and typecheck in the normal Moon discovery and CI sweep. + ci-timing: 'tools/ci-timing' # The macOS .app/.dmg bundler (compass-distribution T3): templates Info.plist, # stages Compass.app/Contents/{MacOS,Resources}, ad-hoc signs (codesign # --sign -, GC6/DL-261) and wraps a UDZO .dmg via hdiutil. Registered so its diff --git a/bun.lock b/bun.lock index f6c35e308..38742dcf2 100644 --- a/bun.lock +++ b/bun.lock @@ -147,6 +147,16 @@ "typescript": "catalog:", }, }, + "tools/ci-timing": { + "name": "@compass/ci-timing", + "bin": { + "ci-timing": "./index.ts", + }, + "devDependencies": { + "@types/bun": "catalog:", + "typescript": "catalog:", + }, + }, "tools/compass-app-dev": { "name": "@compass/compass-app-dev", "bin": { @@ -575,6 +585,8 @@ "@compass/ci-matrix": ["@compass/ci-matrix@workspace:tools/ci-matrix"], + "@compass/ci-timing": ["@compass/ci-timing@workspace:tools/ci-timing"], + "@compass/client": ["@compass/client@workspace:packages/compass-client"], "@compass/compass-app-dev": ["@compass/compass-app-dev@workspace:tools/compass-app-dev"], diff --git a/tools/ci-timing/README.md b/tools/ci-timing/README.md new file mode 100644 index 000000000..77791c78e --- /dev/null +++ b/tools/ci-timing/README.md @@ -0,0 +1,13 @@ +# ci-timing + +Measures GitHub Actions run and job timing for the `ci.yml` workflow by run +class. It reports wall time, runner wait, job duration, skip rate, long pole, +runner minutes, and UTC hours with long waits. + +```sh +bun tools/ci-timing/index.ts --since 2026-10-01 [--until 2026-10-15] [--json] +moon run ci-timing:ci +``` + +The CLI uses the authenticated GitHub CLI to read workflow runs and their job +metadata. It does not read logs or change repository state. diff --git a/tools/ci-timing/biome.json b/tools/ci-timing/biome.json new file mode 100644 index 000000000..9cad0143f --- /dev/null +++ b/tools/ci-timing/biome.json @@ -0,0 +1,5 @@ +{ + "root": false, + "extends": "//", + "linter": { "rules": { "suspicious": { "noConsole": "off" } } } +} diff --git a/tools/ci-timing/core.test.ts b/tools/ci-timing/core.test.ts new file mode 100644 index 000000000..4712ddbb1 --- /dev/null +++ b/tools/ci-timing/core.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, test } from "bun:test"; +import type { JobRecord, RunRecord } from "./core.ts"; +import { AGGREGATOR_JOBS, classify, summarize } from "./core.ts"; + +function job( + name: string, + createdAt: number, + startedAt: number, + completedAt: number, + conclusion: JobRecord["conclusion"] = "success", +): JobRecord { + return { name, conclusion, createdAt, startedAt, completedAt }; +} + +function run( + id: number, + jobs: JobRecord[], + overrides: Partial = {}, +): RunRecord { + return { + id, + event: "pull_request", + headBranch: "feature/example", + conclusion: "success", + runStartedAt: 100, + updatedAt: 200, + jobs, + ...overrides, + }; +} + +describe("classify", () => { + test("covers every run class, including Trunk merge-queue branches", () => { + expect(classify({ event: "pull_request", headBranch: "feature/a" })).toBe( + "pr", + ); + expect( + classify({ event: "pull_request", headBranch: "trunk-merge/123" }), + ).toBe("queue"); + expect(classify({ event: "merge_group", headBranch: "main" })).toBe( + "merge_group", + ); + expect(classify({ event: "push", headBranch: "main" })).toBe("push"); + expect(classify({ event: "schedule", headBranch: "main" })).toBe( + "schedule", + ); + expect(classify({ event: "workflow_dispatch", headBranch: "main" })).toBe( + "dispatch", + ); + }); +}); + +describe("summarize", () => { + test("counts cancelled runs separately and excludes them from measurements", () => { + const summary = summarize([ + run(1, [job("build", 100, 110, 150)], { updatedAt: 160 }), + run(2, [job("build", 100, 110, 190)], { + conclusion: "cancelled", + updatedAt: 200, + }), + ]); + + expect(summary.pr.kept).toBe(1); + expect(summary.pr.cancelled).toBe(1); + expect(summary.pr.wall).toEqual({ + n: 1, + p50: 60, + p90: 60, + p99: 60, + max: 60, + }); + expect(summary.pr.jobDuration.build?.n).toBe(1); + }); + + test("computes per-job skip rate and long-pole share over three runs", () => { + const summary = summarize([ + run(1, [job("build", 100, 101, 110), job("deploy", 100, 102, 120)]), + run(2, [ + job("build", 100, 100, 100, "skipped"), + job("deploy", 100, 103, 115), + ]), + run(3, [job("build", 100, 101, 130), job("deploy", 100, 102, 125)]), + ]); + + expect(summary.pr.skipRate.build).toBeCloseTo(1 / 3); + expect(summary.pr.longPole.build).toBeCloseTo(1 / 3); + expect(summary.pr.longPole.deploy).toBeCloseTo(2 / 3); + }); + test("never counts an aggregator job as the long pole", () => { + const summary = summarize([ + run(1, [job("e2e", 100, 101, 200), job("rollup", 100, 201, 205)]), + ]); + + expect(summary.pr.longPole.e2e).toBe(1); + expect(summary.pr.longPole.rollup).toBeUndefined(); + expect(summary.pr.jobDuration.rollup?.n).toBe(1); + }); + + test("AGGREGATOR_JOBS name ci.yml jobs that wait on the PR gate's jobs", async () => { + const workflow = await Bun.file( + new URL("../../.github/workflows/ci.yml", import.meta.url), + ).text(); + const jobs = Bun.YAML.parse(workflow) as { + jobs: Record; + }; + for (const name of AGGREGATOR_JOBS) { + const needs = [jobs.jobs[name]?.needs ?? []].flat(); + // e2e is the measured long pole, so an aggregator that skipped it would hide it. + expect(needs).toContain("e2e"); + expect(needs.length).toBeGreaterThan(5); + } + }); + test("computes max wait and runner minutes from job timestamps", () => { + const summary = summarize([ + run(1, [job("build", 100, 110, 170), job("test", 100, 125, 185)]), + ]); + + expect(summary.pr.maxWait).toEqual({ + n: 1, + p50: 25, + p90: 25, + p99: 25, + max: 25, + }); + expect(summary.pr.runnerMinutes).toEqual({ + n: 1, + p50: 2, + p90: 2, + p99: 2, + max: 2, + }); + }); + + test("uses nearest-rank quantiles for one and two values", () => { + const one = summarize([run(1, [], { runStartedAt: 100, updatedAt: 150 })]); + const two = summarize([ + run(1, [], { runStartedAt: 100, updatedAt: 110 }), + run(2, [], { runStartedAt: 100, updatedAt: 120 }), + ]); + + expect(one.pr.wall).toEqual({ n: 1, p50: 50, p90: 50, p99: 50, max: 50 }); + expect(two.pr.wall).toEqual({ n: 2, p50: 10, p90: 20, p99: 20, max: 20 }); + }); + + test("buckets waits over 120 seconds by UTC hour", () => { + const firstHour = Date.parse("2026-10-11T00:30:00Z") / 1000; + const secondHour = Date.parse("2026-10-11T01:01:00Z") / 1000; + const summary = summarize([ + run(1, [ + job("slow-a", firstHour, firstHour + 121, firstHour + 130), + job("slow-b", firstHour + 1000, firstHour + 1120, firstHour + 1130), + job("slow-c", secondHour, secondHour + 121, secondHour + 130), + ]), + ]); + + expect(summary.pr.hoursOver120s).toEqual([ + "2026-10-11T00", + "2026-10-11T01", + ]); + }); +}); diff --git a/tools/ci-timing/core.ts b/tools/ci-timing/core.ts new file mode 100644 index 000000000..9cf4a53d4 --- /dev/null +++ b/tools/ci-timing/core.ts @@ -0,0 +1,262 @@ +export type RunClass = + | "pr" + | "queue" + | "merge_group" + | "push" + | "schedule" + | "dispatch"; + +export type JobRecord = { + name: string; + conclusion: "success" | "failure" | "cancelled" | "skipped"; + createdAt: number; + startedAt: number; + completedAt: number; +}; + +export type RunRecord = { + id: number; + event: string; + headBranch: string; + conclusion: "success" | "failure" | "cancelled"; + runStartedAt: number; + updatedAt: number; + jobs: JobRecord[]; +}; + +// Jobs that wait on every other job always finish last, so they can never be the long pole. +export const AGGREGATOR_JOBS: readonly string[] = ["rollup"]; + +export function classify( + run: Pick, +): RunClass { + if (run.event === "pull_request") { + return run.headBranch.startsWith("trunk-merge/") ? "queue" : "pr"; + } + + switch (run.event) { + case "merge_group": + return "merge_group"; + case "push": + return "push"; + case "schedule": + return "schedule"; + case "workflow_dispatch": + return "dispatch"; + default: + throw new Error(`Unsupported workflow event: ${run.event}`); + } +} + +export type Quantiles = { + n: number; + p50: number; + p90: number; + p99: number; + max: number; +}; + +export type ClassSummary = { + kept: number; + cancelled: number; + wall: Quantiles; + maxWait: Quantiles; + runnerMinutes: Quantiles; + jobDuration: Record; + jobWait: Record; + skipRate: Record; + longPole: Record; + hoursOver120s: string[]; +}; + +type MutableClassSummary = { + kept: number; + cancelled: number; + wall: number[]; + maxWait: number[]; + runnerMinutes: number[]; + jobDuration: Map; + jobWait: Map; + jobCount: Map; + skippedCount: Map; + longPoleCount: Map; + hoursOver120s: Set; +}; + +function emptyClassSummary(): MutableClassSummary { + return { + kept: 0, + cancelled: 0, + wall: [], + maxWait: [], + runnerMinutes: [], + jobDuration: new Map(), + jobWait: new Map(), + jobCount: new Map(), + skippedCount: new Map(), + longPoleCount: new Map(), + hoursOver120s: new Set(), + }; +} + +function appendValue( + values: Map, + key: string, + value: number, +): void { + const samples = values.get(key); + if (samples) { + samples.push(value); + } else { + values.set(key, [value]); + } +} + +function increment(values: Map, key: string): void { + values.set(key, (values.get(key) ?? 0) + 1); +} + +function quantiles(values: readonly number[]): Quantiles { + const sorted = [...values].sort((left, right) => left - right); + if (sorted.length === 0) { + return { n: 0, p50: 0, p90: 0, p99: 0, max: 0 }; + } + const nearestRank = (p: number): number => + sorted[Math.ceil(p * sorted.length) - 1] ?? 0; + return { + n: sorted.length, + p50: nearestRank(0.5), + p90: nearestRank(0.9), + p99: nearestRank(0.99), + max: sorted[sorted.length - 1] ?? 0, + }; +} + +function summarizeClass(summary: MutableClassSummary): ClassSummary { + const jobDuration: Record = {}; + const jobWait: Record = {}; + const skipRate: Record = {}; + const longPole: Record = {}; + + for (const [name, samples] of summary.jobDuration) { + jobDuration[name] = quantiles(samples); + } + for (const [name, samples] of summary.jobWait) { + jobWait[name] = quantiles(samples); + } + for (const [name, total] of summary.jobCount) { + skipRate[name] = (summary.skippedCount.get(name) ?? 0) / total; + } + for (const [name, count] of summary.longPoleCount) { + longPole[name] = summary.kept === 0 ? 0 : count / summary.kept; + } + + return { + kept: summary.kept, + cancelled: summary.cancelled, + wall: quantiles(summary.wall), + maxWait: quantiles(summary.maxWait), + runnerMinutes: quantiles(summary.runnerMinutes), + jobDuration, + jobWait, + skipRate, + longPole, + hoursOver120s: [...summary.hoursOver120s].sort(), + }; +} + +function summarizeJob( + job: JobRecord, + summary: MutableClassSummary, + current: { + maxWait: number; + runnerSeconds: number; + latestCompletion: number; + lastJobs: string[]; + }, +): void { + increment(summary.jobCount, job.name); + if (job.conclusion === "skipped") { + increment(summary.skippedCount, job.name); + } + if (job.startedAt <= 0) { + return; + } + const wait = job.startedAt - job.createdAt; + appendValue(summary.jobWait, job.name, wait); + current.maxWait = Math.max(current.maxWait, wait); + if (wait > 120) { + summary.hoursOver120s.add( + new Date(job.createdAt * 1000).toISOString().slice(0, 13), + ); + } + if (job.conclusion === "skipped" || job.completedAt <= 0) { + return; + } + + const duration = job.completedAt - job.startedAt; + appendValue(summary.jobDuration, job.name, duration); + current.runnerSeconds += duration; + if (AGGREGATOR_JOBS.includes(job.name)) { + return; + } + if (job.completedAt > current.latestCompletion) { + current.latestCompletion = job.completedAt; + current.lastJobs.length = 0; + current.lastJobs.push(job.name); + } else if (job.completedAt === current.latestCompletion) { + current.lastJobs.push(job.name); + } +} + +function summarizeRun(summary: MutableClassSummary, run: RunRecord): void { + if (run.conclusion === "cancelled") { + summary.cancelled += 1; + return; + } + + summary.kept += 1; + summary.wall.push(run.updatedAt - run.runStartedAt); + const current = { + maxWait: 0, + runnerSeconds: 0, + latestCompletion: Number.NEGATIVE_INFINITY, + lastJobs: [] as string[], + }; + for (const job of run.jobs) { + summarizeJob(job, summary, current); + } + + summary.maxWait.push(current.maxWait); + summary.runnerMinutes.push(current.runnerSeconds / 60); + for (const name of current.lastJobs) { + increment(summary.longPoleCount, name); + } +} + +export function summarize( + runs: readonly RunRecord[], +): Record { + const summaries: Record = { + pr: emptyClassSummary(), + queue: emptyClassSummary(), + merge_group: emptyClassSummary(), + push: emptyClassSummary(), + schedule: emptyClassSummary(), + dispatch: emptyClassSummary(), + }; + + for (const run of runs) { + summarizeRun(summaries[classify(run)], run); + } + + const result: Record = { + pr: summarizeClass(summaries.pr), + queue: summarizeClass(summaries.queue), + merge_group: summarizeClass(summaries.merge_group), + push: summarizeClass(summaries.push), + schedule: summarizeClass(summaries.schedule), + dispatch: summarizeClass(summaries.dispatch), + }; + return result; +} diff --git a/tools/ci-timing/index.test.ts b/tools/ci-timing/index.test.ts new file mode 100644 index 000000000..fc1b3a617 --- /dev/null +++ b/tools/ci-timing/index.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "bun:test"; +import { readJobs, readRuns } from "./index.ts"; + +function runItem(id: number) { + return { + id, + event: "pull_request", + head_branch: "feature", + conclusion: "success", + run_started_at: "2026-10-15T00:00:00Z", + updated_at: "2026-10-15T00:10:00Z", + }; +} + +function runPage(totalCount: number, ids: readonly number[]) { + return { total_count: totalCount, workflow_runs: ids.map(runItem) }; +} + +function jobItem(conclusion: string) { + return { + name: "e2e", + conclusion, + created_at: "2026-10-15T00:00:00Z", + started_at: "2026-10-15T00:00:05Z", + completed_at: "2026-10-15T00:05:00Z", + }; +} + +describe("readRuns", () => { + test("accepts a complete multi-page listing", () => { + const runs = readRuns([runPage(3, [1, 2]), runPage(3, [3])]); + expect(runs.map((run) => run.id)).toEqual([1, 2, 3]); + }); + + test("rejects a listing cut short of total_count", () => { + expect(() => readRuns([runPage(1200, [1, 2])])).toThrow( + "workflow runs listed 2 of 1200", + ); + }); +}); + +describe("readJobs", () => { + test("reads a timed-out job as a failure that ran", () => { + const [job] = readJobs([{ jobs: [jobItem("timed_out")] }], 7); + expect(job?.conclusion).toBe("failure"); + expect(job?.completedAt).toBeGreaterThan(job?.startedAt ?? 0); + }); + + test("rejects an unknown conclusion", () => { + expect(() => readJobs([{ jobs: [jobItem("exploded")] }], 7)).toThrow( + "unsupported conclusion: exploded", + ); + }); +}); diff --git a/tools/ci-timing/index.ts b/tools/ci-timing/index.ts new file mode 100755 index 000000000..80c302145 --- /dev/null +++ b/tools/ci-timing/index.ts @@ -0,0 +1,396 @@ +#!/usr/bin/env bun +import type { ClassSummary, JobRecord, Quantiles, RunRecord } from "./core.ts"; +import { summarize } from "./core.ts"; + +const RUN_CLASSES = [ + "pr", + "queue", + "merge_group", + "push", + "schedule", + "dispatch", +] as const; +const GH_REPOSITORY = "RigelBuild/compass"; +const GH_WORKFLOW = "ci.yml"; + +type Options = { since: string; until: string; json: boolean }; +type JsonObject = Record; + +function isObject(value: unknown): value is JsonObject { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function parseJson(text: string, label: string): unknown { + try { + return JSON.parse(text); + } catch (error) { + throw new Error(`${label} returned invalid JSON`, { cause: error }); + } +} + +function parseDate(value: string, option: string): string { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) { + throw new Error(`${option} must use YYYY-MM-DD`); + } + const timestamp = Date.parse(`${value}T00:00:00Z`); + if ( + !Number.isFinite(timestamp) || + new Date(timestamp).toISOString().slice(0, 10) !== value + ) { + throw new Error(`${option} is not a valid calendar date`); + } + return value; +} + +function setDateOption( + options: Options, + argument: string, + value: string, +): void { + if (argument === "--since") { + options.since = parseDate(value, argument); + } else { + options.until = parseDate(value, argument); + } +} + +function readOptionValue( + args: readonly string[], + index: number, + argument: string, +): string { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) { + throw new Error(`${argument} requires a YYYY-MM-DD value`); + } + return value; +} + +function parseOptions(args: readonly string[]): Options { + const options: Options = { since: "", until: "", json: false }; + let hasSince = false; + for (let index = 0; index < args.length; index += 1) { + const argument = args[index]; + if (argument === "--json") { + options.json = true; + } else if (argument === "--since" || argument === "--until") { + setDateOption(options, argument, readOptionValue(args, index, argument)); + hasSince ||= argument === "--since"; + index += 1; + } else { + throw new Error(`Unknown argument: ${String(argument)}`); + } + } + + if (!hasSince) { + throw new Error( + "Usage: bun tools/ci-timing/index.ts --since YYYY-MM-DD [--until YYYY-MM-DD] [--json]", + ); + } + if (options.until === "") { + options.until = new Date().toISOString().slice(0, 10); + } + if (options.since > options.until) { + throw new Error("--since must not be after --until"); + } + return options; +} + +async function ghApi(args: readonly string[], label: string): Promise { + const child = Bun.spawn( + ["gh", "api", "--paginate", "--slurp", "--method", "GET", ...args], + { + stdout: "pipe", + stderr: "pipe", + }, + ); + const [stdout, stderr, status] = await Promise.all([ + child.stdout.text(), + child.stderr.text(), + child.exited, + ]); + if (status !== 0) { + throw new Error(`gh api ${label} exited ${status}: ${stderr.trim()}`); + } + return parseJson(stdout, `gh api ${label}`); +} + +function pages(value: unknown, label: string): JsonObject[] { + const candidates = Array.isArray(value) ? value : [value]; + const result: JsonObject[] = []; + for (const candidate of candidates) { + if (!isObject(candidate)) { + throw new Error(`${label} page is not an object`); + } + result.push(candidate); + } + return result; +} + +function pageItems(page: JsonObject, key: string, label: string): JsonObject[] { + const items = page[key]; + if (!Array.isArray(items)) { + throw new Error(`${label} response has no ${key} array`); + } + const result: JsonObject[] = []; + for (const item of items) { + if (!isObject(item)) { + throw new Error(`${label} contains a non-object ${key} item`); + } + result.push(item); + } + return result; +} + +function requiredString( + object: JsonObject, + key: string, + label: string, +): string { + const value = object[key]; + if (typeof value !== "string") { + throw new Error(`${label} is missing string field ${key}`); + } + return value; +} + +function requiredNumber( + object: JsonObject, + key: string, + label: string, +): number { + const value = object[key]; + if (typeof value !== "number" || !Number.isFinite(value)) { + throw new Error(`${label} is missing numeric field ${key}`); + } + return value; +} + +function timestamp(value: unknown, label: string, optional: boolean): number { + if (optional && (value === null || value === undefined)) { + return 0; + } + if (typeof value !== "string") { + throw new Error(`${label} is not a timestamp`); + } + const milliseconds = Date.parse(value); + if (!Number.isFinite(milliseconds)) { + throw new Error(`${label} is not a valid timestamp`); + } + return milliseconds / 1000; +} + +function knownConclusion(value: unknown): RunRecord["conclusion"] | undefined { + if (value === "success" || value === "failure" || value === "cancelled") { + return value; + } + return undefined; +} + +export function readRuns(value: unknown): RunRecord[] { + const result: RunRecord[] = []; + let listed = 0; + let total = 0; + for (const page of pages(value, "workflow runs")) { + total = requiredNumber(page, "total_count", "workflow runs"); + listed += pageItems(page, "workflow_runs", "workflow runs").length; + } + // The runs search stops at 1000 results; a silent cut would skew every quantile. + if (listed < total) { + throw new Error( + `workflow runs listed ${listed} of ${total}; narrow --since/--until below 1000 runs`, + ); + } + for (const page of pages(value, "workflow runs")) { + for (const run of pageItems(page, "workflow_runs", "workflow runs")) { + const conclusion = knownConclusion(run.conclusion); + if (conclusion === undefined) { + continue; + } + result.push({ + id: requiredNumber(run, "id", "workflow run"), + event: requiredString(run, "event", "workflow run"), + headBranch: requiredString(run, "head_branch", "workflow run"), + conclusion, + runStartedAt: timestamp( + run.run_started_at, + "workflow run run_started_at", + true, + ), + updatedAt: timestamp(run.updated_at, "workflow run updated_at", false), + jobs: [], + }); + } + } + return result; +} + +// GitHub's extra job conclusions folded into the record's four: a timeout ran and failed, +// and a job left waiting on approval or superseded never ran. +const JOB_CONCLUSIONS: Readonly> = { + success: "success", + neutral: "success", + failure: "failure", + timed_out: "failure", + cancelled: "cancelled", + skipped: "skipped", + action_required: "skipped", + stale: "skipped", +}; + +export function readJobs(value: unknown, runId: number): RunRecord["jobs"] { + const result: RunRecord["jobs"] = []; + for (const page of pages(value, `jobs for run ${runId}`)) { + for (const job of pageItems(page, "jobs", `jobs for run ${runId}`)) { + const conclusion = + typeof job.conclusion === "string" + ? JOB_CONCLUSIONS[job.conclusion] + : undefined; + if (conclusion === undefined) { + throw new Error( + `Job in run ${runId} has unsupported conclusion: ${String(job.conclusion)}`, + ); + } + result.push({ + name: requiredString(job, "name", `job in run ${runId}`), + conclusion, + createdAt: timestamp( + job.created_at, + `job in run ${runId} created_at`, + false, + ), + startedAt: timestamp( + job.started_at, + `job in run ${runId} started_at`, + true, + ), + completedAt: timestamp( + job.completed_at, + `job in run ${runId} completed_at`, + true, + ), + }); + } + } + return result; +} + +async function mapRuns(runs: readonly RunRecord[]): Promise { + let cursor = 0; + const workers = Array.from({ length: Math.min(8, runs.length) }, async () => { + while (cursor < runs.length) { + const index = cursor; + cursor += 1; + const run = runs[index]; + if (run === undefined) { + throw new Error("Run cursor exceeded workflow run list"); + } + if (run.conclusion !== "cancelled") { + const value = await ghApi( + [ + `repos/${GH_REPOSITORY}/actions/runs/${run.id}/jobs`, + "--field", + "filter=latest", + "--field", + "per_page=100", + ], + `jobs for run ${run.id}`, + ); + run.jobs = readJobs(value, run.id); + } + } + }); + await Promise.all(workers); + return [...runs]; +} + +function fixed(value: number): string { + return value.toFixed(2); +} + +function markdownCell(value: string): string { + return value.replaceAll("|", "\\|").replaceAll("\n", " "); +} + +function metricRow(name: string, values: Quantiles, unit = "s"): string { + return `| ${name} | ${values.n} | ${fixed(values.p50)} ${unit} | ${fixed(values.p90)} ${unit} | ${fixed(values.p99)} ${unit} | ${fixed(values.max)} ${unit} |`; +} + +function renderMarkdown( + summary: Record<(typeof RUN_CLASSES)[number], ClassSummary>, +): string { + const lines: string[] = []; + for (const runClass of RUN_CLASSES) { + const current = summary[runClass]; + lines.push( + `## ${runClass}`, + "", + `Kept: ${current.kept}; cancelled: ${current.cancelled}.`, + "", + "| Metric | n | p50 | p90 | p99 | max |", + "| --- | ---: | ---: | ---: | ---: | ---: |", + metricRow("Wall", current.wall), + metricRow("Per-run max wait", current.maxWait), + metricRow("Runner minutes", current.runnerMinutes, "min"), + "", + "| Job | Skip rate | Long-pole share | Wait p50 / p90 / p99 / max (s) | Duration p50 / p90 / p99 / max (s) |", + "| --- | ---: | ---: | ---: | ---: |", + ); + const jobNames = [ + ...new Set([ + ...Object.keys(current.skipRate), + ...Object.keys(current.jobDuration), + ]), + ].sort(); + for (const name of jobNames) { + const waits = current.jobWait[name]; + const durations = current.jobDuration[name]; + const waitText = waits + ? `${fixed(waits.p50)} / ${fixed(waits.p90)} / ${fixed(waits.p99)} / ${fixed(waits.max)}` + : "—"; + const durationText = durations + ? `${fixed(durations.p50)} / ${fixed(durations.p90)} / ${fixed(durations.p99)} / ${fixed(durations.max)}` + : "—"; + lines.push( + `| ${markdownCell(name)} | ${((current.skipRate[name] ?? 0) * 100).toFixed(1)}% | ${((current.longPole[name] ?? 0) * 100).toFixed(1)}% | ${waitText} | ${durationText} |`, + ); + } + lines.push( + "", + `UTC hours with any job wait over 120 s: ${current.hoursOver120s.join(", ") || "none"}.`, + "", + ); + } + return lines.join("\n"); +} + +async function main(): Promise { + const options = parseOptions(process.argv.slice(2)); + const rawRuns = await ghApi( + [ + `repos/${GH_REPOSITORY}/actions/workflows/${GH_WORKFLOW}/runs`, + "--field", + `created=${options.since}..${options.until}`, + "--field", + "per_page=100", + ], + "workflow runs", + ); + const runs = await mapRuns(readRuns(rawRuns)); + const summary = summarize(runs); + if (options.json) { + console.log(JSON.stringify(summary, null, 2)); + } else { + console.log(renderMarkdown(summary)); + } +} + +if (import.meta.main) { + try { + await main(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(message); + process.exitCode = 1; + } +} diff --git a/tools/ci-timing/moon.yml b/tools/ci-timing/moon.yml new file mode 100644 index 000000000..8d3ae45ce --- /dev/null +++ b/tools/ci-timing/moon.yml @@ -0,0 +1,16 @@ +# yaml-language-server: $schema=https://moonrepo.dev/schemas/project.json +layer: 'tool' +language: 'typescript' +tags: ['bun', 'oss', 'ci-group.bun'] + +tasks: + typecheck: + command: 'bunx tsc --noEmit' + deps: ['install'] + inputs: ['*.ts', 'tsconfig.json', '/tsconfig.base.json', 'package.json', '/bun.lock', '/.github/workflows/ci.yml'] + test: + inputs: ['*.ts', 'tsconfig.json', '/tsconfig.base.json', 'package.json', '/bun.lock', '/.github/workflows/ci.yml'] + ci: + deps: ['typecheck', 'test'] + options: + cache: false diff --git a/tools/ci-timing/package.json b/tools/ci-timing/package.json new file mode 100644 index 000000000..6f414dcb3 --- /dev/null +++ b/tools/ci-timing/package.json @@ -0,0 +1,14 @@ +{ + "name": "@compass/ci-timing", + "private": true, + "type": "module", + "description": "Measures CI workflow timing by run class from GitHub Actions metadata.", + "module": "index.ts", + "bin": { + "ci-timing": "./index.ts" + }, + "devDependencies": { + "@types/bun": "catalog:", + "typescript": "catalog:" + } +} diff --git a/tools/ci-timing/tsconfig.json b/tools/ci-timing/tsconfig.json new file mode 100644 index 000000000..d40cc9e50 --- /dev/null +++ b/tools/ci-timing/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022"], + "moduleDetection": "force", + "allowJs": true, + "allowImportingTsExtensions": true, + "noUncheckedIndexedAccess": true, + "types": ["bun"] + } +}