From ac3956db6e4d8b4b4940dd699dbaa7921fd9388c Mon Sep 17 00:00:00 2001 From: Brian Genisio Date: Thu, 6 Aug 2026 11:44:37 -0400 Subject: [PATCH 1/2] ci(a11y): add PR checks with a shrink-only axe baseline There was no CI on pull requests. Gate unit tests, the build, and an axe scan in both color schemes on today's known violations so the remediation wave cannot silently regress. axe still cannot see A1. Co-authored-by: Cursor --- .github/workflows/pr.yml | 97 ++++++++ a11y-audits/tools/.gitignore | 1 + a11y-audits/tools/README.md | 29 ++- a11y-audits/tools/audit.mjs | 346 ++++++++++++++++++++-------- a11y-audits/tools/baseline.json | 46 ++++ a11y-audits/tools/baseline.mjs | 107 +++++++++ a11y-audits/tools/baseline.test.mjs | 64 +++++ a11y-audits/tools/package.json | 3 + 8 files changed, 600 insertions(+), 93 deletions(-) create mode 100644 .github/workflows/pr.yml create mode 100644 a11y-audits/tools/baseline.json create mode 100644 a11y-audits/tools/baseline.mjs create mode 100644 a11y-audits/tools/baseline.test.mjs diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000..d99b140 --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,97 @@ +# Pull-request checks for ChatCPT. +# +# Includes an axe-core accessibility gate against a checked-in shrink-only +# baseline (a11y-audits/tools/baseline.json). The baseline may only ever +# shrink — new rules or higher node counts fail the build. +# +# IMPORTANT: axe cannot detect live-region over-announcement (finding A1 in +# a11y-audits/8-5-26/audit.md). A green badge here is a floor, not WCAG +# conformance. Manual VoiceOver checks remain mandatory for live-region and +# focus work. + +name: PR + +on: + pull_request: + +jobs: + check: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Populate design system submodule + run: git submodule update --init + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Unit tests + run: npm test + + - name: Build frontend bundle + run: npm run build + + - name: Install accessibility audit tools + working-directory: a11y-audits/tools + run: npm ci + + - name: Baseline helper unit tests + working-directory: a11y-audits/tools + run: npm test + + - name: Install Playwright Chromium + working-directory: a11y-audits/tools + run: npx playwright install chromium --with-deps + + - name: Start app server + # No Octavus credentials in CI — audit.mjs stubs /api/* when A11Y_CI=1. + # A placeholder AGENT_ID keeps the server from warning; the stub never + # reaches the create-session path. + run: | + PORT=3100 AGENT_TARGET=dev OCTAVUS_AGENT_ID=ci-placeholder \ + node server.js > /tmp/chatcpt-ci-server.log 2>&1 & + echo $! > /tmp/chatcpt-ci-server.pid + for i in $(seq 1 30); do + if curl -sf http://127.0.0.1:3100/ >/dev/null; then + echo "Server ready" + exit 0 + fi + sleep 1 + done + echo "Server failed to start" >&2 + cat /tmp/chatcpt-ci-server.log >&2 + exit 1 + + - name: Axe scan (light + dark) against shrink-only baseline + working-directory: a11y-audits/tools + env: + A11Y_CI: '1' + A11Y_BROWSER_CHANNEL: bundled + A11Y_BASE_URL: http://127.0.0.1:3100 + A11Y_OUT: a11y-out-ci + A11Y_BASELINE: ${{ github.workspace }}/a11y-audits/tools/baseline.json + run: npm run audit + + - name: Upload axe report + if: always() + uses: actions/upload-artifact@v4 + with: + name: axe-report + path: a11y-audits/tools/a11y-out-ci/ + if-no-files-found: ignore + + - name: Stop app server + if: always() + run: | + if [ -f /tmp/chatcpt-ci-server.pid ]; then + kill "$(cat /tmp/chatcpt-ci-server.pid)" || true + fi diff --git a/a11y-audits/tools/.gitignore b/a11y-audits/tools/.gitignore index a3041af..3a608c4 100644 --- a/a11y-audits/tools/.gitignore +++ b/a11y-audits/tools/.gitignore @@ -1,2 +1,3 @@ node_modules/ a11y-out/ +a11y-out-ci/ diff --git a/a11y-audits/tools/README.md b/a11y-audits/tools/README.md index ee89a2a..3b7382a 100644 --- a/a11y-audits/tools/README.md +++ b/a11y-audits/tools/README.md @@ -2,7 +2,7 @@ The Playwright + axe-core scripts that produced the evidence in [`../8-5-26/audit.md`](../8-5-26/audit.md). Kept in the repository so findings can be -re-measured after each fix, and because `audit.mjs` is the intended starting point for the +re-measured after each fix, and because `audit.mjs` drives the CI accessibility gate ([#36](https://github.com/CodeSignal/learn_cosmo-chat/issues/36)). These have their own dependency tree. They are **not** part of the application build and are @@ -37,10 +37,37 @@ npm run audit | `A11Y_BASE_URL` | `http://localhost:3100` | Where the app is running | | `A11Y_OUT` | `a11y-out` | Directory for screenshots and `report.json` | | `A11Y_BROWSER_CHANNEL` | `chrome` | Set to `bundled` to use Playwright's own Chromium | +| `A11Y_CI` | unset | `1` stubs `/api/*`, skips the live-agent send flow, and enables the baseline gate | +| `A11Y_BASELINE` | `./baseline.json` when `A11Y_CI=1` | Path to the shrink-only axe baseline | +| `A11Y_UPDATE_BASELINE` | unset | `1` rewrites the baseline from the current run | `A11Y_BROWSER_CHANNEL` defaults to system Chrome because Playwright's bundled Chromium was missing on the audit machine. In CI, run `playwright install chromium` and set it to `bundled`. +### CI gate + +`.github/workflows/pr.yml` runs: + +```bash +npm run audit:ci +``` + +That compares the axe results for empty / settings / settings-with-dropdown in both color +schemes against [`baseline.json`](./baseline.json). The baseline is **shrink-only**: a new +rule or a higher node count fails the build; a fix that removes violations should update +`baseline.json` in the same PR so the floor ratchets down. + +```bash +# after a fix that clears axe violations: +npm run audit:update-baseline +``` + +CI does not exercise streaming or populated-conversation states (no live agent). Those +axe results matched the empty-state shell in the original audit, and the settings / +dropdown states carry the distinctive rules (`aria-input-field-name`, the extra +`region` nodes, dark-mode contrast). Re-run the full `npm run audit` locally when a +fix needs the live-agent states. + ## What each script does **`audit.mjs`** — the main sweep. For each of light and dark mode it walks the app through five diff --git a/a11y-audits/tools/audit.mjs b/a11y-audits/tools/audit.mjs index 3e3f3be..a7ab6d5 100644 --- a/a11y-audits/tools/audit.mjs +++ b/a11y-audits/tools/audit.mjs @@ -1,11 +1,24 @@ import { chromium } from 'playwright'; import { AxeBuilder } from '@axe-core/playwright'; import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { compare, fingerprint, readBaseline, writeBaseline } from './baseline.mjs'; const BASE = process.env.A11Y_BASE_URL ?? 'http://localhost:3100'; const OUT = process.env.A11Y_OUT ?? 'a11y-out'; fs.mkdirSync(OUT, { recursive: true }); +// CI mode skips the live-agent send flow (no Octavus in Actions) and stubs the +// API so the app boots without credentials. It still covers empty + settings + +// settings-with-dropdown in both color schemes — the states that carry today's +// known axe baseline. +const CI = process.env.A11Y_CI === '1'; +const BASELINE_PATH = process.env.A11Y_BASELINE; +const UPDATE_BASELINE = process.env.A11Y_UPDATE_BASELINE === '1'; +const TOOLS_DIR = path.dirname(fileURLToPath(import.meta.url)); +const DEFAULT_BASELINE = path.join(TOOLS_DIR, 'baseline.json'); + const TAGS = ['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa', 'wcag22aa', 'best-practice']; function srgb(c) { @@ -181,6 +194,88 @@ async function scan(page, label) { }; } +async function maybeScreenshot(page, name) { + if (CI) return; + await page.screenshot({ path: `${OUT}/${name}`, fullPage: name.includes('empty') ? false : undefined }); +} + +/** + * Stub the JSON API so the app boots without Octavus. Used only in CI mode — + * the full local audit still talks to a real server/agent. + */ +async function installCiApiStub(page) { + const config = { + temperature: 0.7, + allowCustomInstructions: true, + customInstructions: '', + hideSettings: false, + hideHistory: false, + hideFileUpload: false, + hidePromptControls: false, + hideModelSettings: false, + model: 'anthropic/claude-sonnet-4-6', + allowedModels: ['anthropic/claude-sonnet-4-6'], + thinking: 'medium', + showReasoning: true, + }; + const models = ['anthropic/claude-sonnet-4-6']; + const capabilities = { + 'anthropic/claude-sonnet-4-6': { supportsThinking: true }, + }; + // Two rows so nested-interactive matches the shape the audit recorded + // (active session + another history item), not a single-item sidebar. + const sessions = [ + { + session_id: 'session-1', + title: 'New conversation', + created_at: '2026-08-05T00:00:00.000Z', + updated_at: '2026-08-05T12:00:00.000Z', + }, + { + session_id: 'session-0', + title: 'Earlier chat', + created_at: '2026-08-04T00:00:00.000Z', + updated_at: '2026-08-04T00:00:00.000Z', + }, + ]; + + await page.route('**/api/**', async (route) => { + const req = route.request(); + const url = new URL(req.url()); + const pathname = url.pathname; + const method = req.method(); + const json = (data, status = 200) => + route.fulfill({ + status, + contentType: 'application/json', + body: JSON.stringify(data), + }); + + if (pathname === '/api/session' && method === 'GET') { + return json({ sessionId: 'session-1', messages: [] }); + } + if (pathname === '/api/sessions' && method === 'GET') { + return json({ sessions }); + } + if (pathname === '/api/sessions' && method === 'POST') { + return json({ sessionId: 'session-2', messages: [] }); + } + if (pathname === '/api/config' && method === 'GET') { + return json(config); + } + if (pathname === '/api/models' && method === 'GET') { + return json({ models, capabilities }); + } + if (pathname === '/api/session/save' && method === 'POST') { + return json({ ok: true }); + } + if (pathname === '/api/config/custom-instructions' && method === 'POST') { + return json({ ok: true }); + } + return json({ ok: true }); + }); +} + // Defaults to system Chrome because Playwright's bundled Chromium was missing // on the audit machine. In CI, run `playwright install chromium` and set // A11Y_BROWSER_CHANNEL=bundled. @@ -196,54 +291,73 @@ async function run(scheme) { const page = await ctx.newPage(); const report = { scheme, scans: [], tabOrder: null, contrast: [], notes: [] }; + if (CI) { + await installCiApiStub(page); + report.notes.push('CI mode: API stubbed; streaming/with-messages states skipped (no live agent)'); + } + await page.goto(BASE, { waitUntil: 'networkidle' }); await page.waitForTimeout(1500); + if (CI) { + // Fail fast if the stub/boot path broke — an axe scan of the boot-error + // screen would silently drift away from the real baseline. + const visible = await page.locator('.chat-app').evaluate((el) => getComputedStyle(el).visibility); + if (visible === 'hidden') { + const bootText = await page.locator('#bootError').textContent().catch(() => ''); + throw new Error(`App did not boot in CI mode (chat-app still hidden). bootError: ${bootText}`); + } + } + const boot = await page.locator('#bootError').isVisible().catch(() => false); report.notes.push(`bootError visible: ${boot}`); // 1. Empty state report.scans.push(await scan(page, 'empty-state')); - await page.screenshot({ path: `${OUT}/${scheme}-01-empty.png`, fullPage: false }); - report.tabOrder = await walkTabOrder(page); - report.contrast.push({ state: 'empty', pairs: await contrastSample(page) }); + await maybeScreenshot(page, `${scheme}-01-empty.png`); + if (!CI) { + report.tabOrder = await walkTabOrder(page); + report.contrast.push({ state: 'empty', pairs: await contrastSample(page) }); + } - // 2. Send a prompt to get real message content rendered - try { - await page.fill('#promptInput', 'In two short sentences, what is a prompt? Then show a tiny python code block.'); - await page.click('#sendBtn'); - await page.waitForTimeout(1200); - report.scans.push(await scan(page, 'streaming')); - await page.screenshot({ path: `${OUT}/${scheme}-02-streaming.png` }); - - // aria-live sanity: how much text sits inside the live region while streaming - report.notes.push( - 'live region text length while streaming: ' + - (await page.evaluate(() => (document.querySelector('#chatHistory')?.textContent || '').length)), - ); - - await page.waitForFunction( - () => !document.querySelector('.message--ai--streaming'), - null, - { timeout: 60000 }, - ).catch(() => report.notes.push('stream did not finish within 60s')); - await page.waitForTimeout(800); - report.scans.push(await scan(page, 'with-messages')); - await page.screenshot({ path: `${OUT}/${scheme}-03-messages.png` }); - report.contrast.push({ state: 'messages', pairs: await contrastSample(page) }); - - // focus-destruction test: focus a copy button, force a re-render, see where focus lands - const focusTest = await page.evaluate(() => { - const btn = document.querySelector('.message__hover-btn, .code-block__copy'); - if (!btn) return 'no action button found'; - btn.focus(); - const before = document.activeElement?.className; - document.querySelector('#chatHistory .messages')?.dispatchEvent(new Event('x')); - return `focused: ${before}`; - }); - report.notes.push(`hover-action focus test: ${focusTest}`); - } catch (e) { - report.notes.push(`send flow failed: ${e.message.slice(0, 160)}`); + // 2. Send a prompt to get real message content rendered (full audit only) + if (!CI) { + try { + await page.fill('#promptInput', 'In two short sentences, what is a prompt? Then show a tiny python code block.'); + await page.click('#sendBtn'); + await page.waitForTimeout(1200); + report.scans.push(await scan(page, 'streaming')); + await maybeScreenshot(page, `${scheme}-02-streaming.png`); + + // aria-live sanity: how much text sits inside the live region while streaming + report.notes.push( + 'live region text length while streaming: ' + + (await page.evaluate(() => (document.querySelector('#chatHistory')?.textContent || '').length)), + ); + + await page.waitForFunction( + () => !document.querySelector('.message--ai--streaming'), + null, + { timeout: 60000 }, + ).catch(() => report.notes.push('stream did not finish within 60s')); + await page.waitForTimeout(800); + report.scans.push(await scan(page, 'with-messages')); + await maybeScreenshot(page, `${scheme}-03-messages.png`); + report.contrast.push({ state: 'messages', pairs: await contrastSample(page) }); + + // focus-destruction test: focus a copy button, force a re-render, see where focus lands + const focusTest = await page.evaluate(() => { + const btn = document.querySelector('.message__hover-btn, .code-block__copy'); + if (!btn) return 'no action button found'; + btn.focus(); + const before = document.activeElement?.className; + document.querySelector('#chatHistory .messages')?.dispatchEvent(new Event('x')); + return `focused: ${before}`; + }); + report.notes.push(`hover-action focus test: ${focusTest}`); + } catch (e) { + report.notes.push(`send flow failed: ${e.message.slice(0, 160)}`); + } } // 3. Settings modal @@ -254,48 +368,50 @@ async function run(scheme) { report.notes.push(`settings modal open: ${modalOpen > 0}`); if (modalOpen) { report.scans.push(await scan(page, 'settings-modal')); - await page.screenshot({ path: `${OUT}/${scheme}-04-settings.png` }); - report.contrast.push({ state: 'settings', pairs: await contrastSample(page) }); + await maybeScreenshot(page, `${scheme}-04-settings.png`); + if (!CI) { + report.contrast.push({ state: 'settings', pairs: await contrastSample(page) }); - // Does Tab escape the dialog? - const trap = await page.evaluate(async () => { - const dialog = document.querySelector('.modal-overlay.open'); - const inside = []; + // Does Tab escape the dialog? + const trap = await page.evaluate(async () => { + const dialog = document.querySelector('.modal-overlay.open'); + const inside = []; + for (let i = 0; i < 14; i++) { + const el = document.activeElement; + inside.push({ + in: dialog.contains(el), + tag: el?.tagName.toLowerCase(), + cls: (typeof el?.className === 'string' ? el.className : '').slice(0, 50), + }); + // synthetic Tab is unreliable in evaluate; rely on playwright below + break; + } + return inside; + }); + const trapWalk = []; for (let i = 0; i < 14; i++) { - const el = document.activeElement; - inside.push({ - in: dialog.contains(el), - tag: el?.tagName.toLowerCase(), - cls: (typeof el?.className === 'string' ? el.className : '').slice(0, 50), - }); - // synthetic Tab is unreliable in evaluate; rely on playwright below - break; + await page.keyboard.press('Tab'); + trapWalk.push(await page.evaluate(() => { + const d = document.querySelector('.modal-overlay.open'); + const el = document.activeElement; + return { + insideDialog: d ? d.contains(el) : null, + tag: el?.tagName.toLowerCase(), + id: el?.id || null, + cls: (typeof el?.className === 'string' ? el.className : '').slice(0, 45), + }; + })); } - return inside; - }); - const trapWalk = []; - for (let i = 0; i < 14; i++) { - await page.keyboard.press('Tab'); - trapWalk.push(await page.evaluate(() => { - const d = document.querySelector('.modal-overlay.open'); - const el = document.activeElement; - return { - insideDialog: d ? d.contains(el) : null, - tag: el?.tagName.toLowerCase(), - id: el?.id || null, - cls: (typeof el?.className === 'string' ? el.className : '').slice(0, 45), - }; + report.notes.push(`modal initial focus: ${JSON.stringify(trap)}`); + report.modalTabWalk = trapWalk; + + // duplicate id check + report.notes.push(await page.evaluate(() => { + const ids = [...document.querySelectorAll('[id]')].map((e) => e.id); + const dupes = ids.filter((v, i) => ids.indexOf(v) !== i); + return `duplicate ids: ${JSON.stringify([...new Set(dupes)])}`; })); } - report.notes.push(`modal initial focus: ${JSON.stringify(trap)}`); - report.modalTabWalk = trapWalk; - - // duplicate id check - report.notes.push(await page.evaluate(() => { - const ids = [...document.querySelectorAll('[id]')].map((e) => e.id); - const dupes = ids.filter((v, i) => ids.indexOf(v) !== i); - return `duplicate ids: ${JSON.stringify([...new Set(dupes)])}`; - })); // open the Thinking dropdown, check portal + aria await page.evaluate(() => document.querySelector('#thinkingDropdownEl .dropdown-toggle')?.click()); @@ -307,24 +423,26 @@ async function run(scheme) { return `thinking menu portaled: ${!!menu}; menu inside dialog: ${menu && dialog ? dialog.contains(menu) : 'n/a'}; focus after open: ${active?.tagName}.${typeof active?.className === 'string' ? active.className.slice(0, 30) : ''}; options with aria-selected: ${document.querySelectorAll('.dropdown-menu-item[aria-selected]').length}/${document.querySelectorAll('.dropdown-menu-item').length}`; })); report.scans.push(await scan(page, 'settings-modal-dropdown-open')); - await page.screenshot({ path: `${OUT}/${scheme}-05-dropdown.png` }); + await maybeScreenshot(page, `${scheme}-05-dropdown.png`); } } catch (e) { report.notes.push(`settings flow failed: ${e.message.slice(0, 160)}`); } - // 4. Reflow at 320px - try { - await page.setViewportSize({ width: 320, height: 720 }); - await page.waitForTimeout(500); - const overflow = await page.evaluate(() => ({ - docScrollW: document.documentElement.scrollWidth, - clientW: document.documentElement.clientWidth, - })); - report.notes.push(`320px reflow: ${JSON.stringify(overflow)}`); - await page.screenshot({ path: `${OUT}/${scheme}-06-320px.png` }); - } catch (e) { - report.notes.push(`reflow failed: ${e.message.slice(0, 120)}`); + // 4. Reflow at 320px (full audit only) + if (!CI) { + try { + await page.setViewportSize({ width: 320, height: 720 }); + await page.waitForTimeout(500); + const overflow = await page.evaluate(() => ({ + docScrollW: document.documentElement.scrollWidth, + clientW: document.documentElement.clientWidth, + })); + report.notes.push(`320px reflow: ${JSON.stringify(overflow)}`); + await maybeScreenshot(page, `${scheme}-06-320px.png`); + } catch (e) { + report.notes.push(`reflow failed: ${e.message.slice(0, 120)}`); + } } await browser.close(); @@ -332,7 +450,8 @@ async function run(scheme) { } // Isolated design-system dropdown page (the app only exposes one model, so the -// model dropdown never renders in-app). +// model dropdown never renders in-app). Skipped in CI — the test page's landmark +// noise would dominate the baseline without guarding the app. async function runDsDropdown(scheme) { const browser = await chromium.launch(LAUNCH); const ctx = await browser.newContext({ colorScheme: scheme, viewport: { width: 1200, height: 900 } }); @@ -373,7 +492,7 @@ async function runDsDropdown(scheme) { return (el?.tagName || '') + '.' + (typeof el?.className === 'string' ? el.className : '') + ' | isBody=' + (el === document.body); })); out.axeOpen = await scan(page, 'ds-dropdown-open'); - await page.screenshot({ path: `${OUT}/${scheme}-07-ds-dropdown.png` }); + await maybeScreenshot(page, `${scheme}-07-ds-dropdown.png`); } catch (e) { out.notes.push(`ds dropdown failed: ${e.message.slice(0, 160)}`); } @@ -384,7 +503,9 @@ async function runDsDropdown(scheme) { const all = {}; for (const scheme of ['light', 'dark']) { all[scheme] = await run(scheme); - all[`${scheme}-ds-dropdown`] = await runDsDropdown(scheme); + if (!CI) { + all[`${scheme}-ds-dropdown`] = await runDsDropdown(scheme); + } } // Post-process contrast @@ -434,3 +555,44 @@ for (const [k, v] of Object.entries(all)) { } } } + +// ── Baseline gate ───────────────────────────────────────────── +const actual = fingerprint(all); +const baselineFile = BASELINE_PATH || (CI || UPDATE_BASELINE ? DEFAULT_BASELINE : null); + +if (UPDATE_BASELINE) { + writeBaseline(baselineFile, actual); + console.log(`\nUpdated shrink-only baseline at ${baselineFile}`); + console.log(JSON.stringify(actual, null, 2)); +} else if (baselineFile) { + if (!fs.existsSync(baselineFile)) { + console.error(`\nBaseline file not found: ${baselineFile}`); + console.error('Capture one with A11Y_UPDATE_BASELINE=1'); + process.exit(1); + } + const baseline = readBaseline(baselineFile); + const result = compare(actual, baseline.scans || baseline); + if (result.improvements.length) { + console.log('\nBaseline improvements (update baseline.json in this PR):'); + result.improvements.forEach((line) => console.log(` ✓ ${line}`)); + } + if (result.missingScans.length) { + console.error('\nMissing scans vs baseline (audit did not cover expected states):'); + result.missingScans.forEach((line) => console.error(` ✗ ${line}`)); + } + if (result.regressions.length) { + console.error('\nAxe baseline regressions:'); + result.regressions.forEach((line) => console.error(` ✗ ${line}`)); + } + if (!result.ok) { + console.error('\nAccessibility CI gate failed. The baseline is shrink-only.'); + console.error('Note: axe cannot detect live-region over-announcement (A1).'); + process.exit(1); + } + if (result.improvements.length) { + console.log('\nReminder: commit an updated baseline.json so the floor ratchets down.'); + } + console.log('\nAxe baseline gate passed (shrink-only).'); + console.log('Note: axe cannot detect live-region over-announcement (A1) — a green gate is not conformance.'); +} + diff --git a/a11y-audits/tools/baseline.json b/a11y-audits/tools/baseline.json new file mode 100644 index 0000000..35f1a73 --- /dev/null +++ b/a11y-audits/tools/baseline.json @@ -0,0 +1,46 @@ +{ + "$comment": "Shrink-only axe baseline for PR CI. Node counts may decrease; increases or new rules fail the build. axe cannot detect live-region over-announcement (A1) — a green gate is not conformance.", + "generated": "2026-08-06", + "scans": { + "dark/empty-state": { + "aria-allowed-role": 1, + "color-contrast": 3, + "landmark-one-main": 1, + "nested-interactive": 2, + "region": 1 + }, + "dark/settings-modal": { + "aria-allowed-role": 1, + "color-contrast": 4, + "nested-interactive": 3, + "region": 1 + }, + "dark/settings-modal-dropdown-open": { + "aria-allowed-role": 1, + "aria-input-field-name": 1, + "color-contrast": 4, + "nested-interactive": 3, + "region": 6 + }, + "light/empty-state": { + "aria-allowed-role": 1, + "color-contrast": 3, + "landmark-one-main": 1, + "nested-interactive": 2, + "region": 1 + }, + "light/settings-modal": { + "aria-allowed-role": 1, + "color-contrast": 3, + "nested-interactive": 3, + "region": 1 + }, + "light/settings-modal-dropdown-open": { + "aria-allowed-role": 1, + "aria-input-field-name": 1, + "color-contrast": 3, + "nested-interactive": 3, + "region": 6 + } + } +} diff --git a/a11y-audits/tools/baseline.mjs b/a11y-audits/tools/baseline.mjs new file mode 100644 index 0000000..672bf70 --- /dev/null +++ b/a11y-audits/tools/baseline.mjs @@ -0,0 +1,107 @@ +/** + * Shrink-only axe baseline helpers for the PR CI gate. + * + * The baseline maps "scheme/state" → { ruleId: nodeCount }. A run fails when a + * rule appears that is not in the baseline, or when a rule's node count grows. + * Counts may decrease freely; that is how fix PRs retire violations. + * + * axe cannot detect live-region over-announcement (finding A1). A green compare + * is a floor, not WCAG conformance. + */ + +import fs from 'node:fs'; + +/** Flatten an audit report into { "light/empty-state": { rule: count }, ... }. */ +export function fingerprint(report) { + const out = {}; + for (const [key, value] of Object.entries(report)) { + if (!value || typeof value !== 'object') continue; + + for (const s of value.scans || []) { + out[`${key}/${s.label}`] = countsFromViolations(s.violations); + } + if (value.axe) { + out[`${key}/${value.axe.label}`] = countsFromViolations(value.axe.violations); + } + if (value.axeOpen) { + out[`${key}/${value.axeOpen.label}`] = countsFromViolations(value.axeOpen.violations); + } + } + return sortNested(out); +} + +function countsFromViolations(violations = []) { + const counts = {}; + for (const v of violations) { + counts[v.id] = v.total ?? v.nodes?.length ?? 0; + } + return counts; +} + +function sortNested(obj) { + const out = {}; + for (const key of Object.keys(obj).sort()) { + const inner = obj[key]; + out[key] = Object.fromEntries(Object.entries(inner).sort(([a], [b]) => a.localeCompare(b))); + } + return out; +} + +export function readBaseline(path) { + return JSON.parse(fs.readFileSync(path, 'utf8')); +} + +export function writeBaseline(path, scans) { + const doc = { + $comment: + 'Shrink-only axe baseline for PR CI. Node counts may decrease; increases or new rules fail the build. axe cannot detect live-region over-announcement (A1) — a green gate is not conformance.', + generated: new Date().toISOString().slice(0, 10), + scans, + }; + fs.writeFileSync(path, `${JSON.stringify(doc, null, 2)}\n`); +} + +/** + * Compare an actual fingerprint against a checked-in baseline. + * @returns {{ ok: boolean, regressions: string[], improvements: string[], missingScans: string[] }} + */ +export function compare(actual, baselineScans) { + const regressions = []; + const improvements = []; + const missingScans = []; + + for (const scanKey of Object.keys(baselineScans).sort()) { + if (!(scanKey in actual)) { + missingScans.push(scanKey); + continue; + } + const expected = baselineScans[scanKey]; + const got = actual[scanKey]; + + for (const rule of new Set([...Object.keys(expected), ...Object.keys(got)])) { + const before = expected[rule] ?? 0; + const after = got[rule] ?? 0; + if (after > before) { + regressions.push(`${scanKey}: ${rule} ${before} → ${after}`); + } else if (after < before) { + improvements.push(`${scanKey}: ${rule} ${before} → ${after}`); + } + } + } + + // A scan key that exists only in the actual run is fine (coverage grew), but + // any violations there are by definition new and must fail. + for (const scanKey of Object.keys(actual).sort()) { + if (scanKey in baselineScans) continue; + for (const [rule, count] of Object.entries(actual[scanKey])) { + if (count > 0) regressions.push(`${scanKey}: ${rule} (new scan) 0 → ${count}`); + } + } + + return { + ok: regressions.length === 0 && missingScans.length === 0, + regressions, + improvements, + missingScans, + }; +} diff --git a/a11y-audits/tools/baseline.test.mjs b/a11y-audits/tools/baseline.test.mjs new file mode 100644 index 0000000..2d16563 --- /dev/null +++ b/a11y-audits/tools/baseline.test.mjs @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { compare, fingerprint } from './baseline.mjs'; + +describe('fingerprint', () => { + it('flattens scheme/state rule counts', () => { + const fp = fingerprint({ + light: { + scans: [ + { + label: 'empty-state', + violations: [ + { id: 'color-contrast', total: 3 }, + { id: 'region', total: 1 }, + ], + }, + ], + }, + }); + assert.deepEqual(fp, { + 'light/empty-state': { 'color-contrast': 3, region: 1 }, + }); + }); +}); + +describe('compare', () => { + const baseline = { + 'light/empty-state': { 'color-contrast': 3, region: 1 }, + }; + + it('passes when counts match', () => { + const result = compare(baseline, baseline); + assert.equal(result.ok, true); + assert.deepEqual(result.regressions, []); + }); + + it('passes when counts shrink', () => { + const actual = { 'light/empty-state': { 'color-contrast': 2 } }; + const result = compare(actual, baseline); + assert.equal(result.ok, true); + assert.deepEqual(result.improvements, [ + 'light/empty-state: color-contrast 3 → 2', + 'light/empty-state: region 1 → 0', + ]); + }); + + it('fails when a count grows or a new rule appears', () => { + const actual = { + 'light/empty-state': { 'color-contrast': 4, 'nested-interactive': 1, region: 1 }, + }; + const result = compare(actual, baseline); + assert.equal(result.ok, false); + assert.deepEqual(result.regressions, [ + 'light/empty-state: color-contrast 3 → 4', + 'light/empty-state: nested-interactive 0 → 1', + ]); + }); + + it('fails when a baseline scan is missing from the run', () => { + const result = compare({}, baseline); + assert.equal(result.ok, false); + assert.deepEqual(result.missingScans, ['light/empty-state']); + }); +}); diff --git a/a11y-audits/tools/package.json b/a11y-audits/tools/package.json index 98ae000..97a452b 100644 --- a/a11y-audits/tools/package.json +++ b/a11y-audits/tools/package.json @@ -7,6 +7,9 @@ "license": "ISC", "scripts": { "audit": "node audit.mjs", + "audit:ci": "A11Y_CI=1 A11Y_BROWSER_CHANNEL=bundled A11Y_BASELINE=./baseline.json node audit.mjs", + "audit:update-baseline": "A11Y_CI=1 A11Y_UPDATE_BASELINE=1 node audit.mjs", + "test": "node --test baseline.test.mjs", "verify": "node verify.mjs", "verify:sessions": "node verify2.mjs" }, From 5eabe8976cf0244278485ce0d683cab5e469a4b0 Mon Sep 17 00:00:00 2001 From: Brian Genisio Date: Thu, 6 Aug 2026 12:07:25 -0400 Subject: [PATCH 2/2] ci(a11y): retrigger PR checks after Actions outage Co-authored-by: Cursor