diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000..d99b140 --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,97 @@ +# Pull-request checks for ChatCPT. +# +# Includes an axe-core accessibility gate against a checked-in shrink-only +# baseline (a11y-audits/tools/baseline.json). The baseline may only ever +# shrink — new rules or higher node counts fail the build. +# +# IMPORTANT: axe cannot detect live-region over-announcement (finding A1 in +# a11y-audits/8-5-26/audit.md). A green badge here is a floor, not WCAG +# conformance. Manual VoiceOver checks remain mandatory for live-region and +# focus work. + +name: PR + +on: + pull_request: + +jobs: + check: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Populate design system submodule + run: git submodule update --init + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Unit tests + run: npm test + + - name: Build frontend bundle + run: npm run build + + - name: Install accessibility audit tools + working-directory: a11y-audits/tools + run: npm ci + + - name: Baseline helper unit tests + working-directory: a11y-audits/tools + run: npm test + + - name: Install Playwright Chromium + working-directory: a11y-audits/tools + run: npx playwright install chromium --with-deps + + - name: Start app server + # No Octavus credentials in CI — audit.mjs stubs /api/* when A11Y_CI=1. + # A placeholder AGENT_ID keeps the server from warning; the stub never + # reaches the create-session path. + run: | + PORT=3100 AGENT_TARGET=dev OCTAVUS_AGENT_ID=ci-placeholder \ + node server.js > /tmp/chatcpt-ci-server.log 2>&1 & + echo $! > /tmp/chatcpt-ci-server.pid + for i in $(seq 1 30); do + if curl -sf http://127.0.0.1:3100/ >/dev/null; then + echo "Server ready" + exit 0 + fi + sleep 1 + done + echo "Server failed to start" >&2 + cat /tmp/chatcpt-ci-server.log >&2 + exit 1 + + - name: Axe scan (light + dark) against shrink-only baseline + working-directory: a11y-audits/tools + env: + A11Y_CI: '1' + A11Y_BROWSER_CHANNEL: bundled + A11Y_BASE_URL: http://127.0.0.1:3100 + A11Y_OUT: a11y-out-ci + A11Y_BASELINE: ${{ github.workspace }}/a11y-audits/tools/baseline.json + run: npm run audit + + - name: Upload axe report + if: always() + uses: actions/upload-artifact@v4 + with: + name: axe-report + path: a11y-audits/tools/a11y-out-ci/ + if-no-files-found: ignore + + - name: Stop app server + if: always() + run: | + if [ -f /tmp/chatcpt-ci-server.pid ]; then + kill "$(cat /tmp/chatcpt-ci-server.pid)" || true + fi diff --git a/a11y-audits/tools/.gitignore b/a11y-audits/tools/.gitignore index a3041af..3a608c4 100644 --- a/a11y-audits/tools/.gitignore +++ b/a11y-audits/tools/.gitignore @@ -1,2 +1,3 @@ node_modules/ a11y-out/ +a11y-out-ci/ diff --git a/a11y-audits/tools/README.md b/a11y-audits/tools/README.md index ee89a2a..3b7382a 100644 --- a/a11y-audits/tools/README.md +++ b/a11y-audits/tools/README.md @@ -2,7 +2,7 @@ The Playwright + axe-core scripts that produced the evidence in [`../8-5-26/audit.md`](../8-5-26/audit.md). Kept in the repository so findings can be -re-measured after each fix, and because `audit.mjs` is the intended starting point for the +re-measured after each fix, and because `audit.mjs` drives the CI accessibility gate ([#36](https://github.com/CodeSignal/learn_cosmo-chat/issues/36)). These have their own dependency tree. They are **not** part of the application build and are @@ -37,10 +37,37 @@ npm run audit | `A11Y_BASE_URL` | `http://localhost:3100` | Where the app is running | | `A11Y_OUT` | `a11y-out` | Directory for screenshots and `report.json` | | `A11Y_BROWSER_CHANNEL` | `chrome` | Set to `bundled` to use Playwright's own Chromium | +| `A11Y_CI` | unset | `1` stubs `/api/*`, skips the live-agent send flow, and enables the baseline gate | +| `A11Y_BASELINE` | `./baseline.json` when `A11Y_CI=1` | Path to the shrink-only axe baseline | +| `A11Y_UPDATE_BASELINE` | unset | `1` rewrites the baseline from the current run | `A11Y_BROWSER_CHANNEL` defaults to system Chrome because Playwright's bundled Chromium was missing on the audit machine. In CI, run `playwright install chromium` and set it to `bundled`. +### CI gate + +`.github/workflows/pr.yml` runs: + +```bash +npm run audit:ci +``` + +That compares the axe results for empty / settings / settings-with-dropdown in both color +schemes against [`baseline.json`](./baseline.json). The baseline is **shrink-only**: a new +rule or a higher node count fails the build; a fix that removes violations should update +`baseline.json` in the same PR so the floor ratchets down. + +```bash +# after a fix that clears axe violations: +npm run audit:update-baseline +``` + +CI does not exercise streaming or populated-conversation states (no live agent). Those +axe results matched the empty-state shell in the original audit, and the settings / +dropdown states carry the distinctive rules (`aria-input-field-name`, the extra +`region` nodes, dark-mode contrast). Re-run the full `npm run audit` locally when a +fix needs the live-agent states. + ## What each script does **`audit.mjs`** — the main sweep. For each of light and dark mode it walks the app through five diff --git a/a11y-audits/tools/audit.mjs b/a11y-audits/tools/audit.mjs index 3e3f3be..a7ab6d5 100644 --- a/a11y-audits/tools/audit.mjs +++ b/a11y-audits/tools/audit.mjs @@ -1,11 +1,24 @@ import { chromium } from 'playwright'; import { AxeBuilder } from '@axe-core/playwright'; import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { compare, fingerprint, readBaseline, writeBaseline } from './baseline.mjs'; const BASE = process.env.A11Y_BASE_URL ?? 'http://localhost:3100'; const OUT = process.env.A11Y_OUT ?? 'a11y-out'; fs.mkdirSync(OUT, { recursive: true }); +// CI mode skips the live-agent send flow (no Octavus in Actions) and stubs the +// API so the app boots without credentials. It still covers empty + settings + +// settings-with-dropdown in both color schemes — the states that carry today's +// known axe baseline. +const CI = process.env.A11Y_CI === '1'; +const BASELINE_PATH = process.env.A11Y_BASELINE; +const UPDATE_BASELINE = process.env.A11Y_UPDATE_BASELINE === '1'; +const TOOLS_DIR = path.dirname(fileURLToPath(import.meta.url)); +const DEFAULT_BASELINE = path.join(TOOLS_DIR, 'baseline.json'); + const TAGS = ['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa', 'wcag22aa', 'best-practice']; function srgb(c) { @@ -181,6 +194,88 @@ async function scan(page, label) { }; } +async function maybeScreenshot(page, name) { + if (CI) return; + await page.screenshot({ path: `${OUT}/${name}`, fullPage: name.includes('empty') ? false : undefined }); +} + +/** + * Stub the JSON API so the app boots without Octavus. Used only in CI mode — + * the full local audit still talks to a real server/agent. + */ +async function installCiApiStub(page) { + const config = { + temperature: 0.7, + allowCustomInstructions: true, + customInstructions: '', + hideSettings: false, + hideHistory: false, + hideFileUpload: false, + hidePromptControls: false, + hideModelSettings: false, + model: 'anthropic/claude-sonnet-4-6', + allowedModels: ['anthropic/claude-sonnet-4-6'], + thinking: 'medium', + showReasoning: true, + }; + const models = ['anthropic/claude-sonnet-4-6']; + const capabilities = { + 'anthropic/claude-sonnet-4-6': { supportsThinking: true }, + }; + // Two rows so nested-interactive matches the shape the audit recorded + // (active session + another history item), not a single-item sidebar. + const sessions = [ + { + session_id: 'session-1', + title: 'New conversation', + created_at: '2026-08-05T00:00:00.000Z', + updated_at: '2026-08-05T12:00:00.000Z', + }, + { + session_id: 'session-0', + title: 'Earlier chat', + created_at: '2026-08-04T00:00:00.000Z', + updated_at: '2026-08-04T00:00:00.000Z', + }, + ]; + + await page.route('**/api/**', async (route) => { + const req = route.request(); + const url = new URL(req.url()); + const pathname = url.pathname; + const method = req.method(); + const json = (data, status = 200) => + route.fulfill({ + status, + contentType: 'application/json', + body: JSON.stringify(data), + }); + + if (pathname === '/api/session' && method === 'GET') { + return json({ sessionId: 'session-1', messages: [] }); + } + if (pathname === '/api/sessions' && method === 'GET') { + return json({ sessions }); + } + if (pathname === '/api/sessions' && method === 'POST') { + return json({ sessionId: 'session-2', messages: [] }); + } + if (pathname === '/api/config' && method === 'GET') { + return json(config); + } + if (pathname === '/api/models' && method === 'GET') { + return json({ models, capabilities }); + } + if (pathname === '/api/session/save' && method === 'POST') { + return json({ ok: true }); + } + if (pathname === '/api/config/custom-instructions' && method === 'POST') { + return json({ ok: true }); + } + return json({ ok: true }); + }); +} + // Defaults to system Chrome because Playwright's bundled Chromium was missing // on the audit machine. In CI, run `playwright install chromium` and set // A11Y_BROWSER_CHANNEL=bundled. @@ -196,54 +291,73 @@ async function run(scheme) { const page = await ctx.newPage(); const report = { scheme, scans: [], tabOrder: null, contrast: [], notes: [] }; + if (CI) { + await installCiApiStub(page); + report.notes.push('CI mode: API stubbed; streaming/with-messages states skipped (no live agent)'); + } + await page.goto(BASE, { waitUntil: 'networkidle' }); await page.waitForTimeout(1500); + if (CI) { + // Fail fast if the stub/boot path broke — an axe scan of the boot-error + // screen would silently drift away from the real baseline. + const visible = await page.locator('.chat-app').evaluate((el) => getComputedStyle(el).visibility); + if (visible === 'hidden') { + const bootText = await page.locator('#bootError').textContent().catch(() => ''); + throw new Error(`App did not boot in CI mode (chat-app still hidden). bootError: ${bootText}`); + } + } + const boot = await page.locator('#bootError').isVisible().catch(() => false); report.notes.push(`bootError visible: ${boot}`); // 1. Empty state report.scans.push(await scan(page, 'empty-state')); - await page.screenshot({ path: `${OUT}/${scheme}-01-empty.png`, fullPage: false }); - report.tabOrder = await walkTabOrder(page); - report.contrast.push({ state: 'empty', pairs: await contrastSample(page) }); + await maybeScreenshot(page, `${scheme}-01-empty.png`); + if (!CI) { + report.tabOrder = await walkTabOrder(page); + report.contrast.push({ state: 'empty', pairs: await contrastSample(page) }); + } - // 2. Send a prompt to get real message content rendered - try { - await page.fill('#promptInput', 'In two short sentences, what is a prompt? Then show a tiny python code block.'); - await page.click('#sendBtn'); - await page.waitForTimeout(1200); - report.scans.push(await scan(page, 'streaming')); - await page.screenshot({ path: `${OUT}/${scheme}-02-streaming.png` }); - - // aria-live sanity: how much text sits inside the live region while streaming - report.notes.push( - 'live region text length while streaming: ' + - (await page.evaluate(() => (document.querySelector('#chatHistory')?.textContent || '').length)), - ); - - await page.waitForFunction( - () => !document.querySelector('.message--ai--streaming'), - null, - { timeout: 60000 }, - ).catch(() => report.notes.push('stream did not finish within 60s')); - await page.waitForTimeout(800); - report.scans.push(await scan(page, 'with-messages')); - await page.screenshot({ path: `${OUT}/${scheme}-03-messages.png` }); - report.contrast.push({ state: 'messages', pairs: await contrastSample(page) }); - - // focus-destruction test: focus a copy button, force a re-render, see where focus lands - const focusTest = await page.evaluate(() => { - const btn = document.querySelector('.message__hover-btn, .code-block__copy'); - if (!btn) return 'no action button found'; - btn.focus(); - const before = document.activeElement?.className; - document.querySelector('#chatHistory .messages')?.dispatchEvent(new Event('x')); - return `focused: ${before}`; - }); - report.notes.push(`hover-action focus test: ${focusTest}`); - } catch (e) { - report.notes.push(`send flow failed: ${e.message.slice(0, 160)}`); + // 2. Send a prompt to get real message content rendered (full audit only) + if (!CI) { + try { + await page.fill('#promptInput', 'In two short sentences, what is a prompt? Then show a tiny python code block.'); + await page.click('#sendBtn'); + await page.waitForTimeout(1200); + report.scans.push(await scan(page, 'streaming')); + await maybeScreenshot(page, `${scheme}-02-streaming.png`); + + // aria-live sanity: how much text sits inside the live region while streaming + report.notes.push( + 'live region text length while streaming: ' + + (await page.evaluate(() => (document.querySelector('#chatHistory')?.textContent || '').length)), + ); + + await page.waitForFunction( + () => !document.querySelector('.message--ai--streaming'), + null, + { timeout: 60000 }, + ).catch(() => report.notes.push('stream did not finish within 60s')); + await page.waitForTimeout(800); + report.scans.push(await scan(page, 'with-messages')); + await maybeScreenshot(page, `${scheme}-03-messages.png`); + report.contrast.push({ state: 'messages', pairs: await contrastSample(page) }); + + // focus-destruction test: focus a copy button, force a re-render, see where focus lands + const focusTest = await page.evaluate(() => { + const btn = document.querySelector('.message__hover-btn, .code-block__copy'); + if (!btn) return 'no action button found'; + btn.focus(); + const before = document.activeElement?.className; + document.querySelector('#chatHistory .messages')?.dispatchEvent(new Event('x')); + return `focused: ${before}`; + }); + report.notes.push(`hover-action focus test: ${focusTest}`); + } catch (e) { + report.notes.push(`send flow failed: ${e.message.slice(0, 160)}`); + } } // 3. Settings modal @@ -254,48 +368,50 @@ async function run(scheme) { report.notes.push(`settings modal open: ${modalOpen > 0}`); if (modalOpen) { report.scans.push(await scan(page, 'settings-modal')); - await page.screenshot({ path: `${OUT}/${scheme}-04-settings.png` }); - report.contrast.push({ state: 'settings', pairs: await contrastSample(page) }); + await maybeScreenshot(page, `${scheme}-04-settings.png`); + if (!CI) { + report.contrast.push({ state: 'settings', pairs: await contrastSample(page) }); - // Does Tab escape the dialog? - const trap = await page.evaluate(async () => { - const dialog = document.querySelector('.modal-overlay.open'); - const inside = []; + // Does Tab escape the dialog? + const trap = await page.evaluate(async () => { + const dialog = document.querySelector('.modal-overlay.open'); + const inside = []; + for (let i = 0; i < 14; i++) { + const el = document.activeElement; + inside.push({ + in: dialog.contains(el), + tag: el?.tagName.toLowerCase(), + cls: (typeof el?.className === 'string' ? el.className : '').slice(0, 50), + }); + // synthetic Tab is unreliable in evaluate; rely on playwright below + break; + } + return inside; + }); + const trapWalk = []; for (let i = 0; i < 14; i++) { - const el = document.activeElement; - inside.push({ - in: dialog.contains(el), - tag: el?.tagName.toLowerCase(), - cls: (typeof el?.className === 'string' ? el.className : '').slice(0, 50), - }); - // synthetic Tab is unreliable in evaluate; rely on playwright below - break; + await page.keyboard.press('Tab'); + trapWalk.push(await page.evaluate(() => { + const d = document.querySelector('.modal-overlay.open'); + const el = document.activeElement; + return { + insideDialog: d ? d.contains(el) : null, + tag: el?.tagName.toLowerCase(), + id: el?.id || null, + cls: (typeof el?.className === 'string' ? el.className : '').slice(0, 45), + }; + })); } - return inside; - }); - const trapWalk = []; - for (let i = 0; i < 14; i++) { - await page.keyboard.press('Tab'); - trapWalk.push(await page.evaluate(() => { - const d = document.querySelector('.modal-overlay.open'); - const el = document.activeElement; - return { - insideDialog: d ? d.contains(el) : null, - tag: el?.tagName.toLowerCase(), - id: el?.id || null, - cls: (typeof el?.className === 'string' ? el.className : '').slice(0, 45), - }; + report.notes.push(`modal initial focus: ${JSON.stringify(trap)}`); + report.modalTabWalk = trapWalk; + + // duplicate id check + report.notes.push(await page.evaluate(() => { + const ids = [...document.querySelectorAll('[id]')].map((e) => e.id); + const dupes = ids.filter((v, i) => ids.indexOf(v) !== i); + return `duplicate ids: ${JSON.stringify([...new Set(dupes)])}`; })); } - report.notes.push(`modal initial focus: ${JSON.stringify(trap)}`); - report.modalTabWalk = trapWalk; - - // duplicate id check - report.notes.push(await page.evaluate(() => { - const ids = [...document.querySelectorAll('[id]')].map((e) => e.id); - const dupes = ids.filter((v, i) => ids.indexOf(v) !== i); - return `duplicate ids: ${JSON.stringify([...new Set(dupes)])}`; - })); // open the Thinking dropdown, check portal + aria await page.evaluate(() => document.querySelector('#thinkingDropdownEl .dropdown-toggle')?.click()); @@ -307,24 +423,26 @@ async function run(scheme) { return `thinking menu portaled: ${!!menu}; menu inside dialog: ${menu && dialog ? dialog.contains(menu) : 'n/a'}; focus after open: ${active?.tagName}.${typeof active?.className === 'string' ? active.className.slice(0, 30) : ''}; options with aria-selected: ${document.querySelectorAll('.dropdown-menu-item[aria-selected]').length}/${document.querySelectorAll('.dropdown-menu-item').length}`; })); report.scans.push(await scan(page, 'settings-modal-dropdown-open')); - await page.screenshot({ path: `${OUT}/${scheme}-05-dropdown.png` }); + await maybeScreenshot(page, `${scheme}-05-dropdown.png`); } } catch (e) { report.notes.push(`settings flow failed: ${e.message.slice(0, 160)}`); } - // 4. Reflow at 320px - try { - await page.setViewportSize({ width: 320, height: 720 }); - await page.waitForTimeout(500); - const overflow = await page.evaluate(() => ({ - docScrollW: document.documentElement.scrollWidth, - clientW: document.documentElement.clientWidth, - })); - report.notes.push(`320px reflow: ${JSON.stringify(overflow)}`); - await page.screenshot({ path: `${OUT}/${scheme}-06-320px.png` }); - } catch (e) { - report.notes.push(`reflow failed: ${e.message.slice(0, 120)}`); + // 4. Reflow at 320px (full audit only) + if (!CI) { + try { + await page.setViewportSize({ width: 320, height: 720 }); + await page.waitForTimeout(500); + const overflow = await page.evaluate(() => ({ + docScrollW: document.documentElement.scrollWidth, + clientW: document.documentElement.clientWidth, + })); + report.notes.push(`320px reflow: ${JSON.stringify(overflow)}`); + await maybeScreenshot(page, `${scheme}-06-320px.png`); + } catch (e) { + report.notes.push(`reflow failed: ${e.message.slice(0, 120)}`); + } } await browser.close(); @@ -332,7 +450,8 @@ async function run(scheme) { } // Isolated design-system dropdown page (the app only exposes one model, so the -// model dropdown never renders in-app). +// model dropdown never renders in-app). Skipped in CI — the test page's landmark +// noise would dominate the baseline without guarding the app. async function runDsDropdown(scheme) { const browser = await chromium.launch(LAUNCH); const ctx = await browser.newContext({ colorScheme: scheme, viewport: { width: 1200, height: 900 } }); @@ -373,7 +492,7 @@ async function runDsDropdown(scheme) { return (el?.tagName || '') + '.' + (typeof el?.className === 'string' ? el.className : '') + ' | isBody=' + (el === document.body); })); out.axeOpen = await scan(page, 'ds-dropdown-open'); - await page.screenshot({ path: `${OUT}/${scheme}-07-ds-dropdown.png` }); + await maybeScreenshot(page, `${scheme}-07-ds-dropdown.png`); } catch (e) { out.notes.push(`ds dropdown failed: ${e.message.slice(0, 160)}`); } @@ -384,7 +503,9 @@ async function runDsDropdown(scheme) { const all = {}; for (const scheme of ['light', 'dark']) { all[scheme] = await run(scheme); - all[`${scheme}-ds-dropdown`] = await runDsDropdown(scheme); + if (!CI) { + all[`${scheme}-ds-dropdown`] = await runDsDropdown(scheme); + } } // Post-process contrast @@ -434,3 +555,44 @@ for (const [k, v] of Object.entries(all)) { } } } + +// ── Baseline gate ───────────────────────────────────────────── +const actual = fingerprint(all); +const baselineFile = BASELINE_PATH || (CI || UPDATE_BASELINE ? DEFAULT_BASELINE : null); + +if (UPDATE_BASELINE) { + writeBaseline(baselineFile, actual); + console.log(`\nUpdated shrink-only baseline at ${baselineFile}`); + console.log(JSON.stringify(actual, null, 2)); +} else if (baselineFile) { + if (!fs.existsSync(baselineFile)) { + console.error(`\nBaseline file not found: ${baselineFile}`); + console.error('Capture one with A11Y_UPDATE_BASELINE=1'); + process.exit(1); + } + const baseline = readBaseline(baselineFile); + const result = compare(actual, baseline.scans || baseline); + if (result.improvements.length) { + console.log('\nBaseline improvements (update baseline.json in this PR):'); + result.improvements.forEach((line) => console.log(` ✓ ${line}`)); + } + if (result.missingScans.length) { + console.error('\nMissing scans vs baseline (audit did not cover expected states):'); + result.missingScans.forEach((line) => console.error(` ✗ ${line}`)); + } + if (result.regressions.length) { + console.error('\nAxe baseline regressions:'); + result.regressions.forEach((line) => console.error(` ✗ ${line}`)); + } + if (!result.ok) { + console.error('\nAccessibility CI gate failed. The baseline is shrink-only.'); + console.error('Note: axe cannot detect live-region over-announcement (A1).'); + process.exit(1); + } + if (result.improvements.length) { + console.log('\nReminder: commit an updated baseline.json so the floor ratchets down.'); + } + console.log('\nAxe baseline gate passed (shrink-only).'); + console.log('Note: axe cannot detect live-region over-announcement (A1) — a green gate is not conformance.'); +} + diff --git a/a11y-audits/tools/baseline.json b/a11y-audits/tools/baseline.json new file mode 100644 index 0000000..35f1a73 --- /dev/null +++ b/a11y-audits/tools/baseline.json @@ -0,0 +1,46 @@ +{ + "$comment": "Shrink-only axe baseline for PR CI. Node counts may decrease; increases or new rules fail the build. axe cannot detect live-region over-announcement (A1) — a green gate is not conformance.", + "generated": "2026-08-06", + "scans": { + "dark/empty-state": { + "aria-allowed-role": 1, + "color-contrast": 3, + "landmark-one-main": 1, + "nested-interactive": 2, + "region": 1 + }, + "dark/settings-modal": { + "aria-allowed-role": 1, + "color-contrast": 4, + "nested-interactive": 3, + "region": 1 + }, + "dark/settings-modal-dropdown-open": { + "aria-allowed-role": 1, + "aria-input-field-name": 1, + "color-contrast": 4, + "nested-interactive": 3, + "region": 6 + }, + "light/empty-state": { + "aria-allowed-role": 1, + "color-contrast": 3, + "landmark-one-main": 1, + "nested-interactive": 2, + "region": 1 + }, + "light/settings-modal": { + "aria-allowed-role": 1, + "color-contrast": 3, + "nested-interactive": 3, + "region": 1 + }, + "light/settings-modal-dropdown-open": { + "aria-allowed-role": 1, + "aria-input-field-name": 1, + "color-contrast": 3, + "nested-interactive": 3, + "region": 6 + } + } +} diff --git a/a11y-audits/tools/baseline.mjs b/a11y-audits/tools/baseline.mjs new file mode 100644 index 0000000..672bf70 --- /dev/null +++ b/a11y-audits/tools/baseline.mjs @@ -0,0 +1,107 @@ +/** + * Shrink-only axe baseline helpers for the PR CI gate. + * + * The baseline maps "scheme/state" → { ruleId: nodeCount }. A run fails when a + * rule appears that is not in the baseline, or when a rule's node count grows. + * Counts may decrease freely; that is how fix PRs retire violations. + * + * axe cannot detect live-region over-announcement (finding A1). A green compare + * is a floor, not WCAG conformance. + */ + +import fs from 'node:fs'; + +/** Flatten an audit report into { "light/empty-state": { rule: count }, ... }. */ +export function fingerprint(report) { + const out = {}; + for (const [key, value] of Object.entries(report)) { + if (!value || typeof value !== 'object') continue; + + for (const s of value.scans || []) { + out[`${key}/${s.label}`] = countsFromViolations(s.violations); + } + if (value.axe) { + out[`${key}/${value.axe.label}`] = countsFromViolations(value.axe.violations); + } + if (value.axeOpen) { + out[`${key}/${value.axeOpen.label}`] = countsFromViolations(value.axeOpen.violations); + } + } + return sortNested(out); +} + +function countsFromViolations(violations = []) { + const counts = {}; + for (const v of violations) { + counts[v.id] = v.total ?? v.nodes?.length ?? 0; + } + return counts; +} + +function sortNested(obj) { + const out = {}; + for (const key of Object.keys(obj).sort()) { + const inner = obj[key]; + out[key] = Object.fromEntries(Object.entries(inner).sort(([a], [b]) => a.localeCompare(b))); + } + return out; +} + +export function readBaseline(path) { + return JSON.parse(fs.readFileSync(path, 'utf8')); +} + +export function writeBaseline(path, scans) { + const doc = { + $comment: + 'Shrink-only axe baseline for PR CI. Node counts may decrease; increases or new rules fail the build. axe cannot detect live-region over-announcement (A1) — a green gate is not conformance.', + generated: new Date().toISOString().slice(0, 10), + scans, + }; + fs.writeFileSync(path, `${JSON.stringify(doc, null, 2)}\n`); +} + +/** + * Compare an actual fingerprint against a checked-in baseline. + * @returns {{ ok: boolean, regressions: string[], improvements: string[], missingScans: string[] }} + */ +export function compare(actual, baselineScans) { + const regressions = []; + const improvements = []; + const missingScans = []; + + for (const scanKey of Object.keys(baselineScans).sort()) { + if (!(scanKey in actual)) { + missingScans.push(scanKey); + continue; + } + const expected = baselineScans[scanKey]; + const got = actual[scanKey]; + + for (const rule of new Set([...Object.keys(expected), ...Object.keys(got)])) { + const before = expected[rule] ?? 0; + const after = got[rule] ?? 0; + if (after > before) { + regressions.push(`${scanKey}: ${rule} ${before} → ${after}`); + } else if (after < before) { + improvements.push(`${scanKey}: ${rule} ${before} → ${after}`); + } + } + } + + // A scan key that exists only in the actual run is fine (coverage grew), but + // any violations there are by definition new and must fail. + for (const scanKey of Object.keys(actual).sort()) { + if (scanKey in baselineScans) continue; + for (const [rule, count] of Object.entries(actual[scanKey])) { + if (count > 0) regressions.push(`${scanKey}: ${rule} (new scan) 0 → ${count}`); + } + } + + return { + ok: regressions.length === 0 && missingScans.length === 0, + regressions, + improvements, + missingScans, + }; +} diff --git a/a11y-audits/tools/baseline.test.mjs b/a11y-audits/tools/baseline.test.mjs new file mode 100644 index 0000000..2d16563 --- /dev/null +++ b/a11y-audits/tools/baseline.test.mjs @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { compare, fingerprint } from './baseline.mjs'; + +describe('fingerprint', () => { + it('flattens scheme/state rule counts', () => { + const fp = fingerprint({ + light: { + scans: [ + { + label: 'empty-state', + violations: [ + { id: 'color-contrast', total: 3 }, + { id: 'region', total: 1 }, + ], + }, + ], + }, + }); + assert.deepEqual(fp, { + 'light/empty-state': { 'color-contrast': 3, region: 1 }, + }); + }); +}); + +describe('compare', () => { + const baseline = { + 'light/empty-state': { 'color-contrast': 3, region: 1 }, + }; + + it('passes when counts match', () => { + const result = compare(baseline, baseline); + assert.equal(result.ok, true); + assert.deepEqual(result.regressions, []); + }); + + it('passes when counts shrink', () => { + const actual = { 'light/empty-state': { 'color-contrast': 2 } }; + const result = compare(actual, baseline); + assert.equal(result.ok, true); + assert.deepEqual(result.improvements, [ + 'light/empty-state: color-contrast 3 → 2', + 'light/empty-state: region 1 → 0', + ]); + }); + + it('fails when a count grows or a new rule appears', () => { + const actual = { + 'light/empty-state': { 'color-contrast': 4, 'nested-interactive': 1, region: 1 }, + }; + const result = compare(actual, baseline); + assert.equal(result.ok, false); + assert.deepEqual(result.regressions, [ + 'light/empty-state: color-contrast 3 → 4', + 'light/empty-state: nested-interactive 0 → 1', + ]); + }); + + it('fails when a baseline scan is missing from the run', () => { + const result = compare({}, baseline); + assert.equal(result.ok, false); + assert.deepEqual(result.missingScans, ['light/empty-state']); + }); +}); diff --git a/a11y-audits/tools/package.json b/a11y-audits/tools/package.json index 98ae000..97a452b 100644 --- a/a11y-audits/tools/package.json +++ b/a11y-audits/tools/package.json @@ -7,6 +7,9 @@ "license": "ISC", "scripts": { "audit": "node audit.mjs", + "audit:ci": "A11Y_CI=1 A11Y_BROWSER_CHANNEL=bundled A11Y_BASELINE=./baseline.json node audit.mjs", + "audit:update-baseline": "A11Y_CI=1 A11Y_UPDATE_BASELINE=1 node audit.mjs", + "test": "node --test baseline.test.mjs", "verify": "node verify.mjs", "verify:sessions": "node verify2.mjs" },