From 78daf0cdc6751fc04f4f285fb5184698af486ea9 Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Fri, 2 Oct 2026 14:28:17 -0700 Subject: [PATCH] ci: report Rust and web test coverage on pull requests Nothing in CI showed how much of the code the tests exercise, so a PR could drop coverage without anyone noticing. Rust coverage runs in its own job with cargo-llvm-cov and nextest. The instrumented build is slower, and a separate runner keeps the partitioned test jobs unchanged. Test code is excluded from the figures so they describe the code under test. A failing test does not fail this job, because the partitions already report that; the numbers are still written. Web coverage comes from Vitest's v8 provider, which the frontend job now runs in place of the plain test command. The provider is pinned to the locked Vitest version so adding it does not move the test runner. Coverage `include` covers all of src, so a module no test imports counts against the total instead of being invisible. A report job collates both into the job summary and a single pull request comment that is edited in place, the same way the API contract report is. It only reports and never fails the run. The summary is rendered by a small script that tolerates a missing input, so a broken test job shows up as "no report" rather than a second red check. The lcov files are kept as artifacts for 14 days. There is no external coverage service, so no history is kept across commits. --- .github/scripts/coverage-summary.sh | 91 ++++++++++++++ .github/workflows/ci.yml | 105 +++++++++++++++- web/.gitignore | 3 + web/package-lock.json | 178 +++++++++++++++++++++++++--- web/package.json | 1 + web/vitest.config.ts | 9 +- 6 files changed, 369 insertions(+), 18 deletions(-) create mode 100644 .github/scripts/coverage-summary.sh diff --git a/.github/scripts/coverage-summary.sh b/.github/scripts/coverage-summary.sh new file mode 100644 index 000000000..825e2cddd --- /dev/null +++ b/.github/scripts/coverage-summary.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Render the coverage numbers from a CI run as Markdown, for the job summary and +# the pull-request comment. +# +# Usage: coverage-summary.sh +# +# Either file may be missing: the report says which side did not produce one +# rather than failing, because a red report job would read as "coverage is +# broken" when the cause is a test job that failed for its own reasons. +# +# Inputs: +# - Rust: `cargo llvm-cov report --json --summary-only` (llvm-cov export format). +# - Web: Vitest's `json-summary` reporter (istanbul's coverage-summary.json). + +set -euo pipefail + +rust_json="${1:-}" +web_json="${2:-}" + +pct() { + # covered, total -> "12.3%", or "n/a" when there is nothing to cover. + awk -v c="$1" -v t="$2" 'BEGIN { if (t == 0) print "n/a"; else printf "%.1f%%\n", 100 * c / t }' +} + +echo "### Coverage" +echo + +echo "#### Backend (Rust)" +echo +if [ -n "$rust_json" ] && [ -f "$rust_json" ]; then + echo "| Crate | Lines | Functions | Regions |" + echo "| --- | ---: | ---: | ---: |" + + # Group files by crate: crates//..., migration/..., else the root + # `codex` binary crate. Paths are absolute on the runner, so match on the + # segment rather than a prefix. + jq -r ' + .data[0].files + | map({ + crate: ( + if (.filename | test("/crates/[^/]+/")) then (.filename | capture("/crates/(?[^/]+)/").c) + elif (.filename | test("/migration/")) then "migration" + else "codex" + end + ), + s: .summary + }) + | group_by(.crate) + | map({ + crate: .[0].crate, + lc: (map(.s.lines.covered) | add), lt: (map(.s.lines.count) | add), + fc: (map(.s.functions.covered) | add), ft: (map(.s.functions.count) | add), + rc: (map(.s.regions.covered) | add), rt: (map(.s.regions.count) | add) + }) + | sort_by(.crate)[] + | [.crate, .lc, .lt, .fc, .ft, .rc, .rt] + | @tsv + ' "$rust_json" | while IFS=$'\t' read -r crate lc lt fc ft rc rt; do + echo "| \`$crate\` | $(pct "$lc" "$lt") | $(pct "$fc" "$ft") | $(pct "$rc" "$rt") |" + done + + read -r lc lt fc ft rc rt < <(jq -r ' + .data[0].totals + | [.lines.covered, .lines.count, .functions.covered, .functions.count, + .regions.covered, .regions.count] + | @tsv + ' "$rust_json") + echo "| **Total** | **$(pct "$lc" "$lt")** | **$(pct "$fc" "$ft")** | **$(pct "$rc" "$rt")** |" +else + echo "_No report: the Rust coverage job did not produce one. Check its logs._" +fi +echo + +echo "#### Frontend (web)" +echo +if [ -n "$web_json" ] && [ -f "$web_json" ]; then + echo "| Lines | Statements | Functions | Branches |" + echo "| ---: | ---: | ---: | ---: |" + read -r lc lt sc st fc ft bc bt < <(jq -r ' + .total + | [.lines.covered, .lines.total, .statements.covered, .statements.total, + .functions.covered, .functions.total, .branches.covered, .branches.total] + | @tsv + ' "$web_json") + echo "| $(pct "$lc" "$lt") | $(pct "$sc" "$st") | $(pct "$fc" "$ft") | $(pct "$bc" "$bt") |" +else + echo "_No report: the frontend job did not produce one. Check its logs._" +fi +echo + +echo "Line-level reports (lcov) are attached to the workflow run as the \`coverage-rust\` and \`coverage-web\` artifacts." diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d97ba068..a0185f56f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ name: CI permissions: contents: read packages: write - # The API Contract job posts its report as a pull-request comment. + # The API Contract and Coverage Report jobs post pull-request comments. pull-requests: write on: @@ -67,6 +67,97 @@ jobs: - name: Run tests (partition ${{ matrix.partition }}/5) run: cargo nextest run --archive-file nextest-archive.tar.zst --partition hash:${{ matrix.partition }}/5 + # Measure backend test coverage. Reports only: it never fails the run, and the + # partitioned test jobs above stay the source of truth for pass/fail. + # Instrumented binaries are slower, so this runs on its own runner in + # parallel rather than inside the test partitions. + coverage: + name: Coverage (Rust) + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + SCCACHE_GHA_ENABLED: "true" + SCCACHE_GHA_VERSION: coverage + RUSTC_WRAPPER: sccache + steps: + - uses: actions/checkout@v4 + - name: Install mold linker + run: sudo apt-get update && sudo apt-get install -y mold + # Not in rust-toolchain.toml, so local builds do not all download it. + - name: Install llvm-tools + run: rustup component add llvm-tools-preview + - name: Install cargo-llvm-cov and cargo-nextest + uses: taiki-e/install-action@v2 + with: + tool: cargo-llvm-cov,nextest + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.9 + # A failing test is reported by the test partitions; here it would only + # turn the coverage check red as well. The numbers are still written. + - name: Run tests with coverage + continue-on-error: true + run: cargo llvm-cov nextest --workspace --features rar --no-report --no-fail-fast + # Test code is excluded so the figure is about the code under test. + - name: Write reports + if: always() + run: | + IGNORE='(^|/)tests/' + cargo llvm-cov report --ignore-filename-regex "$IGNORE" --lcov --output-path rust-lcov.info + cargo llvm-cov report --ignore-filename-regex "$IGNORE" --json --summary-only --output-path rust-coverage.json + - name: Upload coverage + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-rust + path: | + rust-coverage.json + rust-lcov.info + if-no-files-found: ignore + retention-days: 14 + + # Collate both coverage reports into the job summary and one pull-request + # comment, edited in place like the API contract report. + coverage-report: + name: Coverage Report + needs: [coverage, frontend] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - name: Download coverage + uses: actions/download-artifact@v4 + continue-on-error: true + with: + pattern: coverage-* + path: coverage + - name: Render the report + run: | + bash .github/scripts/coverage-summary.sh \ + coverage/coverage-rust/rust-coverage.json \ + coverage/coverage-web/coverage-summary.json > coverage.md + cat coverage.md >> "$GITHUB_STEP_SUMMARY" + - name: Comment the coverage report + if: github.event.pull_request.head.repo.full_name == github.repository + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + run: | + MARKER='' + BODY="${MARKER}"$'\n'"$(cat coverage.md)" + + ID=$(gh api "repos/${REPO}/issues/${PR}/comments" --paginate \ + --jq "[.[] | select(.body | startswith(\"${MARKER}\")) | .id] | first // empty") + + if [ -n "$ID" ]; then + gh api -X PATCH "repos/${REPO}/issues/comments/${ID}" -f body="$BODY" >/dev/null + echo "Updated comment ${ID}" + else + gh api -X POST "repos/${REPO}/issues/${PR}/comments" -f body="$BODY" >/dev/null + echo "Created comment" + fi + # Run linting checks lint: name: Lint @@ -238,7 +329,17 @@ jobs: with: timeout_minutes: 10 max_attempts: 3 - command: cd web && npm run test:run + command: cd web && npm run test:coverage + - name: Upload coverage + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-web + path: | + web/coverage/coverage-summary.json + web/coverage/lcov.info + if-no-files-found: ignore + retention-days: 14 - name: Build frontend working-directory: web run: npm run build diff --git a/web/.gitignore b/web/.gitignore index a547bf36d..91450046f 100644 --- a/web/.gitignore +++ b/web/.gitignore @@ -12,6 +12,9 @@ dist dist-ssr *.local +# Test coverage output (npm run test:coverage) +coverage + # Editor directories and files .vscode/* !.vscode/extensions.json diff --git a/web/package-lock.json b/web/package-lock.json index 690e1485d..291ac0b37 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -67,6 +67,7 @@ "@types/react-dom": "^19.2.3", "@types/react-router-dom": "^5.3.3", "@vitejs/plugin-react-swc": "^4.2.3", + "@vitest/coverage-v8": "^4.0.18", "@vitest/ui": "^4.0.18", "fake-indexeddb": "^6.2.5", "globals": "^16.5.0", @@ -422,9 +423,9 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", - "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", "dev": true, "license": "MIT", "engines": { @@ -432,9 +433,9 @@ } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", - "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", "dev": true, "license": "MIT", "engines": { @@ -481,13 +482,13 @@ } }, "node_modules/@babel/parser": { - "version": "7.29.3", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.3.tgz", - "integrity": "sha512-b3ctpQwp+PROvU/cttc4OYl4MzfJUWy6FZg+PMXfzmt/+39iHVF0sDfqay8TQM3JA2EUOyKcFZt75jWriQijsA==", + "version": "7.29.9", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.9.tgz", + "integrity": "sha512-CjXrNHTnvqBVqHgdBysY3vk2T8tpJHb5/RMeHJBTyVa9xgugCB0CJTx/3oO8RV2QRQP391RWpB7D6hLjm8V9uA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.29.0" + "@babel/types": "^7.29.8" }, "bin": { "parser": "bin/babel-parser.js" @@ -1660,19 +1661,29 @@ } }, "node_modules/@babel/types": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", - "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.8.tgz", + "integrity": "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-string-parser": "^7.27.1", - "@babel/helper-validator-identifier": "^7.28.5" + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, + "node_modules/@bcoe/v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@biomejs/biome": { "version": "2.4.4", "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.4.4.tgz", @@ -5083,6 +5094,37 @@ "vite": "^4 || ^5 || ^6 || ^7" } }, + "node_modules/@vitest/coverage-v8": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.0.18.tgz", + "integrity": "sha512-7i+N2i0+ME+2JFZhfuz7Tg/FqKtilHjGyGvoHYQ6iLV0zahbsJ9sljC9OcFcPDbhYKCet+sG8SsVqlyGvPflZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@bcoe/v8-coverage": "^1.0.2", + "@vitest/utils": "4.0.18", + "ast-v8-to-istanbul": "^0.3.10", + "istanbul-lib-coverage": "^3.2.2", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "^3.2.0", + "magicast": "^0.5.1", + "obug": "^2.1.1", + "std-env": "^3.10.0", + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@vitest/browser": "4.0.18", + "vitest": "4.0.18" + }, + "peerDependenciesMeta": { + "@vitest/browser": { + "optional": true + } + } + }, "node_modules/@vitest/expect": { "version": "4.0.18", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.0.18.tgz", @@ -5376,6 +5418,25 @@ "node": ">=12" } }, + "node_modules/ast-v8-to-istanbul": { + "version": "0.3.12", + "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-0.3.12.tgz", + "integrity": "sha512-BRRC8VRZY2R4Z4lFIL35MwNXmwVqBityvOIwETtsCSwvjl0IdgFsy9NhdaA6j74nUdtJJlIypeRhpDam19Wq3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.31", + "estree-walker": "^3.0.3", + "js-tokens": "^10.0.0" + } + }, + "node_modules/ast-v8-to-istanbul/node_modules/js-tokens": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", + "integrity": "sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==", + "dev": true, + "license": "MIT" + }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", @@ -7486,6 +7547,13 @@ "node": ">=18" } }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, "node_modules/html-url-attributes": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/html-url-attributes/-/html-url-attributes-3.0.1.tgz", @@ -8142,6 +8210,45 @@ "dev": true, "license": "ISC" }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/jackspeak": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.2.3.tgz", @@ -8653,6 +8760,18 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/magicast": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.5.tgz", + "integrity": "sha512-UicdXN8zQ3JHlxVq+28afMXPr1z7WNY6+7EJnzTdQWkTAlMLF5fNCCKxJHBQwGaNGR11581EiQmQzx73+MvszA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7", + "source-map-js": "^1.2.1" + } + }, "node_modules/make-cancellable-promise": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/make-cancellable-promise/-/make-cancellable-promise-2.0.0.tgz", @@ -8662,6 +8781,35 @@ "url": "https://github.com/wojtekmaj/make-cancellable-promise?sponsor=1" } }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/make-event-props": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/make-event-props/-/make-event-props-2.0.0.tgz", diff --git a/web/package.json b/web/package.json index fa02428e7..63d9ace21 100644 --- a/web/package.json +++ b/web/package.json @@ -76,6 +76,7 @@ "@types/react-dom": "^19.2.3", "@types/react-router-dom": "^5.3.3", "@vitejs/plugin-react-swc": "^4.2.3", + "@vitest/coverage-v8": "^4.0.18", "@vitest/ui": "^4.0.18", "fake-indexeddb": "^6.2.5", "globals": "^16.5.0", diff --git a/web/vitest.config.ts b/web/vitest.config.ts index b34516632..a06c3db99 100644 --- a/web/vitest.config.ts +++ b/web/vitest.config.ts @@ -33,10 +33,17 @@ export default defineConfig({ maxWorkers: "50%", coverage: { provider: "v8", - reporter: ["text", "json", "html"], + // `json-summary` and `lcov` feed the CI coverage report; `html` is for + // browsing locally (`npm run test:coverage`, then open coverage/index.html). + reporter: ["text-summary", "json-summary", "lcov", "html"], + // Without `include`, Vitest only counts files some test happened to + // import, so a module nothing tests is invisible and the total flatters. + include: ["src/**/*.{ts,tsx}"], exclude: [ "node_modules/", "src/test/", + "src/mocks/", + "**/*.test.{ts,tsx}", "**/*.d.ts", "**/*.config.*", "**/mockData",