diff --git a/.github/scripts/coverage-summary.sh b/.github/scripts/coverage-summary.sh new file mode 100644 index 00000000..825e2cdd --- /dev/null +++ b/.github/scripts/coverage-summary.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Render the coverage numbers from a CI run as Markdown, for the job summary and +# the pull-request comment. +# +# Usage: coverage-summary.sh +# +# Either file may be missing: the report says which side did not produce one +# rather than failing, because a red report job would read as "coverage is +# broken" when the cause is a test job that failed for its own reasons. +# +# Inputs: +# - Rust: `cargo llvm-cov report --json --summary-only` (llvm-cov export format). +# - Web: Vitest's `json-summary` reporter (istanbul's coverage-summary.json). + +set -euo pipefail + +rust_json="${1:-}" +web_json="${2:-}" + +pct() { + # covered, total -> "12.3%", or "n/a" when there is nothing to cover. + awk -v c="$1" -v t="$2" 'BEGIN { if (t == 0) print "n/a"; else printf "%.1f%%\n", 100 * c / t }' +} + +echo "### Coverage" +echo + +echo "#### Backend (Rust)" +echo +if [ -n "$rust_json" ] && [ -f "$rust_json" ]; then + echo "| Crate | Lines | Functions | Regions |" + echo "| --- | ---: | ---: | ---: |" + + # Group files by crate: crates//..., migration/..., else the root + # `codex` binary crate. Paths are absolute on the runner, so match on the + # segment rather than a prefix. + jq -r ' + .data[0].files + | map({ + crate: ( + if (.filename | test("/crates/[^/]+/")) then (.filename | capture("/crates/(?[^/]+)/").c) + elif (.filename | test("/migration/")) then "migration" + else "codex" + end + ), + s: .summary + }) + | group_by(.crate) + | map({ + crate: .[0].crate, + lc: (map(.s.lines.covered) | add), lt: (map(.s.lines.count) | add), + fc: (map(.s.functions.covered) | add), ft: (map(.s.functions.count) | add), + rc: (map(.s.regions.covered) | add), rt: (map(.s.regions.count) | add) + }) + | sort_by(.crate)[] + | [.crate, .lc, .lt, .fc, .ft, .rc, .rt] + | @tsv + ' "$rust_json" | while IFS=$'\t' read -r crate lc lt fc ft rc rt; do + echo "| \`$crate\` | $(pct "$lc" "$lt") | $(pct "$fc" "$ft") | $(pct "$rc" "$rt") |" + done + + read -r lc lt fc ft rc rt < <(jq -r ' + .data[0].totals + | [.lines.covered, .lines.count, .functions.covered, .functions.count, + .regions.covered, .regions.count] + | @tsv + ' "$rust_json") + echo "| **Total** | **$(pct "$lc" "$lt")** | **$(pct "$fc" "$ft")** | **$(pct "$rc" "$rt")** |" +else + echo "_No report: the Rust coverage job did not produce one. Check its logs._" +fi +echo + +echo "#### Frontend (web)" +echo +if [ -n "$web_json" ] && [ -f "$web_json" ]; then + echo "| Lines | Statements | Functions | Branches |" + echo "| ---: | ---: | ---: | ---: |" + read -r lc lt sc st fc ft bc bt < <(jq -r ' + .total + | [.lines.covered, .lines.total, .statements.covered, .statements.total, + .functions.covered, .functions.total, .branches.covered, .branches.total] + | @tsv + ' "$web_json") + echo "| $(pct "$lc" "$lt") | $(pct "$sc" "$st") | $(pct "$fc" "$ft") | $(pct "$bc" "$bt") |" +else + echo "_No report: the frontend job did not produce one. Check its logs._" +fi +echo + +echo "Line-level reports (lcov) are attached to the workflow run as the \`coverage-rust\` and \`coverage-web\` artifacts." diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d97ba06..a0185f56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ name: CI permissions: contents: read packages: write - # The API Contract job posts its report as a pull-request comment. + # The API Contract and Coverage Report jobs post pull-request comments. pull-requests: write on: @@ -67,6 +67,97 @@ jobs: - name: Run tests (partition ${{ matrix.partition }}/5) run: cargo nextest run --archive-file nextest-archive.tar.zst --partition hash:${{ matrix.partition }}/5 + # Measure backend test coverage. Reports only: it never fails the run, and the + # partitioned test jobs above stay the source of truth for pass/fail. + # Instrumented binaries are slower, so this runs on its own runner in + # parallel rather than inside the test partitions. + coverage: + name: Coverage (Rust) + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + SCCACHE_GHA_ENABLED: "true" + SCCACHE_GHA_VERSION: coverage + RUSTC_WRAPPER: sccache + steps: + - uses: actions/checkout@v4 + - name: Install mold linker + run: sudo apt-get update && sudo apt-get install -y mold + # Not in rust-toolchain.toml, so local builds do not all download it. + - name: Install llvm-tools + run: rustup component add llvm-tools-preview + - name: Install cargo-llvm-cov and cargo-nextest + uses: taiki-e/install-action@v2 + with: + tool: cargo-llvm-cov,nextest + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.9 + # A failing test is reported by the test partitions; here it would only + # turn the coverage check red as well. The numbers are still written. + - name: Run tests with coverage + continue-on-error: true + run: cargo llvm-cov nextest --workspace --features rar --no-report --no-fail-fast + # Test code is excluded so the figure is about the code under test. + - name: Write reports + if: always() + run: | + IGNORE='(^|/)tests/' + cargo llvm-cov report --ignore-filename-regex "$IGNORE" --lcov --output-path rust-lcov.info + cargo llvm-cov report --ignore-filename-regex "$IGNORE" --json --summary-only --output-path rust-coverage.json + - name: Upload coverage + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-rust + path: | + rust-coverage.json + rust-lcov.info + if-no-files-found: ignore + retention-days: 14 + + # Collate both coverage reports into the job summary and one pull-request + # comment, edited in place like the API contract report. + coverage-report: + name: Coverage Report + needs: [coverage, frontend] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - name: Download coverage + uses: actions/download-artifact@v4 + continue-on-error: true + with: + pattern: coverage-* + path: coverage + - name: Render the report + run: | + bash .github/scripts/coverage-summary.sh \ + coverage/coverage-rust/rust-coverage.json \ + coverage/coverage-web/coverage-summary.json > coverage.md + cat coverage.md >> "$GITHUB_STEP_SUMMARY" + - name: Comment the coverage report + if: github.event.pull_request.head.repo.full_name == github.repository + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + run: | + MARKER='' + BODY="${MARKER}"$'\n'"$(cat coverage.md)" + + ID=$(gh api "repos/${REPO}/issues/${PR}/comments" --paginate \ + --jq "[.[] | select(.body | startswith(\"${MARKER}\")) | .id] | first // empty") + + if [ -n "$ID" ]; then + gh api -X PATCH "repos/${REPO}/issues/comments/${ID}" -f body="$BODY" >/dev/null + echo "Updated comment ${ID}" + else + gh api -X POST "repos/${REPO}/issues/${PR}/comments" -f body="$BODY" >/dev/null + echo "Created comment" + fi + # Run linting checks lint: name: Lint @@ -238,7 +329,17 @@ jobs: with: timeout_minutes: 10 max_attempts: 3 - command: cd web && npm run test:run + command: cd web && npm run test:coverage + - name: Upload coverage + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-web + path: | + web/coverage/coverage-summary.json + web/coverage/lcov.info + if-no-files-found: ignore + retention-days: 14 - name: Build frontend working-directory: web run: npm run build diff --git a/web/.gitignore b/web/.gitignore index a547bf36..91450046 100644 --- a/web/.gitignore +++ b/web/.gitignore @@ -12,6 +12,9 @@ dist dist-ssr *.local +# Test coverage output (npm run test:coverage) +coverage + # Editor directories and files .vscode/* !.vscode/extensions.json diff --git a/web/package-lock.json b/web/package-lock.json index 690e1485..291ac0b3 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -67,6 +67,7 @@ "@types/react-dom": "^19.2.3", "@types/react-router-dom": "^5.3.3", "@vitejs/plugin-react-swc": "^4.2.3", + "@vitest/coverage-v8": "^4.0.18", "@vitest/ui": "^4.0.18", "fake-indexeddb": "^6.2.5", "globals": "^16.5.0", @@ -422,9 +423,9 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", - "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", "dev": true, "license": "MIT", "engines": { @@ -432,9 +433,9 @@ } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", - "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", "dev": true, "license": "MIT", "engines": { @@ -481,13 +482,13 @@ } }, "node_modules/@babel/parser": { - "version": "7.29.3", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.3.tgz", - "integrity": "sha512-b3ctpQwp+PROvU/cttc4OYl4MzfJUWy6FZg+PMXfzmt/+39iHVF0sDfqay8TQM3JA2EUOyKcFZt75jWriQijsA==", + "version": "7.29.9", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.9.tgz", + "integrity": "sha512-CjXrNHTnvqBVqHgdBysY3vk2T8tpJHb5/RMeHJBTyVa9xgugCB0CJTx/3oO8RV2QRQP391RWpB7D6hLjm8V9uA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.29.0" + "@babel/types": "^7.29.8" }, "bin": { "parser": "bin/babel-parser.js" @@ -1660,19 +1661,29 @@ } }, "node_modules/@babel/types": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", - "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.8.tgz", + "integrity": "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-string-parser": "^7.27.1", - "@babel/helper-validator-identifier": "^7.28.5" + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, + "node_modules/@bcoe/v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@biomejs/biome": { "version": "2.4.4", "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.4.4.tgz", @@ -5083,6 +5094,37 @@ "vite": "^4 || ^5 || ^6 || ^7" } }, + "node_modules/@vitest/coverage-v8": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.0.18.tgz", + "integrity": "sha512-7i+N2i0+ME+2JFZhfuz7Tg/FqKtilHjGyGvoHYQ6iLV0zahbsJ9sljC9OcFcPDbhYKCet+sG8SsVqlyGvPflZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@bcoe/v8-coverage": "^1.0.2", + "@vitest/utils": "4.0.18", + "ast-v8-to-istanbul": "^0.3.10", + "istanbul-lib-coverage": "^3.2.2", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "^3.2.0", + "magicast": "^0.5.1", + "obug": "^2.1.1", + "std-env": "^3.10.0", + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@vitest/browser": "4.0.18", + "vitest": "4.0.18" + }, + "peerDependenciesMeta": { + "@vitest/browser": { + "optional": true + } + } + }, "node_modules/@vitest/expect": { "version": "4.0.18", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.0.18.tgz", @@ -5376,6 +5418,25 @@ "node": ">=12" } }, + "node_modules/ast-v8-to-istanbul": { + "version": "0.3.12", + "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-0.3.12.tgz", + "integrity": "sha512-BRRC8VRZY2R4Z4lFIL35MwNXmwVqBityvOIwETtsCSwvjl0IdgFsy9NhdaA6j74nUdtJJlIypeRhpDam19Wq3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.31", + "estree-walker": "^3.0.3", + "js-tokens": "^10.0.0" + } + }, + "node_modules/ast-v8-to-istanbul/node_modules/js-tokens": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", + "integrity": "sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==", + "dev": true, + "license": "MIT" + }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", @@ -7486,6 +7547,13 @@ "node": ">=18" } }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, "node_modules/html-url-attributes": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/html-url-attributes/-/html-url-attributes-3.0.1.tgz", @@ -8142,6 +8210,45 @@ "dev": true, "license": "ISC" }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/jackspeak": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.2.3.tgz", @@ -8653,6 +8760,18 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/magicast": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.5.tgz", + "integrity": "sha512-UicdXN8zQ3JHlxVq+28afMXPr1z7WNY6+7EJnzTdQWkTAlMLF5fNCCKxJHBQwGaNGR11581EiQmQzx73+MvszA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7", + "source-map-js": "^1.2.1" + } + }, "node_modules/make-cancellable-promise": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/make-cancellable-promise/-/make-cancellable-promise-2.0.0.tgz", @@ -8662,6 +8781,35 @@ "url": "https://github.com/wojtekmaj/make-cancellable-promise?sponsor=1" } }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/make-event-props": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/make-event-props/-/make-event-props-2.0.0.tgz", diff --git a/web/package.json b/web/package.json index fa02428e..63d9ace2 100644 --- a/web/package.json +++ b/web/package.json @@ -76,6 +76,7 @@ "@types/react-dom": "^19.2.3", "@types/react-router-dom": "^5.3.3", "@vitejs/plugin-react-swc": "^4.2.3", + "@vitest/coverage-v8": "^4.0.18", "@vitest/ui": "^4.0.18", "fake-indexeddb": "^6.2.5", "globals": "^16.5.0", diff --git a/web/vitest.config.ts b/web/vitest.config.ts index b3451663..a06c3db9 100644 --- a/web/vitest.config.ts +++ b/web/vitest.config.ts @@ -33,10 +33,17 @@ export default defineConfig({ maxWorkers: "50%", coverage: { provider: "v8", - reporter: ["text", "json", "html"], + // `json-summary` and `lcov` feed the CI coverage report; `html` is for + // browsing locally (`npm run test:coverage`, then open coverage/index.html). + reporter: ["text-summary", "json-summary", "lcov", "html"], + // Without `include`, Vitest only counts files some test happened to + // import, so a module nothing tests is invisible and the total flatters. + include: ["src/**/*.{ts,tsx}"], exclude: [ "node_modules/", "src/test/", + "src/mocks/", + "**/*.test.{ts,tsx}", "**/*.d.ts", "**/*.config.*", "**/mockData",