ci: report Rust and web test coverage on pull requests

.github/scripts/coverage-summary.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# Render the coverage numbers from a CI run as Markdown, for the job summary and
+# the pull-request comment.
+#
+# Usage: coverage-summary.sh <rust-summary.json> <web-coverage-summary.json>
+#
+# Either file may be missing: the report says which side did not produce one
+# rather than failing, because a red report job would read as "coverage is
+# broken" when the cause is a test job that failed for its own reasons.
+#
+# Inputs:
+# - Rust: `cargo llvm-cov report --json --summary-only` (llvm-cov export format).
+# - Web: Vitest's `json-summary` reporter (istanbul's coverage-summary.json).
+
+set -euo pipefail
+
+rust_json="${1:-}"
+web_json="${2:-}"
+
+pct() {
+  # covered, total -> "12.3%", or "n/a" when there is nothing to cover.
+  awk -v c="$1" -v t="$2" 'BEGIN { if (t == 0) print "n/a"; else printf "%.1f%%\n", 100 * c / t }'
+}
+
+echo "### Coverage"
+echo
+
+echo "#### Backend (Rust)"
+echo
+if [ -n "$rust_json" ] && [ -f "$rust_json" ]; then
+  echo "| Crate | Lines | Functions | Regions |"
+  echo "| --- | ---: | ---: | ---: |"
+
+  # Group files by crate: crates/<name>/..., migration/..., else the root
+  # `codex` binary crate. Paths are absolute on the runner, so match on the
+  # segment rather than a prefix.
+  jq -r '
+    .data[0].files
+    | map({
+        crate: (
+          if (.filename | test("/crates/[^/]+/")) then (.filename | capture("/crates/(?<c>[^/]+)/").c)
+          elif (.filename | test("/migration/")) then "migration"
+          else "codex"
+          end
+        ),
+        s: .summary
+      })
+    | group_by(.crate)
+    | map({
+        crate: .[0].crate,
+        lc: (map(.s.lines.covered) | add), lt: (map(.s.lines.count) | add),
+        fc: (map(.s.functions.covered) | add), ft: (map(.s.functions.count) | add),
+        rc: (map(.s.regions.covered) | add), rt: (map(.s.regions.count) | add)
+      })
+    | sort_by(.crate)[]
+    | [.crate, .lc, .lt, .fc, .ft, .rc, .rt]
+    | @tsv
+  ' "$rust_json" | while IFS=$'\t' read -r crate lc lt fc ft rc rt; do
+    echo "| \`$crate\` | $(pct "$lc" "$lt") | $(pct "$fc" "$ft") | $(pct "$rc" "$rt") |"
+  done
+
+  read -r lc lt fc ft rc rt < <(jq -r '
+    .data[0].totals
+    | [.lines.covered, .lines.count, .functions.covered, .functions.count,
+       .regions.covered, .regions.count]
+    | @tsv
+  ' "$rust_json")
+  echo "| **Total** | **$(pct "$lc" "$lt")** | **$(pct "$fc" "$ft")** | **$(pct "$rc" "$rt")** |"
+else
+  echo "_No report: the Rust coverage job did not produce one. Check its logs._"
+fi
+echo
+
+echo "#### Frontend (web)"
+echo
+if [ -n "$web_json" ] && [ -f "$web_json" ]; then
+  echo "| Lines | Statements | Functions | Branches |"
+  echo "| ---: | ---: | ---: | ---: |"
+  read -r lc lt sc st fc ft bc bt < <(jq -r '
+    .total
+    | [.lines.covered, .lines.total, .statements.covered, .statements.total,
+       .functions.covered, .functions.total, .branches.covered, .branches.total]
+    | @tsv
+  ' "$web_json")
+  echo "| $(pct "$lc" "$lt") | $(pct "$sc" "$st") | $(pct "$fc" "$ft") | $(pct "$bc" "$bt") |"
+else
+  echo "_No report: the frontend job did not produce one. Check its logs._"
+fi
+echo
+
+echo "<sub>Line-level reports (lcov) are attached to the workflow run as the \`coverage-rust\` and \`coverage-web\` artifacts.</sub>"

.github/workflows/ci.yml

@@ -6,7 +6,7 @@ name: CI
 permissions:
   contents: read
   packages: write
-  # The API Contract job posts its report as a pull-request comment.
+  # The API Contract and Coverage Report jobs post pull-request comments.
   pull-requests: write
 
 on:
@@ -67,6 +67,97 @@ jobs:
       - name: Run tests (partition ${{ matrix.partition }}/5)
         run: cargo nextest run --archive-file nextest-archive.tar.zst --partition hash:${{ matrix.partition }}/5
 
+  # Measure backend test coverage. Reports only: it never fails the run, and the
+  # partitioned test jobs above stay the source of truth for pass/fail.
+  # Instrumented binaries are slower, so this runs on its own runner in
+  # parallel rather than inside the test partitions.
+  coverage:
+    name: Coverage (Rust)
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    env:
+      SCCACHE_GHA_ENABLED: "true"
+      SCCACHE_GHA_VERSION: coverage
+      RUSTC_WRAPPER: sccache
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install mold linker
+        run: sudo apt-get update && sudo apt-get install -y mold
+      # Not in rust-toolchain.toml, so local builds do not all download it.
+      - name: Install llvm-tools
+        run: rustup component add llvm-tools-preview
+      - name: Install cargo-llvm-cov and cargo-nextest
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-llvm-cov,nextest
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.9
+      # A failing test is reported by the test partitions; here it would only
+      # turn the coverage check red as well. The numbers are still written.
+      - name: Run tests with coverage
+        continue-on-error: true
+        run: cargo llvm-cov nextest --workspace --features rar --no-report --no-fail-fast
+      # Test code is excluded so the figure is about the code under test.
+      - name: Write reports
+        if: always()
+        run: |
+          IGNORE='(^|/)tests/'
+          cargo llvm-cov report --ignore-filename-regex "$IGNORE" --lcov --output-path rust-lcov.info
+          cargo llvm-cov report --ignore-filename-regex "$IGNORE" --json --summary-only --output-path rust-coverage.json
+      - name: Upload coverage
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-rust
+          path: |
+            rust-coverage.json
+            rust-lcov.info
+          if-no-files-found: ignore
+          retention-days: 14
+
+  # Collate both coverage reports into the job summary and one pull-request
+  # comment, edited in place like the API contract report.
+  coverage-report:
+    name: Coverage Report
+    needs: [coverage, frontend]
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download coverage
+        uses: actions/download-artifact@v4
+        continue-on-error: true
+        with:
+          pattern: coverage-*
+          path: coverage
+      - name: Render the report
+        run: |
+          bash .github/scripts/coverage-summary.sh \
+            coverage/coverage-rust/rust-coverage.json \
+            coverage/coverage-web/coverage-summary.json > coverage.md
+          cat coverage.md >> "$GITHUB_STEP_SUMMARY"
+      - name: Comment the coverage report
+        if: github.event.pull_request.head.repo.full_name == github.repository
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR: ${{ github.event.pull_request.number }}
+          REPO: ${{ github.repository }}
+        run: |
+          MARKER='<!-- coverage-report -->'
+          BODY="${MARKER}"$'\n'"$(cat coverage.md)"
+
+          ID=$(gh api "repos/${REPO}/issues/${PR}/comments" --paginate \
+                 --jq "[.[] | select(.body | startswith(\"${MARKER}\")) | .id] | first // empty")
+
+          if [ -n "$ID" ]; then
+            gh api -X PATCH "repos/${REPO}/issues/comments/${ID}" -f body="$BODY" >/dev/null
+            echo "Updated comment ${ID}"
+          else
+            gh api -X POST "repos/${REPO}/issues/${PR}/comments" -f body="$BODY" >/dev/null
+            echo "Created comment"
+          fi
+
   # Run linting checks
   lint:
     name: Lint
@@ -238,7 +329,17 @@ jobs:
         with:
           timeout_minutes: 10
           max_attempts: 3
-          command: cd web && npm run test:run
+          command: cd web && npm run test:coverage
+      - name: Upload coverage
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-web
+          path: |
+            web/coverage/coverage-summary.json
+            web/coverage/lcov.info
+          if-no-files-found: ignore
+          retention-days: 14
       - name: Build frontend
         working-directory: web
         run: npm run build

web/.gitignore

@@ -12,6 +12,9 @@ dist
 dist-ssr
 *.local
 
+# Test coverage output (npm run test:coverage)
+coverage
+
 # Editor directories and files
 .vscode/*
 !.vscode/extensions.json