From 02e8dcec1a72b2e358e78430d631a8c1e2a839a9 Mon Sep 17 00:00:00 2001 From: Jake Fineman Date: Fri, 11 Sep 2026 09:19:56 -0400 Subject: [PATCH] fix(release): generate the SBOM after install so it enumerates what ships The sbom job pointed syft at a bare, never-installed wheel (pip download --no-deps), so the shipped SPDX SBOM had one entry and zero runtime dependencies, and the old "packages[] is non-empty" check waved it through. Install the published wheel plus its resolved runtime dependencies into an isolated pip install --target tree, scan that tree, and fail loud via scripts/release/validate_sbom.py unless wave-sdk and every unconditional Requires-Dist appear with a version. Validator is unit-tested in tests/test_release_scripts.py. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/release.yml | 87 +++++++++++++---- scripts/release/validate_sbom.py | 160 +++++++++++++++++++++++++++++++ tests/test_release_scripts.py | 129 +++++++++++++++++++++++++ 3 files changed, 356 insertions(+), 20 deletions(-) create mode 100644 scripts/release/validate_sbom.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b8726a2..b9a5e1e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,12 +26,17 @@ name: release # sbom -- contents: read (no repo write -- only builds an artifact). # Downloads the ACTUAL PUBLISHED wheel from PyPI by # name==version (`pip download wave-sdk==`, never a -# hand-built files.pythonhosted.org URL) so the SBOM always -# describes exactly what PyPI serves, not a possibly -# nondeterministic local rebuild; generates SPDX-JSON with -# anchore/sbom-action (wraps syft, pinned to a full commit SHA) -# and uploads it as a *workflow* artifact for `release` to pick -# up -- kept fully separate from `publish`'s id-token: write. +# hand-built files.pythonhosted.org URL), then INSTALLS that +# exact wheel plus its resolved runtime-dependency closure into +# an isolated `pip install --target` directory and generates +# SPDX-JSON from that INSTALLED tree with anchore/sbom-action +# (wraps syft, pinned to a full commit SHA). Scanning only the +# bare, never-installed wheel produced a one-entry SBOM with +# zero dependencies; the SBOM must enumerate what actually +# ships. Fails loud if wave-sdk or any unconditional declared +# runtime dependency is missing from the result. Uploads it as +# a *workflow* artifact for `release` to pick up -- kept fully +# separate from `publish`'s id-token: write. # release -- contents: write only. Creates/updates the GitHub Release for # the tag with the sdist+wheel AND the SBOM attached in the SAME # `gh release create`/`upload` call, so a Release is never @@ -319,13 +324,22 @@ jobs: # here as a build artifact, then attached by `release` (below) in the SAME # `gh release create`/`upload` call as dist/*, so a Release is never # visible without its SBOM (no window where the Release exists but the - # SBOM doesn't). Downloads the ACTUAL PUBLISHED wheel from PyPI by + # SBOM doesn't). + # + # Provenance: downloads the ACTUAL PUBLISHED wheel from PyPI by # name==version (never a hand-built files.pythonhosted.org URL; # --only-binary=:all: so a missing wheel fails loud instead of silently # SBOM-ing the sdist) rather than reusing `verify`'s locally built dist/ # artifact: a rebuild is not guaranteed byte-identical to what PyPI # actually serves, and the SBOM must describe what ships, not what was # built pre-publish. + # + # Install-before-scan (defect fix): syft used to scan the directory holding + # that bare, never-installed wheel, so the SBOM had one entry and zero + # runtime dependencies -- and "packages[] is non-empty" waved it through. + # Now the wheel is INSTALLED (deps resolved from PyPI) into an isolated + # `pip install --target` tree, syft scans THAT, and validate_sbom.py fails + # loud unless wave-sdk and every unconditional Requires-Dist are present. # --------------------------------------------------------------------------- sbom: name: generate SBOM @@ -339,7 +353,7 @@ jobs: with: python-version: "3.12" - - name: Download the published wheel from PyPI (by name==version, never a direct URL) + - name: Download the published wheel from PyPI (by name==version, never a hand-built URL) env: VERSION: ${{ needs.verify.outputs.version }} run: | @@ -366,25 +380,58 @@ jobs: fi ls -la sbom-src - - name: Generate SPDX-JSON SBOM for the published wheel + - name: Install the published wheel + its runtime dependencies into an isolated target tree + run: | + set -euo pipefail + # Exactly one wheel must have been downloaded above -- refuse to guess otherwise. + # (Without nullglob an unmatched pattern stays literal; the -f test catches that.) + WHEELS=(sbom-src/*.whl) + if [ "${#WHEELS[@]}" -ne 1 ] || [ ! -f "${WHEELS[0]}" ]; then + echo "::error::expected exactly one .whl in sbom-src, found: ${WHEELS[*]}" + exit 1 + fi + mkdir -p sbom-env + # Install the exact downloaded wheel FILE (not `wave-sdk==VERSION` again), so + # the bytes syft describes are the ones `pip download` fetched and listed above. + # --target: a plain directory -- no venv bootstrap packages (pip/setuptools) + # and no runner site-packages leak into the inventory. + # --only-binary=:all:: dependencies must come as wheels; a dependency that + # would need an sdist build (arbitrary setup.py execution) fails + # loud here instead of running build scripts in the release job. + python -m pip install --no-cache-dir --only-binary=:all: --target sbom-env "${WHEELS[0]}" + echo "installed distributions:" + for d in sbom-env/*.dist-info; do basename "$d"; done + + - name: Generate SPDX-JSON SBOM for the installed tree (published wheel + resolved runtime dependencies) uses: anchore/sbom-action@3ad7283483fc7af8ff2b4ea19663c2d5ca935e26 # v0.24.2 with: - path: sbom-src + path: sbom-env format: spdx-json output-file: wave_sdk-${{ needs.verify.outputs.version }}.spdx.json - - name: Validate the SBOM has a non-empty packages[] + # See the matching step in `verify`/`publish`: release tooling is pinned + # to the workflow's own ref (github.sha), never to the tag being + # released, so a backfilled tag from before this script existed still + # finds it. The validator itself lives in scripts/release/ (unit-tested + # in tests/test_release_scripts.py) rather than inlined here. + - name: Check out release tooling from the workflow's own ref + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.sha }} + sparse-checkout: scripts/release + sparse-checkout-cone-mode: false + persist-credentials: false + path: .release-tooling + + # Fail loud unless wave-sdk AND every unconditional runtime dependency + # from the installed wheel's own Requires-Dist appear in packages[] with + # a real version. The old "packages[] is non-empty" check passed on the + # single self-describing entry the uninstalled scan produced. + - name: Validate the SBOM enumerates wave-sdk and every declared runtime dependency env: FILE: wave_sdk-${{ needs.verify.outputs.version }}.spdx.json - run: | - set -euo pipefail - test -s "$FILE" || { echo "::error::${FILE} is missing or empty"; exit 1; } - python3 -c " - import json, os - d = json.load(open(os.environ['FILE'])) - assert d.get('packages'), 'SBOM has no packages[]' - print('SBOM OK:', d.get('spdxVersion'), '-', len(d['packages']), 'package(s)') - " + VERSION: ${{ needs.verify.outputs.version }} + run: python3 .release-tooling/scripts/release/validate_sbom.py "$FILE" --target sbom-env --version "$VERSION" - name: Upload SBOM as a workflow artifact (for the release job to attach) uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 diff --git a/scripts/release/validate_sbom.py b/scripts/release/validate_sbom.py new file mode 100644 index 0000000..eaaa0f2 --- /dev/null +++ b/scripts/release/validate_sbom.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Fail loud unless an SPDX SBOM enumerates wave-sdk AND every unconditional +runtime dependency the installed wheel declares. + +Used by `release.yml`'s `sbom` job after syft has scanned the INSTALLED tree +(`pip install --target `). The check this replaces +only asserted `packages[]` was non-empty -- which an SBOM generated from a +bare, never-installed wheel satisfied with a single self-describing entry and +zero dependencies. That is the defect: an SBOM that predates install is a +claim, not an inventory. + +The floor is derived from the installed wheel's OWN metadata (`Requires-Dist` +in `-.dist-info/METADATA` under `--target`), not from a +hardcoded list, so it tracks `pyproject.toml` as dependencies change. Only +UNCONDITIONAL requirements count (no `;` marker): extras and +`python_version`-gated dependencies are legitimately absent from the release +runner's interpreter. Names are compared PEP 503-normalized on both sides so +`pydantic_core` (dist-info) and `pydantic-core` (syft) agree. + +Deliberately a plain stdlib script (not inlined YAML) so it can be run and +unit-tested locally against a real or synthetic SBOM without pushing a tag. +Like the other scripts here it is invoked from `.release-tooling/` and never +assumes anything about its own file location. + +Usage: + python3 scripts/release/validate_sbom.py wave_sdk-2.2.0.spdx.json \ + --target sbom-env --version 2.2.0 + +Exit 0 if the SBOM passes, 1 with a `::error::` line if it does not, 2 on bad +usage (missing file / no installed distribution to derive a floor from). +""" +from __future__ import annotations + +import argparse +import importlib.metadata as md +import json +import re +import sys +from pathlib import Path + +DIST_NAME = "wave-sdk" +_NO_VERSION = (None, "", "NOASSERTION") +_REQ_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") + + +class ValidationError(Exception): + """A checked, expected validation failure (exit 1).""" + + +class UsageError(Exception): + """Inputs are unusable rather than merely failing the check (exit 2).""" + + +def normalize(name: str) -> str: + """PEP 503 name normalization.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def installed_distribution(target: Path, name: str = DIST_NAME) -> md.Distribution: + """Find exactly one installed distribution called `name` under `target`. + + Discovers every `*.dist-info` on `target` and filters by normalized + metadata `Name`, rather than relying on `discover(name=...)`, whose own + normalization behaviour differs across Python versions. + """ + if not target.is_dir(): + raise UsageError(f"--target {target} is not a directory") + wanted = normalize(name) + found = [ + d for d in md.Distribution.discover(path=[str(target)]) + if normalize(d.metadata["Name"] or "") == wanted + ] + if len(found) != 1: + raise UsageError(f"expected exactly one installed {name} under {target}, found {len(found)}") + return found[0] + + +def unconditional_requirements(dist: md.Distribution) -> set[str]: + """Normalized names of the distribution's runtime deps that carry no marker.""" + names: set[str] = set() + for req in dist.requires or []: + if ";" in req: + continue + match = _REQ_NAME.match(req.strip()) + if match: + names.add(normalize(match.group(0))) + return names + + +def versioned_package_names(doc: dict) -> set[str]: + """Normalized names of SBOM packages[] entries that carry a real version.""" + return { + normalize(p["name"]) + for p in doc.get("packages") or [] + if p.get("name") and p.get("versionInfo") not in _NO_VERSION + } + + +def validate(sbom_file: Path, target: Path, version: str | None) -> str: + """Return a one-line summary on success; raise ValidationError/UsageError otherwise.""" + if not sbom_file.is_file() or sbom_file.stat().st_size == 0: + raise UsageError(f"{sbom_file} is missing or empty") + with sbom_file.open(encoding="utf-8") as fh: + doc = json.load(fh) + packages = doc.get("packages") or [] + if not packages: + raise ValidationError(f"{sbom_file} has an empty packages[] array") + + dist = installed_distribution(target) + if version is not None and dist.version != version: + raise ValidationError(f"installed {DIST_NAME} version {dist.version} != release version {version}") + + required = unconditional_requirements(dist) + if not required: + raise ValidationError( + f"installed {DIST_NAME} declares no unconditional runtime dependencies " + "-- refusing to validate against a floor of 0" + ) + + expected = {normalize(DIST_NAME)} | required + versioned = versioned_package_names(doc) + missing = sorted(expected - versioned) + floor = len(expected) + summary = ( + f"SBOM {doc.get('spdxVersion')}: {len(packages)} packages[], {len(versioned)} with a real " + f"versionInfo; floor {floor} ({DIST_NAME} + {len(required)} declared runtime deps: " + f"{', '.join(sorted(required))})" + ) + if missing: + raise ValidationError( + f"{summary}\n::error::sbom: missing versioned entries for: {', '.join(missing)} " + "-- syft likely scanned an uninstalled tree" + ) + if len(versioned) < floor: + raise ValidationError(f"{summary}\n::error::sbom: only {len(versioned)} versioned package(s), below the floor of {floor}") + return summary + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("sbom_file", type=Path, help="SPDX JSON produced from the installed tree") + parser.add_argument("--target", type=Path, required=True, help="directory `pip install --target` populated") + parser.add_argument("--version", default=None, help="release version the installed wheel must carry") + args = parser.parse_args(argv[1:]) + + try: + summary = validate(args.sbom_file, args.target, args.version) + except ValidationError as exc: + print(f"::error::sbom: {exc}", file=sys.stderr) + return 1 + except UsageError as exc: + print(f"::error::sbom: {exc}", file=sys.stderr) + return 2 + print(summary) + print(f"SBOM OK: {DIST_NAME} and every declared runtime dependency are enumerated") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/test_release_scripts.py b/tests/test_release_scripts.py index a8fc3c4..3f8bfa7 100644 --- a/tests/test_release_scripts.py +++ b/tests/test_release_scripts.py @@ -17,6 +17,7 @@ """ from __future__ import annotations +import json import subprocess import sys import textwrap @@ -134,3 +135,131 @@ def _current_repo_version() -> str: data = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text()) return data["project"]["version"] + + +# --------------------------------------------------------------------------- +# scripts/release/validate_sbom.py -- the SBOM must enumerate what ships. +# +# Regression under test: release.yml's `sbom` job used to point syft at a +# bare, never-installed wheel, producing a one-entry SBOM with zero runtime +# dependencies that the old "packages[] is non-empty" check waved through. +# The validator derives its floor from the INSTALLED wheel's Requires-Dist +# (under --target) and fails loud when any unconditional dependency is absent. +# --------------------------------------------------------------------------- +VALIDATE_SBOM = REPO_ROOT / "scripts" / "release" / "validate_sbom.py" + +_FAKE_REQUIRES = [ + "httpx>=0.25.0", + "pydantic>=2.0.0", + 'eval-type-backport>=0.2.0; python_version < "3.10"', # marker-gated: not part of the floor + 'websocket-client>=1.0.0; extra == "realtime"', # extra: not part of the floor +] + + +def _write_fake_target(tmp_path: Path, version: str = "9.9.9", requires=_FAKE_REQUIRES) -> Path: + """A minimal `pip install --target`-shaped tree: just wave-sdk's dist-info.""" + target = tmp_path / "sbom-env" + dist_info = target / f"wave_sdk-{version}.dist-info" + dist_info.mkdir(parents=True) + lines = ["Metadata-Version: 2.1", "Name: wave-sdk", f"Version: {version}"] + lines += [f"Requires-Dist: {req}" for req in requires] + (dist_info / "METADATA").write_text("\n".join(lines) + "\n") + return target + + +def _write_sbom(tmp_path: Path, packages: list) -> Path: + sbom = tmp_path / "wave_sdk-9.9.9.spdx.json" + sbom.write_text(json.dumps({"spdxVersion": "SPDX-2.3", "packages": packages})) + return sbom + + +def _pkg(name: str, version: str | None = "1.0.0") -> dict: + entry = {"name": name} + if version is not None: + entry["versionInfo"] = version + return entry + + +def _run_validate_sbom(*args: str, cwd: Path) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, str(VALIDATE_SBOM), *args], + cwd=cwd, + capture_output=True, + text=True, + timeout=30, + ) + + +def test_validate_sbom_passes_when_every_unconditional_dependency_is_enumerated(tmp_path): + target = _write_fake_target(tmp_path) + # syft reports dist-info names as-is; underscores must normalize to match. + sbom = _write_sbom(tmp_path, [_pkg("wave_sdk", "9.9.9"), _pkg("httpx"), _pkg("pydantic"), _pkg("pydantic_core")]) + + result = _run_validate_sbom(str(sbom), "--target", str(target), "--version", "9.9.9", cwd=tmp_path) + + assert result.returncode == 0, result.stderr + assert "floor 3 (wave-sdk + 2 declared runtime deps: httpx, pydantic)" in result.stdout + assert "SBOM OK" in result.stdout + + +def test_validate_sbom_fails_loud_on_the_uninstalled_wheel_shape(tmp_path): + """The exact defect: a single self-describing entry and no dependencies.""" + target = _write_fake_target(tmp_path) + sbom = _write_sbom(tmp_path, [_pkg("wave-sdk", "9.9.9")]) + + result = _run_validate_sbom(str(sbom), "--target", str(target), "--version", "9.9.9", cwd=tmp_path) + + assert result.returncode == 1 + assert "::error::sbom:" in result.stderr + assert "missing versioned entries for: httpx, pydantic" in result.stderr + + +def test_validate_sbom_ignores_entries_without_a_real_version(tmp_path): + target = _write_fake_target(tmp_path) + sbom = _write_sbom(tmp_path, [_pkg("wave-sdk", "9.9.9"), _pkg("httpx", "NOASSERTION"), _pkg("pydantic", None)]) + + result = _run_validate_sbom(str(sbom), "--target", str(target), cwd=tmp_path) + + assert result.returncode == 1 + assert "missing versioned entries for: httpx, pydantic" in result.stderr + + +def test_validate_sbom_fails_on_empty_packages(tmp_path): + target = _write_fake_target(tmp_path) + sbom = _write_sbom(tmp_path, []) + + result = _run_validate_sbom(str(sbom), "--target", str(target), cwd=tmp_path) + + assert result.returncode == 1 + assert "empty packages[]" in result.stderr + + +def test_validate_sbom_fails_on_release_version_mismatch(tmp_path): + target = _write_fake_target(tmp_path, version="9.9.9") + sbom = _write_sbom(tmp_path, [_pkg("wave-sdk", "9.9.9"), _pkg("httpx"), _pkg("pydantic")]) + + result = _run_validate_sbom(str(sbom), "--target", str(target), "--version", "1.0.0", cwd=tmp_path) + + assert result.returncode == 1 + assert "installed wave-sdk version 9.9.9 != release version 1.0.0" in result.stderr + + +def test_validate_sbom_refuses_a_floor_of_zero(tmp_path): + target = _write_fake_target(tmp_path, requires=['websocket-client>=1.0.0; extra == "realtime"']) + sbom = _write_sbom(tmp_path, [_pkg("wave-sdk", "9.9.9")]) + + result = _run_validate_sbom(str(sbom), "--target", str(target), cwd=tmp_path) + + assert result.returncode == 1 + assert "floor of 0" in result.stderr + + +def test_validate_sbom_usage_error_when_nothing_is_installed(tmp_path): + empty_target = tmp_path / "sbom-env" + empty_target.mkdir() + sbom = _write_sbom(tmp_path, [_pkg("wave-sdk", "9.9.9")]) + + result = _run_validate_sbom(str(sbom), "--target", str(empty_target), cwd=tmp_path) + + assert result.returncode == 2 + assert "expected exactly one installed wave-sdk" in result.stderr