From 8d2e4cb7ea84bbcb5fdf3f9b6ef497c6d105cb9c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 3 Oct 2026 05:59:01 +0100 Subject: [PATCH 1/2] ci: generate release benchmark projects with each version's own generator The release performance comparison generated its benchmark projects with the candidate's generator and compiled them with the previous release, so any generator syntax newer than that release (such as FUNCTION descriptions) failed the baseline before timing. The baseline now runs projects written by its own generator: the release tag's source is extracted with git archive (or taken from --baseline-source) and its write_pristine_projects runs with this interpreter, with cwd and PYTHONPATH set to that source, through the checkout generator runner shared with the same-runner compile ratio job. The candidate keeps its own generator. Generation failures stop the comparison with the generator's stderr tail, and the report and evidence record the baseline's generator. --- .../_helpers/measure.py | 33 ++++-- .../release_performance/_helpers/benchmark.py | 25 ++++- .../release_performance/_helpers/report.py | 3 + .../release_performance/_helpers/versions.py | 25 +++++ .../release_performance/_helpers/workflow.py | 38 ++++++- scripts/release_performance/constants.py | 10 ++ scripts/release_performance/main/compare.py | 10 ++ scripts/release_performance/models.py | 2 + .../release_performance/_test_types.py | 1 + .../scripts/release_performance/helpers.py | 35 ++++++ .../test_compare_release_performance.py | 25 +++++ .../_helpers/_test_types.py | 17 +++ .../release_performance/_helpers/helpers.py | 41 +++++++ .../_helpers/test_benchmark.py | 105 ++++++++++++++++++ .../_helpers/test_report.py | 3 + 15 files changed, 361 insertions(+), 12 deletions(-) create mode 100644 tests/e2e/scripts/release_performance/helpers.py create mode 100644 tests/unit/scripts/release_performance/_helpers/test_benchmark.py diff --git a/scripts/compile_performance_ratio/_helpers/measure.py b/scripts/compile_performance_ratio/_helpers/measure.py index 833667933e..000408bcfc 100644 --- a/scripts/compile_performance_ratio/_helpers/measure.py +++ b/scripts/compile_performance_ratio/_helpers/measure.py @@ -55,18 +55,37 @@ def write_benchmark_project(*, kind: str, project_dir: Path, models: int) -> Non ) +def run_checkout_generator( + *, + checkout: Path, + python: Path, + entry: str, + arguments: tuple[str, ...], + environment: dict[str, str], +) -> subprocess.CompletedProcess[str]: + """Run generator code from another checkout's source tree with this `python`.""" + + return subprocess.run( + [str(python), "-c", entry, *arguments], + cwd=checkout, + capture_output=True, + text=True, + env={**environment, PYTHONPATH_KEY: str(checkout)}, + check=False, + ) + + def write_base_benchmark_project( *, base_root: Path, python: Path, kind: str, project_dir: Path, models: int ) -> None: """Write the base project with the base checkout's own generator, run by `python`.""" - completed: subprocess.CompletedProcess[str] = subprocess.run( - [str(python), "-c", BASE_GENERATOR_ENTRY, kind, str(project_dir), str(models)], - cwd=base_root, - capture_output=True, - text=True, - env={**_compile_environment(), PYTHONPATH_KEY: str(base_root)}, - check=False, + completed: subprocess.CompletedProcess[str] = run_checkout_generator( + checkout=base_root, + python=python, + entry=BASE_GENERATOR_ENTRY, + arguments=(kind, str(project_dir), str(models)), + environment=_compile_environment(), ) if completed.returncode != 0: raise CompileComparisonError( diff --git a/scripts/release_performance/_helpers/benchmark.py b/scripts/release_performance/_helpers/benchmark.py index 7dfd8cbb53..7b12925a98 100644 --- a/scripts/release_performance/_helpers/benchmark.py +++ b/scripts/release_performance/_helpers/benchmark.py @@ -11,7 +11,9 @@ from pathlib import Path from typing import cast +from scripts.compile_performance_ratio._helpers.measure import run_checkout_generator from scripts.release_performance.constants import ( + BASELINE_GENERATOR_ENTRY, BUILD_PROJECT, BUILD_WARMUPS, COMMAND_TIMEOUT_SECONDS, @@ -20,6 +22,7 @@ EXCLUDED_ENVIRONMENT_PREFIX, INSPECTION_PROJECT, INSPECTION_WARMUPS, + PRISTINE_DIRECTORY, STDERR_TAIL_CHARACTERS, TIME_FORMAT, ) @@ -42,7 +45,7 @@ def write_pristine_projects(*, root: Path, inspection_models: int, build_models: int) -> Path: """Generate the inspection and build benchmarks once so every version reads the same files.""" - pristine: Path = root / "pristine" + pristine: Path = root / PRISTINE_DIRECTORY write_inspection_benchmark_project( project_dir=pristine / INSPECTION_PROJECT, model_count=inspection_models ) @@ -50,6 +53,26 @@ def write_pristine_projects(*, root: Path, inspection_models: int, build_models: return pristine +def write_baseline_pristine_projects( + *, source: Path, root: Path, inspection_models: int, build_models: int +) -> Path: + """Generate the baseline's benchmarks with the baseline source's own generator.""" + + completed: subprocess.CompletedProcess[str] = run_checkout_generator( + checkout=source, + python=Path(sys.executable).absolute(), + entry=BASELINE_GENERATOR_ENTRY, + arguments=(str(root), str(inspection_models), str(build_models)), + environment=_environment(), + ) + if completed.returncode != 0: + raise ReleasePerformanceError( + f"Baseline benchmark generation with the generator in {source} failed with exit " + f"{completed.returncode}:\n{completed.stderr[-STDERR_TAIL_CHARACTERS:]}" + ) + return root / PRISTINE_DIRECTORY + + def prepare_version_projects( *, version: InstalledVersion, pristine: Path, root: Path ) -> dict[str, Path]: diff --git a/scripts/release_performance/_helpers/report.py b/scripts/release_performance/_helpers/report.py index 350c0c3d50..2d09f050d1 100644 --- a/scripts/release_performance/_helpers/report.py +++ b/scripts/release_performance/_helpers/report.py @@ -31,6 +31,9 @@ def comparison_markdown(*, comparison: ReleaseComparison) -> str: f"{comparison.runs} interleaved runs per version; wall and CPU are medians, " "peak RSS is the worst run.", "", + "Benchmark projects are generated per side: the baseline runs projects from its own " + f"generator ({comparison.baseline_generator}), the candidate from the candidate's.", + "", f"Limits: wall {_percent(MAX_WALL_RATIO)} (ignored under " f"{MIN_WALL_REGRESSION_SECONDS:g} s), CPU {_percent(MAX_CPU_RATIO)} (ignored under " f"{MIN_CPU_REGRESSION_SECONDS:g} s), peak RSS {_percent(MAX_RSS_RATIO)} (ignored under " diff --git a/scripts/release_performance/_helpers/versions.py b/scripts/release_performance/_helpers/versions.py index 616405439b..25b5a8b0bb 100644 --- a/scripts/release_performance/_helpers/versions.py +++ b/scripts/release_performance/_helpers/versions.py @@ -3,12 +3,14 @@ from __future__ import annotations import hashlib +import io import json import platform import re import shutil import subprocess import sys +import tarfile import time import urllib.request from pathlib import Path @@ -22,6 +24,7 @@ PYPI_SIMPLE_JSON, PYPI_SIMPLE_URL, RELEASE_TAG_PATTERN, + RELEASE_TAG_PREFIX, STDERR_TAIL_CHARACTERS, UNIVERSAL_WHEEL_MACHINE, WHEEL_MACHINE_ALIASES, @@ -118,6 +121,28 @@ def tagged_versions(*, repo_dir: Path) -> tuple[str, ...]: ) +def release_source(*, version: str, repo_dir: Path, destination: Path) -> Path: + """Extract the source tree of release tag v from repo_dir into destination.""" + + tag: str = RELEASE_TAG_PREFIX + version + completed: subprocess.CompletedProcess[bytes] = subprocess.run( + ["git", "-C", str(repo_dir), "archive", "--format=tar", tag], + capture_output=True, + check=False, + ) + if completed.returncode != 0: + raise ReleasePerformanceError( + f"Release tag {tag} is not available in {repo_dir}, so the baseline's own benchmark " + "generator cannot be used; fetch the release tags or pass --baseline-source: " + + _tail(completed.stderr.decode("utf-8", errors="replace")) + ) + shutil.rmtree(destination, ignore_errors=True) + destination.mkdir(parents=True) + with tarfile.open(fileobj=io.BytesIO(completed.stdout)) as archive: + archive.extractall(destination, filter="data") + return destination + + def resolve_baseline(*, candidate: str, repo_dir: Path) -> str: """Return the highest published or not-yet-published tagged release below the candidate.""" diff --git a/scripts/release_performance/_helpers/workflow.py b/scripts/release_performance/_helpers/workflow.py index 5aefcaca53..6258cf6359 100644 --- a/scripts/release_performance/_helpers/workflow.py +++ b/scripts/release_performance/_helpers/workflow.py @@ -13,6 +13,7 @@ from scripts.release_performance._helpers.benchmark import ( compare_command, prepare_version_projects, + write_baseline_pristine_projects, write_pristine_projects, ) from scripts.release_performance._helpers.report import ( @@ -30,13 +31,17 @@ install_published, install_wheel, installed_version, + release_source, resolve_baseline, ) from scripts.release_performance.constants import ( + BASELINE_GENERATED_DIRECTORY, BASELINE_LABEL, BASELINE_PUBLICATION_WAIT_SECONDS, + BASELINE_SOURCE_DIRECTORY, BENCHMARK_COMMANDS, CANDIDATE_LABEL, + RELEASE_TAG_PREFIX, REPO_ROOT, ) from scripts.release_performance.exceptions import ReleasePerformanceError @@ -104,16 +109,40 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: file=sys.stderr, ) load_before: tuple[float, float, float] = os.getloadavg() - print("Generating benchmark projects", file=sys.stderr) - pristine: Path = write_pristine_projects( - root=root, inspection_models=options.inspection_models, build_models=options.build_models + print("Generating candidate benchmark projects", file=sys.stderr) + pristine: dict[str, Path] = { + CANDIDATE_LABEL: write_pristine_projects( + root=root, + inspection_models=options.inspection_models, + build_models=options.build_models, + ) + } + baseline_generator: str = ( + str(options.baseline_source) + if options.baseline_source is not None + else RELEASE_TAG_PREFIX + baseline.version + ) + print(f"Generating baseline benchmark projects with {baseline_generator}", file=sys.stderr) + pristine[BASELINE_LABEL] = write_baseline_pristine_projects( + source=( + options.baseline_source.absolute() + if options.baseline_source is not None + else release_source( + version=baseline.version, + repo_dir=REPO_ROOT, + destination=root / BASELINE_SOURCE_DIRECTORY, + ) + ), + root=root / BASELINE_GENERATED_DIRECTORY, + inspection_models=options.inspection_models, + build_models=options.build_models, ) versions: tuple[InstalledVersion, InstalledVersion] = (baseline, candidate) projects: dict[str, dict[str, Path]] = {} for version in versions: print(f"Warming caches for {version.label} {version.version}", file=sys.stderr) projects[version.label] = prepare_version_projects( - version=version, pristine=pristine, root=root + version=version, pristine=pristine[version.label], root=root ) measured: list[CommandComparison] = [] skipped: list[SkippedCommand] = [] @@ -133,6 +162,7 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: return ReleaseComparison( baseline_version=baseline.version, candidate_version=candidate.version, + baseline_generator=baseline_generator, runs=options.runs, runner=RunnerContext( cpu_model=cpu_model(), diff --git a/scripts/release_performance/constants.py b/scripts/release_performance/constants.py index 7ecb9cd08d..95301a25ba 100644 --- a/scripts/release_performance/constants.py +++ b/scripts/release_performance/constants.py @@ -42,6 +42,16 @@ TIME_FORMAT: str = "%e %M %U %S" COMPILE_COMMAND: str = "compile" STDERR_TAIL_CHARACTERS: int = 2_000 +PRISTINE_DIRECTORY: str = "pristine" +BASELINE_GENERATED_DIRECTORY: str = "baseline-generated" +BASELINE_SOURCE_DIRECTORY: str = "baseline-source" +RELEASE_TAG_PREFIX: str = "v" +BASELINE_GENERATOR_ENTRY: str = ( + "import sys; from pathlib import Path; " + "from scripts.release_performance._helpers.benchmark import write_pristine_projects; " + "write_pristine_projects(" + "root=Path(sys.argv[1]), inspection_models=int(sys.argv[2]), build_models=int(sys.argv[3]))" +) BENCHMARK_COMMANDS: tuple[BenchmarkCommand, ...] = ( BenchmarkCommand( diff --git a/scripts/release_performance/main/compare.py b/scripts/release_performance/main/compare.py index 73e65a04e2..d9993e8868 100644 --- a/scripts/release_performance/main/compare.py +++ b/scripts/release_performance/main/compare.py @@ -39,6 +39,15 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions: parser.add_argument("--python", default=DEFAULT_PYTHON) parser.add_argument("--inspection-models", type=int, default=DEFAULT_INSPECTION_MODELS) parser.add_argument("--build-models", type=int, default=DEFAULT_BUILD_MODELS) + parser.add_argument( + "--baseline-source", + type=Path, + default=None, + help=( + "Source tree whose benchmark generator writes the baseline's projects. Defaults to " + "the baseline's release tag extracted from this repository." + ), + ) parser.add_argument("--work-dir", type=Path, default=None) parser.add_argument("--output", type=Path, default=None) args: argparse.Namespace = parser.parse_args(argv) @@ -53,4 +62,5 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions: build_models=args.build_models, work_dir=args.work_dir, output=args.output, + baseline_source=args.baseline_source, ) diff --git a/scripts/release_performance/models.py b/scripts/release_performance/models.py index 2f71f3734a..1b5ea3a827 100644 --- a/scripts/release_performance/models.py +++ b/scripts/release_performance/models.py @@ -68,6 +68,7 @@ class RunnerContext: class ReleaseComparison: baseline_version: str candidate_version: str + baseline_generator: str runs: int runner: RunnerContext commands: tuple[CommandComparison, ...] @@ -84,3 +85,4 @@ class ComparisonOptions: build_models: int work_dir: Path | None output: Path | None + baseline_source: Path | None = None diff --git a/tests/e2e/scripts/release_performance/_test_types.py b/tests/e2e/scripts/release_performance/_test_types.py index 5c697e59ae..922f5e5296 100644 --- a/tests/e2e/scripts/release_performance/_test_types.py +++ b/tests/e2e/scripts/release_performance/_test_types.py @@ -12,3 +12,4 @@ class ReleasePerformanceTestCase: build_models: int expected_return_code: int expected_fragments: tuple[str, ...] + expected_marked_projects: tuple[str, ...] diff --git a/tests/e2e/scripts/release_performance/helpers.py b/tests/e2e/scripts/release_performance/helpers.py new file mode 100644 index 0000000000..46729069b3 --- /dev/null +++ b/tests/e2e/scripts/release_performance/helpers.py @@ -0,0 +1,35 @@ +"""Fixtures for release performance comparisons with per-side benchmark generators.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +BASELINE_MARKER: str = "BASELINE_MARKER" +_MARKING_WRAPPER: str = """ + +_unmarked_write_pristine_projects = write_pristine_projects + + +def write_pristine_projects(*, root: Path, inspection_models: int, build_models: int) -> Path: + pristine: Path = _unmarked_write_pristine_projects( + root=root, inspection_models=inspection_models, build_models=build_models + ) + for project in pristine.iterdir(): + (project / "BASELINE_MARKER").write_text("baseline", encoding="utf-8") + return pristine +""" + + +def write_marked_baseline_source(*, repo_root: Path, destination: Path) -> Path: + """Copy this checkout's generators and mark every project the copy generates.""" + + for name in ("scripts", "tests"): + _ = shutil.copytree( + repo_root / name, destination / name, ignore=shutil.ignore_patterns("__pycache__") + ) + benchmark: Path = destination / "scripts" / "release_performance" / "_helpers" / "benchmark.py" + _ = benchmark.write_text( + benchmark.read_text(encoding="utf-8") + _MARKING_WRAPPER, encoding="utf-8" + ) + return destination diff --git a/tests/e2e/scripts/release_performance/test_compare_release_performance.py b/tests/e2e/scripts/release_performance/test_compare_release_performance.py index 1395cc1af0..7380ce3776 100644 --- a/tests/e2e/scripts/release_performance/test_compare_release_performance.py +++ b/tests/e2e/scripts/release_performance/test_compare_release_performance.py @@ -12,6 +12,10 @@ from scripts.release_performance.constants import BENCHMARK_COMMANDS from tests.e2e.scripts.release_performance._test_types import ReleasePerformanceTestCase +from tests.e2e.scripts.release_performance.helpers import ( + BASELINE_MARKER, + write_marked_baseline_source, +) _REPO_ROOT: Path = Path(__file__).resolve().parents[4] @@ -28,8 +32,14 @@ "| `plan --json` |", "| `build (empty warehouse)` |", "| `lineage column trace` |", + "Benchmark projects are generated per side", "**Result: passed.**", ), + expected_marked_projects=( + "baseline/build", + "baseline/build-template", + "baseline/inspection", + ), ), ), ids=lambda case: case.description, @@ -42,6 +52,9 @@ def test_given_two_installations_when_comparing_release_performance_then_reports environment: dict[str, str] = dict(os.environ) environment["GITHUB_STEP_SUMMARY"] = str(summary) installation: str = str(Path(sys.executable).parent.parent) + baseline_source: Path = write_marked_baseline_source( + repo_root=_REPO_ROOT, destination=tmp_path / "baseline-source" + ) result: subprocess.CompletedProcess[str] = subprocess.run( [ @@ -58,6 +71,8 @@ def test_given_two_installations_when_comparing_release_performance_then_reports str(test_case.inspection_models), "--build-models", str(test_case.build_models), + "--baseline-source", + str(baseline_source), "--work-dir", str(tmp_path / "work"), "--output", @@ -70,6 +85,16 @@ def test_given_two_installations_when_comparing_release_performance_then_reports check=False, ) + work: Path = tmp_path / "work" + assert ( + tuple( + sorted( + marker.parent.relative_to(work).as_posix() + for marker in work.glob(f"*/*/{BASELINE_MARKER}") + ) + ) + == test_case.expected_marked_projects + ) output: str = result.stdout + result.stderr assert result.returncode == test_case.expected_return_code, output markdown: str = summary.read_text(encoding="utf-8") diff --git a/tests/unit/scripts/release_performance/_helpers/_test_types.py b/tests/unit/scripts/release_performance/_helpers/_test_types.py index 3e0b2aa9e6..5a7bf9c666 100644 --- a/tests/unit/scripts/release_performance/_helpers/_test_types.py +++ b/tests/unit/scripts/release_performance/_helpers/_test_types.py @@ -60,3 +60,20 @@ class ChooseBaselineTestCase: files: tuple[dict[str, object], ...] tags: tuple[str, ...] expected_baseline: str | None + + +@dataclass(frozen=True) +class BaselineGeneratorTestCase: + description: str + generator: str + expected_projects: tuple[str, ...] + expected_error_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class ReleaseSourceTestCase: + description: str + tag: str + requested_version: str + expected_files: tuple[str, ...] + expected_error_fragments: tuple[str, ...] diff --git a/tests/unit/scripts/release_performance/_helpers/helpers.py b/tests/unit/scripts/release_performance/_helpers/helpers.py index 7fa7c1385f..9b9b7d20b8 100644 --- a/tests/unit/scripts/release_performance/_helpers/helpers.py +++ b/tests/unit/scripts/release_performance/_helpers/helpers.py @@ -1,8 +1,23 @@ """Builders for release performance comparison test data.""" +import subprocess +from pathlib import Path + from scripts.release_performance.models import CommandComparison, CommandSample MIB: int = 1024 * 1024 +MARKER_GENERATOR: str = """from pathlib import Path + + +def write_pristine_projects(*, root: Path, inspection_models: int, build_models: int) -> Path: + for name, models in (("inspection", inspection_models), ("build", build_models)): + (root / "pristine" / name).mkdir(parents=True) + (root / "pristine" / name / "BASELINE_MARKER").write_text(str(models)) + return root / "pristine" +""" +FAILING_GENERATOR: str = """def write_pristine_projects(*, root, inspection_models, build_models): + raise RuntimeError("baseline generator exploded") +""" def comparison( @@ -29,3 +44,29 @@ def release_file(*, filename: str) -> dict[str, object]: def _sample(*, values: tuple[float, float, int]) -> CommandSample: wall, cpu, rss_mib = values return CommandSample(wall_seconds=wall, cpu_seconds=cpu, peak_rss_bytes=rss_mib * MIB) + + +def write_baseline_source(*, root: Path, generator: str) -> Path: + """Write a source tree whose only content is a release benchmark generator.""" + + helpers: Path = root / "scripts" / "release_performance" / "_helpers" + helpers.mkdir(parents=True) + for package in (root / "scripts", helpers.parent, helpers): + _ = (package / "__init__.py").write_text("", encoding="utf-8") + _ = (helpers / "benchmark.py").write_text(generator, encoding="utf-8") + return root + + +def tagged_repository(*, root: Path, tag: str) -> Path: + """Create a one-commit git repository with a tracked file and the given tag.""" + + root.mkdir(parents=True) + _ = (root / "release.txt").write_text(tag, encoding="utf-8") + for command in ( + ("init", "--quiet"), + ("add", "release.txt"), + ("-c", "user.name=t", "-c", "user.email=t@example.com", "commit", "--quiet", "-m", "r"), + ("tag", tag), + ): + _ = subprocess.run(["git", "-C", str(root), *command], check=True, capture_output=True) + return root diff --git a/tests/unit/scripts/release_performance/_helpers/test_benchmark.py b/tests/unit/scripts/release_performance/_helpers/test_benchmark.py new file mode 100644 index 0000000000..bce6daa78c --- /dev/null +++ b/tests/unit/scripts/release_performance/_helpers/test_benchmark.py @@ -0,0 +1,105 @@ +"""Unit tests for generating each version's release benchmark projects.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.release_performance._helpers.benchmark import write_baseline_pristine_projects +from scripts.release_performance._helpers.versions import release_source +from scripts.release_performance.exceptions import ReleasePerformanceError +from tests.unit.scripts.release_performance._helpers._test_types import ( + BaselineGeneratorTestCase, + ReleaseSourceTestCase, +) +from tests.unit.scripts.release_performance._helpers.helpers import ( + FAILING_GENERATOR, + MARKER_GENERATOR, + tagged_repository, + write_baseline_source, +) + + +@pytest.mark.parametrize( + "test_case", + [ + BaselineGeneratorTestCase( + description="the baseline source's own generator writes the baseline projects", + generator=MARKER_GENERATOR, + expected_projects=("build", "inspection"), + expected_error_fragments=(), + ), + BaselineGeneratorTestCase( + description="a failing baseline generator is reported with its stderr", + generator=FAILING_GENERATOR, + expected_projects=(), + expected_error_fragments=( + "Baseline benchmark generation with the generator in", + "baseline generator exploded", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_baseline_source_when_generating_then_uses_its_own_generator( + test_case: BaselineGeneratorTestCase, tmp_path: Path +) -> None: + source: Path = write_baseline_source(root=tmp_path / "source", generator=test_case.generator) + root: Path = tmp_path / "generated" + error: str = "" + + try: + _ = write_baseline_pristine_projects( + source=source, root=root, inspection_models=30, build_models=10 + ) + except ReleasePerformanceError as raised: + error = str(raised) + + pristine: Path = root / "pristine" + assert tuple(sorted(path.parent.name for path in pristine.glob("*/BASELINE_MARKER"))) == ( + test_case.expected_projects + ) + assert all(fragment in error for fragment in test_case.expected_error_fragments), error + + +@pytest.mark.parametrize( + "test_case", + [ + ReleaseSourceTestCase( + description="the release tag's tree is extracted", + tag="v1.2.3", + requested_version="1.2.3", + expected_files=("release.txt",), + expected_error_fragments=(), + ), + ReleaseSourceTestCase( + description="a missing release tag explains how to provide the source", + tag="v1.2.3", + requested_version="1.2.2", + expected_files=(), + expected_error_fragments=("Release tag v1.2.2 is not available", "--baseline-source"), + ), + ], + ids=lambda case: case.description, +) +def test_given_release_tag_when_extracting_source_then_returns_its_tree( + test_case: ReleaseSourceTestCase, tmp_path: Path +) -> None: + repository: Path = tagged_repository(root=tmp_path / "repo", tag=test_case.tag) + destination: Path = tmp_path / "source" + error: str = "" + + try: + _ = release_source( + version=test_case.requested_version, repo_dir=repository, destination=destination + ) + except ReleasePerformanceError as raised: + error = str(raised) + + assert tuple(sorted(path.name for path in destination.glob("*"))) == test_case.expected_files + assert all(fragment in error for fragment in test_case.expected_error_fragments), error + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/release_performance/_helpers/test_report.py b/tests/unit/scripts/release_performance/_helpers/test_report.py index 6a0cfc8b7f..5ba5bffe62 100644 --- a/tests/unit/scripts/release_performance/_helpers/test_report.py +++ b/tests/unit/scripts/release_performance/_helpers/test_report.py @@ -31,6 +31,8 @@ "### Release performance: 0.126.2 (candidate) vs 0.126.1 (baseline)", "Same runner: Example CPU, 4 CPUs; load average 0.50/0.40/0.30 before", "3 interleaved runs per version", + "Benchmark projects are generated per side: the baseline runs projects from its " + "own generator (v0.126.1), the candidate from the candidate's.", "Limits: wall +25% (ignored under 0.5 s), CPU +25% (ignored under 0.5 s), " "peak RSS +25% (ignored under 32 MiB).", "| `compile (warm cache)` | 6.00 → 6.10 | 1.017x | 6.50 → 6.40 | 0.985x " @@ -82,6 +84,7 @@ def test_given_release_comparison_when_rendering_summary_then_reports_each_comma release: ReleaseComparison = ReleaseComparison( baseline_version="0.126.1", candidate_version="0.126.2", + baseline_generator="v0.126.1", runs=3, runner=RunnerContext( cpu_model="Example CPU", From 6e875b2eff3f5242da802c82a8a078687cd143d2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 3 Oct 2026 06:10:03 +0100 Subject: [PATCH 2/2] test: make the release-performance self-comparison E2E independent of runner noise The E2E compares one build with itself using a single run, so asserting a passed verdict depended on wall-time noise (a 1.31x wall ratio at 0.99x CPU failed CI). It now asserts the report structure, every command row, the per-side generator markers and the recorded evidence, and accepts either verdict as long as the exit code matches it. The production 1.25x limits are unchanged; the verdict unit tests gain fixed-timing cases for a wall-only regression at CPU parity and a wall-only slowdown just under the limit. Fixes CHI-481 --- .../scripts/release_performance/_test_types.py | 2 +- .../test_compare_release_performance.py | 10 ++++++---- .../_helpers/test_verdict.py | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/tests/e2e/scripts/release_performance/_test_types.py b/tests/e2e/scripts/release_performance/_test_types.py index 922f5e5296..8143febd53 100644 --- a/tests/e2e/scripts/release_performance/_test_types.py +++ b/tests/e2e/scripts/release_performance/_test_types.py @@ -10,6 +10,6 @@ class ReleasePerformanceTestCase: description: str inspection_models: int build_models: int - expected_return_code: int + expected_outcomes: tuple[tuple[int, bool], ...] expected_fragments: tuple[str, ...] expected_marked_projects: tuple[str, ...] diff --git a/tests/e2e/scripts/release_performance/test_compare_release_performance.py b/tests/e2e/scripts/release_performance/test_compare_release_performance.py index 7380ce3776..a28887ff85 100644 --- a/tests/e2e/scripts/release_performance/test_compare_release_performance.py +++ b/tests/e2e/scripts/release_performance/test_compare_release_performance.py @@ -24,16 +24,16 @@ "test_case", ( ReleasePerformanceTestCase( - description="the same build as candidate and baseline passes every command", + description="the same build as candidate and baseline reports every command", inspection_models=300, build_models=100, - expected_return_code=0, + expected_outcomes=((0, True), (1, False)), expected_fragments=( "| `plan --json` |", "| `build (empty warehouse)` |", "| `lineage column trace` |", "Benchmark projects are generated per side", - "**Result: passed.**", + "**Result: ", ), expected_marked_projects=( "baseline/build", @@ -96,10 +96,12 @@ def test_given_two_installations_when_comparing_release_performance_then_reports == test_case.expected_marked_projects ) output: str = result.stdout + result.stderr - assert result.returncode == test_case.expected_return_code, output markdown: str = summary.read_text(encoding="utf-8") for fragment in test_case.expected_fragments: assert fragment in markdown + assert (result.returncode, "**Result: passed.**" in markdown) in ( + test_case.expected_outcomes + ), output recorded: dict[str, object] = json.loads(evidence.read_text(encoding="utf-8")) assert [command["name"] for command in recorded["commands"]] == [ command.name for command in BENCHMARK_COMMANDS diff --git a/tests/unit/scripts/release_performance/_helpers/test_verdict.py b/tests/unit/scripts/release_performance/_helpers/test_verdict.py index a46d3a46ea..3a4bc8ca5e 100644 --- a/tests/unit/scripts/release_performance/_helpers/test_verdict.py +++ b/tests/unit/scripts/release_performance/_helpers/test_verdict.py @@ -63,6 +63,24 @@ ), expected_regressed=(False, True, False), ), + MetricVerdictTestCase( + description="a wall-only regression at CPU parity fails on wall alone", + comparison=comparison( + name="build (empty warehouse)", + baseline=((10.0, 6.4, 330),) * 3, + candidate=((13.14, 6.36, 330),) * 3, + ), + expected_regressed=(True, False, False), + ), + MetricVerdictTestCase( + description="a wall-only slowdown under the ratio limit at CPU parity passes", + comparison=comparison( + name="build (empty warehouse)", + baseline=((10.0, 6.4, 330),) * 3, + candidate=((12.4, 6.4, 330),) * 3, + ), + expected_regressed=(False, False, False), + ), MetricVerdictTestCase( description="one slow outlier does not move the median time", comparison=comparison(