diff --git a/scripts/compile_performance_ratio/_helpers/measure.py b/scripts/compile_performance_ratio/_helpers/measure.py index 833667933e..000408bcfc 100644 --- a/scripts/compile_performance_ratio/_helpers/measure.py +++ b/scripts/compile_performance_ratio/_helpers/measure.py @@ -55,18 +55,37 @@ def write_benchmark_project(*, kind: str, project_dir: Path, models: int) -> Non ) +def run_checkout_generator( + *, + checkout: Path, + python: Path, + entry: str, + arguments: tuple[str, ...], + environment: dict[str, str], +) -> subprocess.CompletedProcess[str]: + """Run generator code from another checkout's source tree with this `python`.""" + + return subprocess.run( + [str(python), "-c", entry, *arguments], + cwd=checkout, + capture_output=True, + text=True, + env={**environment, PYTHONPATH_KEY: str(checkout)}, + check=False, + ) + + def write_base_benchmark_project( *, base_root: Path, python: Path, kind: str, project_dir: Path, models: int ) -> None: """Write the base project with the base checkout's own generator, run by `python`.""" - completed: subprocess.CompletedProcess[str] = subprocess.run( - [str(python), "-c", BASE_GENERATOR_ENTRY, kind, str(project_dir), str(models)], - cwd=base_root, - capture_output=True, - text=True, - env={**_compile_environment(), PYTHONPATH_KEY: str(base_root)}, - check=False, + completed: subprocess.CompletedProcess[str] = run_checkout_generator( + checkout=base_root, + python=python, + entry=BASE_GENERATOR_ENTRY, + arguments=(kind, str(project_dir), str(models)), + environment=_compile_environment(), ) if completed.returncode != 0: raise CompileComparisonError( diff --git a/scripts/release_performance/_helpers/benchmark.py b/scripts/release_performance/_helpers/benchmark.py index 7dfd8cbb53..7b12925a98 100644 --- a/scripts/release_performance/_helpers/benchmark.py +++ b/scripts/release_performance/_helpers/benchmark.py @@ -11,7 +11,9 @@ from pathlib import Path from typing import cast +from scripts.compile_performance_ratio._helpers.measure import run_checkout_generator from scripts.release_performance.constants import ( + BASELINE_GENERATOR_ENTRY, BUILD_PROJECT, BUILD_WARMUPS, COMMAND_TIMEOUT_SECONDS, @@ -20,6 +22,7 @@ EXCLUDED_ENVIRONMENT_PREFIX, INSPECTION_PROJECT, INSPECTION_WARMUPS, + PRISTINE_DIRECTORY, STDERR_TAIL_CHARACTERS, TIME_FORMAT, ) @@ -42,7 +45,7 @@ def write_pristine_projects(*, root: Path, inspection_models: int, build_models: int) -> Path: """Generate the inspection and build benchmarks once so every version reads the same files.""" - pristine: Path = root / "pristine" + pristine: Path = root / PRISTINE_DIRECTORY write_inspection_benchmark_project( project_dir=pristine / INSPECTION_PROJECT, model_count=inspection_models ) @@ -50,6 +53,26 @@ def write_pristine_projects(*, root: Path, inspection_models: int, build_models: return pristine +def write_baseline_pristine_projects( + *, source: Path, root: Path, inspection_models: int, build_models: int +) -> Path: + """Generate the baseline's benchmarks with the baseline source's own generator.""" + + completed: subprocess.CompletedProcess[str] = run_checkout_generator( + checkout=source, + python=Path(sys.executable).absolute(), + entry=BASELINE_GENERATOR_ENTRY, + arguments=(str(root), str(inspection_models), str(build_models)), + environment=_environment(), + ) + if completed.returncode != 0: + raise ReleasePerformanceError( + f"Baseline benchmark generation with the generator in {source} failed with exit " + f"{completed.returncode}:\n{completed.stderr[-STDERR_TAIL_CHARACTERS:]}" + ) + return root / PRISTINE_DIRECTORY + + def prepare_version_projects( *, version: InstalledVersion, pristine: Path, root: Path ) -> dict[str, Path]: diff --git a/scripts/release_performance/_helpers/report.py b/scripts/release_performance/_helpers/report.py index 350c0c3d50..2d09f050d1 100644 --- a/scripts/release_performance/_helpers/report.py +++ b/scripts/release_performance/_helpers/report.py @@ -31,6 +31,9 @@ def comparison_markdown(*, comparison: ReleaseComparison) -> str: f"{comparison.runs} interleaved runs per version; wall and CPU are medians, " "peak RSS is the worst run.", "", + "Benchmark projects are generated per side: the baseline runs projects from its own " + f"generator ({comparison.baseline_generator}), the candidate from the candidate's.", + "", f"Limits: wall {_percent(MAX_WALL_RATIO)} (ignored under " f"{MIN_WALL_REGRESSION_SECONDS:g} s), CPU {_percent(MAX_CPU_RATIO)} (ignored under " f"{MIN_CPU_REGRESSION_SECONDS:g} s), peak RSS {_percent(MAX_RSS_RATIO)} (ignored under " diff --git a/scripts/release_performance/_helpers/versions.py b/scripts/release_performance/_helpers/versions.py index 616405439b..25b5a8b0bb 100644 --- a/scripts/release_performance/_helpers/versions.py +++ b/scripts/release_performance/_helpers/versions.py @@ -3,12 +3,14 @@ from __future__ import annotations import hashlib +import io import json import platform import re import shutil import subprocess import sys +import tarfile import time import urllib.request from pathlib import Path @@ -22,6 +24,7 @@ PYPI_SIMPLE_JSON, PYPI_SIMPLE_URL, RELEASE_TAG_PATTERN, + RELEASE_TAG_PREFIX, STDERR_TAIL_CHARACTERS, UNIVERSAL_WHEEL_MACHINE, WHEEL_MACHINE_ALIASES, @@ -118,6 +121,28 @@ def tagged_versions(*, repo_dir: Path) -> tuple[str, ...]: ) +def release_source(*, version: str, repo_dir: Path, destination: Path) -> Path: + """Extract the source tree of release tag v from repo_dir into destination.""" + + tag: str = RELEASE_TAG_PREFIX + version + completed: subprocess.CompletedProcess[bytes] = subprocess.run( + ["git", "-C", str(repo_dir), "archive", "--format=tar", tag], + capture_output=True, + check=False, + ) + if completed.returncode != 0: + raise ReleasePerformanceError( + f"Release tag {tag} is not available in {repo_dir}, so the baseline's own benchmark " + "generator cannot be used; fetch the release tags or pass --baseline-source: " + + _tail(completed.stderr.decode("utf-8", errors="replace")) + ) + shutil.rmtree(destination, ignore_errors=True) + destination.mkdir(parents=True) + with tarfile.open(fileobj=io.BytesIO(completed.stdout)) as archive: + archive.extractall(destination, filter="data") + return destination + + def resolve_baseline(*, candidate: str, repo_dir: Path) -> str: """Return the highest published or not-yet-published tagged release below the candidate.""" diff --git a/scripts/release_performance/_helpers/workflow.py b/scripts/release_performance/_helpers/workflow.py index 5aefcaca53..6258cf6359 100644 --- a/scripts/release_performance/_helpers/workflow.py +++ b/scripts/release_performance/_helpers/workflow.py @@ -13,6 +13,7 @@ from scripts.release_performance._helpers.benchmark import ( compare_command, prepare_version_projects, + write_baseline_pristine_projects, write_pristine_projects, ) from scripts.release_performance._helpers.report import ( @@ -30,13 +31,17 @@ install_published, install_wheel, installed_version, + release_source, resolve_baseline, ) from scripts.release_performance.constants import ( + BASELINE_GENERATED_DIRECTORY, BASELINE_LABEL, BASELINE_PUBLICATION_WAIT_SECONDS, + BASELINE_SOURCE_DIRECTORY, BENCHMARK_COMMANDS, CANDIDATE_LABEL, + RELEASE_TAG_PREFIX, REPO_ROOT, ) from scripts.release_performance.exceptions import ReleasePerformanceError @@ -104,16 +109,40 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: file=sys.stderr, ) load_before: tuple[float, float, float] = os.getloadavg() - print("Generating benchmark projects", file=sys.stderr) - pristine: Path = write_pristine_projects( - root=root, inspection_models=options.inspection_models, build_models=options.build_models + print("Generating candidate benchmark projects", file=sys.stderr) + pristine: dict[str, Path] = { + CANDIDATE_LABEL: write_pristine_projects( + root=root, + inspection_models=options.inspection_models, + build_models=options.build_models, + ) + } + baseline_generator: str = ( + str(options.baseline_source) + if options.baseline_source is not None + else RELEASE_TAG_PREFIX + baseline.version + ) + print(f"Generating baseline benchmark projects with {baseline_generator}", file=sys.stderr) + pristine[BASELINE_LABEL] = write_baseline_pristine_projects( + source=( + options.baseline_source.absolute() + if options.baseline_source is not None + else release_source( + version=baseline.version, + repo_dir=REPO_ROOT, + destination=root / BASELINE_SOURCE_DIRECTORY, + ) + ), + root=root / BASELINE_GENERATED_DIRECTORY, + inspection_models=options.inspection_models, + build_models=options.build_models, ) versions: tuple[InstalledVersion, InstalledVersion] = (baseline, candidate) projects: dict[str, dict[str, Path]] = {} for version in versions: print(f"Warming caches for {version.label} {version.version}", file=sys.stderr) projects[version.label] = prepare_version_projects( - version=version, pristine=pristine, root=root + version=version, pristine=pristine[version.label], root=root ) measured: list[CommandComparison] = [] skipped: list[SkippedCommand] = [] @@ -133,6 +162,7 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: return ReleaseComparison( baseline_version=baseline.version, candidate_version=candidate.version, + baseline_generator=baseline_generator, runs=options.runs, runner=RunnerContext( cpu_model=cpu_model(), diff --git a/scripts/release_performance/constants.py b/scripts/release_performance/constants.py index 7ecb9cd08d..95301a25ba 100644 --- a/scripts/release_performance/constants.py +++ b/scripts/release_performance/constants.py @@ -42,6 +42,16 @@ TIME_FORMAT: str = "%e %M %U %S" COMPILE_COMMAND: str = "compile" STDERR_TAIL_CHARACTERS: int = 2_000 +PRISTINE_DIRECTORY: str = "pristine" +BASELINE_GENERATED_DIRECTORY: str = "baseline-generated" +BASELINE_SOURCE_DIRECTORY: str = "baseline-source" +RELEASE_TAG_PREFIX: str = "v" +BASELINE_GENERATOR_ENTRY: str = ( + "import sys; from pathlib import Path; " + "from scripts.release_performance._helpers.benchmark import write_pristine_projects; " + "write_pristine_projects(" + "root=Path(sys.argv[1]), inspection_models=int(sys.argv[2]), build_models=int(sys.argv[3]))" +) BENCHMARK_COMMANDS: tuple[BenchmarkCommand, ...] = ( BenchmarkCommand( diff --git a/scripts/release_performance/main/compare.py b/scripts/release_performance/main/compare.py index 73e65a04e2..d9993e8868 100644 --- a/scripts/release_performance/main/compare.py +++ b/scripts/release_performance/main/compare.py @@ -39,6 +39,15 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions: parser.add_argument("--python", default=DEFAULT_PYTHON) parser.add_argument("--inspection-models", type=int, default=DEFAULT_INSPECTION_MODELS) parser.add_argument("--build-models", type=int, default=DEFAULT_BUILD_MODELS) + parser.add_argument( + "--baseline-source", + type=Path, + default=None, + help=( + "Source tree whose benchmark generator writes the baseline's projects. Defaults to " + "the baseline's release tag extracted from this repository." + ), + ) parser.add_argument("--work-dir", type=Path, default=None) parser.add_argument("--output", type=Path, default=None) args: argparse.Namespace = parser.parse_args(argv) @@ -53,4 +62,5 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions: build_models=args.build_models, work_dir=args.work_dir, output=args.output, + baseline_source=args.baseline_source, ) diff --git a/scripts/release_performance/models.py b/scripts/release_performance/models.py index 2f71f3734a..1b5ea3a827 100644 --- a/scripts/release_performance/models.py +++ b/scripts/release_performance/models.py @@ -68,6 +68,7 @@ class RunnerContext: class ReleaseComparison: baseline_version: str candidate_version: str + baseline_generator: str runs: int runner: RunnerContext commands: tuple[CommandComparison, ...] @@ -84,3 +85,4 @@ class ComparisonOptions: build_models: int work_dir: Path | None output: Path | None + baseline_source: Path | None = None diff --git a/tests/e2e/scripts/release_performance/_test_types.py b/tests/e2e/scripts/release_performance/_test_types.py index 5c697e59ae..8143febd53 100644 --- a/tests/e2e/scripts/release_performance/_test_types.py +++ b/tests/e2e/scripts/release_performance/_test_types.py @@ -10,5 +10,6 @@ class ReleasePerformanceTestCase: description: str inspection_models: int build_models: int - expected_return_code: int + expected_outcomes: tuple[tuple[int, bool], ...] expected_fragments: tuple[str, ...] + expected_marked_projects: tuple[str, ...] diff --git a/tests/e2e/scripts/release_performance/helpers.py b/tests/e2e/scripts/release_performance/helpers.py new file mode 100644 index 0000000000..46729069b3 --- /dev/null +++ b/tests/e2e/scripts/release_performance/helpers.py @@ -0,0 +1,35 @@ +"""Fixtures for release performance comparisons with per-side benchmark generators.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +BASELINE_MARKER: str = "BASELINE_MARKER" +_MARKING_WRAPPER: str = """ + +_unmarked_write_pristine_projects = write_pristine_projects + + +def write_pristine_projects(*, root: Path, inspection_models: int, build_models: int) -> Path: + pristine: Path = _unmarked_write_pristine_projects( + root=root, inspection_models=inspection_models, build_models=build_models + ) + for project in pristine.iterdir(): + (project / "BASELINE_MARKER").write_text("baseline", encoding="utf-8") + return pristine +""" + + +def write_marked_baseline_source(*, repo_root: Path, destination: Path) -> Path: + """Copy this checkout's generators and mark every project the copy generates.""" + + for name in ("scripts", "tests"): + _ = shutil.copytree( + repo_root / name, destination / name, ignore=shutil.ignore_patterns("__pycache__") + ) + benchmark: Path = destination / "scripts" / "release_performance" / "_helpers" / "benchmark.py" + _ = benchmark.write_text( + benchmark.read_text(encoding="utf-8") + _MARKING_WRAPPER, encoding="utf-8" + ) + return destination diff --git a/tests/e2e/scripts/release_performance/test_compare_release_performance.py b/tests/e2e/scripts/release_performance/test_compare_release_performance.py index 1395cc1af0..a28887ff85 100644 --- a/tests/e2e/scripts/release_performance/test_compare_release_performance.py +++ b/tests/e2e/scripts/release_performance/test_compare_release_performance.py @@ -12,6 +12,10 @@ from scripts.release_performance.constants import BENCHMARK_COMMANDS from tests.e2e.scripts.release_performance._test_types import ReleasePerformanceTestCase +from tests.e2e.scripts.release_performance.helpers import ( + BASELINE_MARKER, + write_marked_baseline_source, +) _REPO_ROOT: Path = Path(__file__).resolve().parents[4] @@ -20,15 +24,21 @@ "test_case", ( ReleasePerformanceTestCase( - description="the same build as candidate and baseline passes every command", + description="the same build as candidate and baseline reports every command", inspection_models=300, build_models=100, - expected_return_code=0, + expected_outcomes=((0, True), (1, False)), expected_fragments=( "| `plan --json` |", "| `build (empty warehouse)` |", "| `lineage column trace` |", - "**Result: passed.**", + "Benchmark projects are generated per side", + "**Result: ", + ), + expected_marked_projects=( + "baseline/build", + "baseline/build-template", + "baseline/inspection", ), ), ), @@ -42,6 +52,9 @@ def test_given_two_installations_when_comparing_release_performance_then_reports environment: dict[str, str] = dict(os.environ) environment["GITHUB_STEP_SUMMARY"] = str(summary) installation: str = str(Path(sys.executable).parent.parent) + baseline_source: Path = write_marked_baseline_source( + repo_root=_REPO_ROOT, destination=tmp_path / "baseline-source" + ) result: subprocess.CompletedProcess[str] = subprocess.run( [ @@ -58,6 +71,8 @@ def test_given_two_installations_when_comparing_release_performance_then_reports str(test_case.inspection_models), "--build-models", str(test_case.build_models), + "--baseline-source", + str(baseline_source), "--work-dir", str(tmp_path / "work"), "--output", @@ -70,11 +85,23 @@ def test_given_two_installations_when_comparing_release_performance_then_reports check=False, ) + work: Path = tmp_path / "work" + assert ( + tuple( + sorted( + marker.parent.relative_to(work).as_posix() + for marker in work.glob(f"*/*/{BASELINE_MARKER}") + ) + ) + == test_case.expected_marked_projects + ) output: str = result.stdout + result.stderr - assert result.returncode == test_case.expected_return_code, output markdown: str = summary.read_text(encoding="utf-8") for fragment in test_case.expected_fragments: assert fragment in markdown + assert (result.returncode, "**Result: passed.**" in markdown) in ( + test_case.expected_outcomes + ), output recorded: dict[str, object] = json.loads(evidence.read_text(encoding="utf-8")) assert [command["name"] for command in recorded["commands"]] == [ command.name for command in BENCHMARK_COMMANDS diff --git a/tests/unit/scripts/release_performance/_helpers/_test_types.py b/tests/unit/scripts/release_performance/_helpers/_test_types.py index 3e0b2aa9e6..5a7bf9c666 100644 --- a/tests/unit/scripts/release_performance/_helpers/_test_types.py +++ b/tests/unit/scripts/release_performance/_helpers/_test_types.py @@ -60,3 +60,20 @@ class ChooseBaselineTestCase: files: tuple[dict[str, object], ...] tags: tuple[str, ...] expected_baseline: str | None + + +@dataclass(frozen=True) +class BaselineGeneratorTestCase: + description: str + generator: str + expected_projects: tuple[str, ...] + expected_error_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class ReleaseSourceTestCase: + description: str + tag: str + requested_version: str + expected_files: tuple[str, ...] + expected_error_fragments: tuple[str, ...] diff --git a/tests/unit/scripts/release_performance/_helpers/helpers.py b/tests/unit/scripts/release_performance/_helpers/helpers.py index 7fa7c1385f..9b9b7d20b8 100644 --- a/tests/unit/scripts/release_performance/_helpers/helpers.py +++ b/tests/unit/scripts/release_performance/_helpers/helpers.py @@ -1,8 +1,23 @@ """Builders for release performance comparison test data.""" +import subprocess +from pathlib import Path + from scripts.release_performance.models import CommandComparison, CommandSample MIB: int = 1024 * 1024 +MARKER_GENERATOR: str = """from pathlib import Path + + +def write_pristine_projects(*, root: Path, inspection_models: int, build_models: int) -> Path: + for name, models in (("inspection", inspection_models), ("build", build_models)): + (root / "pristine" / name).mkdir(parents=True) + (root / "pristine" / name / "BASELINE_MARKER").write_text(str(models)) + return root / "pristine" +""" +FAILING_GENERATOR: str = """def write_pristine_projects(*, root, inspection_models, build_models): + raise RuntimeError("baseline generator exploded") +""" def comparison( @@ -29,3 +44,29 @@ def release_file(*, filename: str) -> dict[str, object]: def _sample(*, values: tuple[float, float, int]) -> CommandSample: wall, cpu, rss_mib = values return CommandSample(wall_seconds=wall, cpu_seconds=cpu, peak_rss_bytes=rss_mib * MIB) + + +def write_baseline_source(*, root: Path, generator: str) -> Path: + """Write a source tree whose only content is a release benchmark generator.""" + + helpers: Path = root / "scripts" / "release_performance" / "_helpers" + helpers.mkdir(parents=True) + for package in (root / "scripts", helpers.parent, helpers): + _ = (package / "__init__.py").write_text("", encoding="utf-8") + _ = (helpers / "benchmark.py").write_text(generator, encoding="utf-8") + return root + + +def tagged_repository(*, root: Path, tag: str) -> Path: + """Create a one-commit git repository with a tracked file and the given tag.""" + + root.mkdir(parents=True) + _ = (root / "release.txt").write_text(tag, encoding="utf-8") + for command in ( + ("init", "--quiet"), + ("add", "release.txt"), + ("-c", "user.name=t", "-c", "user.email=t@example.com", "commit", "--quiet", "-m", "r"), + ("tag", tag), + ): + _ = subprocess.run(["git", "-C", str(root), *command], check=True, capture_output=True) + return root diff --git a/tests/unit/scripts/release_performance/_helpers/test_benchmark.py b/tests/unit/scripts/release_performance/_helpers/test_benchmark.py new file mode 100644 index 0000000000..bce6daa78c --- /dev/null +++ b/tests/unit/scripts/release_performance/_helpers/test_benchmark.py @@ -0,0 +1,105 @@ +"""Unit tests for generating each version's release benchmark projects.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.release_performance._helpers.benchmark import write_baseline_pristine_projects +from scripts.release_performance._helpers.versions import release_source +from scripts.release_performance.exceptions import ReleasePerformanceError +from tests.unit.scripts.release_performance._helpers._test_types import ( + BaselineGeneratorTestCase, + ReleaseSourceTestCase, +) +from tests.unit.scripts.release_performance._helpers.helpers import ( + FAILING_GENERATOR, + MARKER_GENERATOR, + tagged_repository, + write_baseline_source, +) + + +@pytest.mark.parametrize( + "test_case", + [ + BaselineGeneratorTestCase( + description="the baseline source's own generator writes the baseline projects", + generator=MARKER_GENERATOR, + expected_projects=("build", "inspection"), + expected_error_fragments=(), + ), + BaselineGeneratorTestCase( + description="a failing baseline generator is reported with its stderr", + generator=FAILING_GENERATOR, + expected_projects=(), + expected_error_fragments=( + "Baseline benchmark generation with the generator in", + "baseline generator exploded", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_baseline_source_when_generating_then_uses_its_own_generator( + test_case: BaselineGeneratorTestCase, tmp_path: Path +) -> None: + source: Path = write_baseline_source(root=tmp_path / "source", generator=test_case.generator) + root: Path = tmp_path / "generated" + error: str = "" + + try: + _ = write_baseline_pristine_projects( + source=source, root=root, inspection_models=30, build_models=10 + ) + except ReleasePerformanceError as raised: + error = str(raised) + + pristine: Path = root / "pristine" + assert tuple(sorted(path.parent.name for path in pristine.glob("*/BASELINE_MARKER"))) == ( + test_case.expected_projects + ) + assert all(fragment in error for fragment in test_case.expected_error_fragments), error + + +@pytest.mark.parametrize( + "test_case", + [ + ReleaseSourceTestCase( + description="the release tag's tree is extracted", + tag="v1.2.3", + requested_version="1.2.3", + expected_files=("release.txt",), + expected_error_fragments=(), + ), + ReleaseSourceTestCase( + description="a missing release tag explains how to provide the source", + tag="v1.2.3", + requested_version="1.2.2", + expected_files=(), + expected_error_fragments=("Release tag v1.2.2 is not available", "--baseline-source"), + ), + ], + ids=lambda case: case.description, +) +def test_given_release_tag_when_extracting_source_then_returns_its_tree( + test_case: ReleaseSourceTestCase, tmp_path: Path +) -> None: + repository: Path = tagged_repository(root=tmp_path / "repo", tag=test_case.tag) + destination: Path = tmp_path / "source" + error: str = "" + + try: + _ = release_source( + version=test_case.requested_version, repo_dir=repository, destination=destination + ) + except ReleasePerformanceError as raised: + error = str(raised) + + assert tuple(sorted(path.name for path in destination.glob("*"))) == test_case.expected_files + assert all(fragment in error for fragment in test_case.expected_error_fragments), error + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/release_performance/_helpers/test_report.py b/tests/unit/scripts/release_performance/_helpers/test_report.py index 6a0cfc8b7f..5ba5bffe62 100644 --- a/tests/unit/scripts/release_performance/_helpers/test_report.py +++ b/tests/unit/scripts/release_performance/_helpers/test_report.py @@ -31,6 +31,8 @@ "### Release performance: 0.126.2 (candidate) vs 0.126.1 (baseline)", "Same runner: Example CPU, 4 CPUs; load average 0.50/0.40/0.30 before", "3 interleaved runs per version", + "Benchmark projects are generated per side: the baseline runs projects from its " + "own generator (v0.126.1), the candidate from the candidate's.", "Limits: wall +25% (ignored under 0.5 s), CPU +25% (ignored under 0.5 s), " "peak RSS +25% (ignored under 32 MiB).", "| `compile (warm cache)` | 6.00 → 6.10 | 1.017x | 6.50 → 6.40 | 0.985x " @@ -82,6 +84,7 @@ def test_given_release_comparison_when_rendering_summary_then_reports_each_comma release: ReleaseComparison = ReleaseComparison( baseline_version="0.126.1", candidate_version="0.126.2", + baseline_generator="v0.126.1", runs=3, runner=RunnerContext( cpu_model="Example CPU", diff --git a/tests/unit/scripts/release_performance/_helpers/test_verdict.py b/tests/unit/scripts/release_performance/_helpers/test_verdict.py index a46d3a46ea..3a4bc8ca5e 100644 --- a/tests/unit/scripts/release_performance/_helpers/test_verdict.py +++ b/tests/unit/scripts/release_performance/_helpers/test_verdict.py @@ -63,6 +63,24 @@ ), expected_regressed=(False, True, False), ), + MetricVerdictTestCase( + description="a wall-only regression at CPU parity fails on wall alone", + comparison=comparison( + name="build (empty warehouse)", + baseline=((10.0, 6.4, 330),) * 3, + candidate=((13.14, 6.36, 330),) * 3, + ), + expected_regressed=(True, False, False), + ), + MetricVerdictTestCase( + description="a wall-only slowdown under the ratio limit at CPU parity passes", + comparison=comparison( + name="build (empty warehouse)", + baseline=((10.0, 6.4, 330),) * 3, + candidate=((12.4, 6.4, 330),) * 3, + ), + expected_regressed=(False, False, False), + ), MetricVerdictTestCase( description="one slow outlier does not move the median time", comparison=comparison(