ci: generate baseline benchmark projects with the baseline release generator

scripts/compile_performance_ratio/_helpers/measure.py

@@ -55,18 +55,37 @@ def write_benchmark_project(*, kind: str, project_dir: Path, models: int) -> Non
     )
 
 
+def run_checkout_generator(
+    *,
+    checkout: Path,
+    python: Path,
+    entry: str,
+    arguments: tuple[str, ...],
+    environment: dict[str, str],
+) -> subprocess.CompletedProcess[str]:
+    """Run generator code from another checkout's source tree with this `python`."""
+
+    return subprocess.run(
+        [str(python), "-c", entry, *arguments],
+        cwd=checkout,
+        capture_output=True,
+        text=True,
+        env={**environment, PYTHONPATH_KEY: str(checkout)},
+        check=False,
+    )
+
+
 def write_base_benchmark_project(
     *, base_root: Path, python: Path, kind: str, project_dir: Path, models: int
 ) -> None:
     """Write the base project with the base checkout's own generator, run by `python`."""
 
-    completed: subprocess.CompletedProcess[str] = subprocess.run(
-        [str(python), "-c", BASE_GENERATOR_ENTRY, kind, str(project_dir), str(models)],
-        cwd=base_root,
-        capture_output=True,
-        text=True,
-        env={**_compile_environment(), PYTHONPATH_KEY: str(base_root)},
-        check=False,
+    completed: subprocess.CompletedProcess[str] = run_checkout_generator(
+        checkout=base_root,
+        python=python,
+        entry=BASE_GENERATOR_ENTRY,
+        arguments=(kind, str(project_dir), str(models)),
+        environment=_compile_environment(),
     )
     if completed.returncode != 0:
         raise CompileComparisonError(

scripts/release_performance/_helpers/benchmark.py

@@ -11,7 +11,9 @@
 from pathlib import Path
 from typing import cast
 
+from scripts.compile_performance_ratio._helpers.measure import run_checkout_generator
 from scripts.release_performance.constants import (
+    BASELINE_GENERATOR_ENTRY,
     BUILD_PROJECT,
     BUILD_WARMUPS,
     COMMAND_TIMEOUT_SECONDS,
@@ -20,6 +22,7 @@
     EXCLUDED_ENVIRONMENT_PREFIX,
     INSPECTION_PROJECT,
     INSPECTION_WARMUPS,
+    PRISTINE_DIRECTORY,
     STDERR_TAIL_CHARACTERS,
     TIME_FORMAT,
 )
@@ -42,14 +45,34 @@
 def write_pristine_projects(*, root: Path, inspection_models: int, build_models: int) -> Path:
     """Generate the inspection and build benchmarks once so every version reads the same files."""
 
-    pristine: Path = root / "pristine"
+    pristine: Path = root / PRISTINE_DIRECTORY
     write_inspection_benchmark_project(
         project_dir=pristine / INSPECTION_PROJECT, model_count=inspection_models
     )
     write_build_benchmark_project(project_dir=pristine / BUILD_PROJECT, model_count=build_models)
     return pristine
 
 
+def write_baseline_pristine_projects(
+    *, source: Path, root: Path, inspection_models: int, build_models: int
+) -> Path:
+    """Generate the baseline's benchmarks with the baseline source's own generator."""
+
+    completed: subprocess.CompletedProcess[str] = run_checkout_generator(
+        checkout=source,
+        python=Path(sys.executable).absolute(),
+        entry=BASELINE_GENERATOR_ENTRY,
+        arguments=(str(root), str(inspection_models), str(build_models)),
+        environment=_environment(),
+    )
+    if completed.returncode != 0:
+        raise ReleasePerformanceError(
+            f"Baseline benchmark generation with the generator in {source} failed with exit "
+            f"{completed.returncode}:\n{completed.stderr[-STDERR_TAIL_CHARACTERS:]}"
+        )
+    return root / PRISTINE_DIRECTORY
+
+
 def prepare_version_projects(
     *, version: InstalledVersion, pristine: Path, root: Path
 ) -> dict[str, Path]:

scripts/release_performance/_helpers/report.py

@@ -31,6 +31,9 @@ def comparison_markdown(*, comparison: ReleaseComparison) -> str:
         f"{comparison.runs} interleaved runs per version; wall and CPU are medians, "
         "peak RSS is the worst run.",
         "",
+        "Benchmark projects are generated per side: the baseline runs projects from its own "
+        f"generator ({comparison.baseline_generator}), the candidate from the candidate's.",
+        "",
         f"Limits: wall {_percent(MAX_WALL_RATIO)} (ignored under "
         f"{MIN_WALL_REGRESSION_SECONDS:g} s), CPU {_percent(MAX_CPU_RATIO)} (ignored under "
         f"{MIN_CPU_REGRESSION_SECONDS:g} s), peak RSS {_percent(MAX_RSS_RATIO)} (ignored under "

scripts/release_performance/_helpers/versions.py

@@ -3,12 +3,14 @@
 from __future__ import annotations
 
 import hashlib
+import io
 import json
 import platform
 import re
 import shutil
 import subprocess
 import sys
+import tarfile
 import time
 import urllib.request
 from pathlib import Path
@@ -22,6 +24,7 @@
     PYPI_SIMPLE_JSON,
     PYPI_SIMPLE_URL,
     RELEASE_TAG_PATTERN,
+    RELEASE_TAG_PREFIX,
     STDERR_TAIL_CHARACTERS,
     UNIVERSAL_WHEEL_MACHINE,
     WHEEL_MACHINE_ALIASES,
@@ -118,6 +121,28 @@ def tagged_versions(*, repo_dir: Path) -> tuple[str, ...]:
     )
 
 
+def release_source(*, version: str, repo_dir: Path, destination: Path) -> Path:
+    """Extract the source tree of release tag v<version> from repo_dir into destination."""
+
+    tag: str = RELEASE_TAG_PREFIX + version
+    completed: subprocess.CompletedProcess[bytes] = subprocess.run(
+        ["git", "-C", str(repo_dir), "archive", "--format=tar", tag],
+        capture_output=True,
+        check=False,
+    )
+    if completed.returncode != 0:
+        raise ReleasePerformanceError(
+            f"Release tag {tag} is not available in {repo_dir}, so the baseline's own benchmark "
+            "generator cannot be used; fetch the release tags or pass --baseline-source: "
+            + _tail(completed.stderr.decode("utf-8", errors="replace"))
+        )
+    shutil.rmtree(destination, ignore_errors=True)
+    destination.mkdir(parents=True)
+    with tarfile.open(fileobj=io.BytesIO(completed.stdout)) as archive:
+        archive.extractall(destination, filter="data")
+    return destination
+
+
 def resolve_baseline(*, candidate: str, repo_dir: Path) -> str:
     """Return the highest published or not-yet-published tagged release below the candidate."""
 

scripts/release_performance/_helpers/workflow.py

@@ -13,6 +13,7 @@
 from scripts.release_performance._helpers.benchmark import (
     compare_command,
     prepare_version_projects,
+    write_baseline_pristine_projects,
     write_pristine_projects,
 )
 from scripts.release_performance._helpers.report import (
@@ -30,13 +31,17 @@
     install_published,
     install_wheel,
     installed_version,
+    release_source,
     resolve_baseline,
 )
 from scripts.release_performance.constants import (
+    BASELINE_GENERATED_DIRECTORY,
     BASELINE_LABEL,
     BASELINE_PUBLICATION_WAIT_SECONDS,
+    BASELINE_SOURCE_DIRECTORY,
     BENCHMARK_COMMANDS,
     CANDIDATE_LABEL,
+    RELEASE_TAG_PREFIX,
     REPO_ROOT,
 )
 from scripts.release_performance.exceptions import ReleasePerformanceError
@@ -104,16 +109,40 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison:
         file=sys.stderr,
     )
     load_before: tuple[float, float, float] = os.getloadavg()
-    print("Generating benchmark projects", file=sys.stderr)
-    pristine: Path = write_pristine_projects(
-        root=root, inspection_models=options.inspection_models, build_models=options.build_models
+    print("Generating candidate benchmark projects", file=sys.stderr)
+    pristine: dict[str, Path] = {
+        CANDIDATE_LABEL: write_pristine_projects(
+            root=root,
+            inspection_models=options.inspection_models,
+            build_models=options.build_models,
+        )
+    }
+    baseline_generator: str = (
+        str(options.baseline_source)
+        if options.baseline_source is not None
+        else RELEASE_TAG_PREFIX + baseline.version
+    )
+    print(f"Generating baseline benchmark projects with {baseline_generator}", file=sys.stderr)
+    pristine[BASELINE_LABEL] = write_baseline_pristine_projects(
+        source=(
+            options.baseline_source.absolute()
+            if options.baseline_source is not None
+            else release_source(
+                version=baseline.version,
+                repo_dir=REPO_ROOT,
+                destination=root / BASELINE_SOURCE_DIRECTORY,
+            )
+        ),
+        root=root / BASELINE_GENERATED_DIRECTORY,
+        inspection_models=options.inspection_models,
+        build_models=options.build_models,
     )
     versions: tuple[InstalledVersion, InstalledVersion] = (baseline, candidate)
     projects: dict[str, dict[str, Path]] = {}
     for version in versions:
         print(f"Warming caches for {version.label} {version.version}", file=sys.stderr)
         projects[version.label] = prepare_version_projects(
-            version=version, pristine=pristine, root=root
+            version=version, pristine=pristine[version.label], root=root
         )
     measured: list[CommandComparison] = []
     skipped: list[SkippedCommand] = []
@@ -133,6 +162,7 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison:
     return ReleaseComparison(
         baseline_version=baseline.version,
         candidate_version=candidate.version,
+        baseline_generator=baseline_generator,
         runs=options.runs,
         runner=RunnerContext(
             cpu_model=cpu_model(),

scripts/release_performance/constants.py

@@ -42,6 +42,16 @@
 TIME_FORMAT: str = "%e %M %U %S"
 COMPILE_COMMAND: str = "compile"
 STDERR_TAIL_CHARACTERS: int = 2_000
+PRISTINE_DIRECTORY: str = "pristine"
+BASELINE_GENERATED_DIRECTORY: str = "baseline-generated"
+BASELINE_SOURCE_DIRECTORY: str = "baseline-source"
+RELEASE_TAG_PREFIX: str = "v"
+BASELINE_GENERATOR_ENTRY: str = (
+    "import sys; from pathlib import Path; "
+    "from scripts.release_performance._helpers.benchmark import write_pristine_projects; "
+    "write_pristine_projects("
+    "root=Path(sys.argv[1]), inspection_models=int(sys.argv[2]), build_models=int(sys.argv[3]))"
+)
 
 BENCHMARK_COMMANDS: tuple[BenchmarkCommand, ...] = (
     BenchmarkCommand(

scripts/release_performance/main/compare.py

@@ -39,6 +39,15 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions:
     parser.add_argument("--python", default=DEFAULT_PYTHON)
     parser.add_argument("--inspection-models", type=int, default=DEFAULT_INSPECTION_MODELS)
     parser.add_argument("--build-models", type=int, default=DEFAULT_BUILD_MODELS)
+    parser.add_argument(
+        "--baseline-source",
+        type=Path,
+        default=None,
+        help=(
+            "Source tree whose benchmark generator writes the baseline's projects. Defaults to "
+            "the baseline's release tag extracted from this repository."
+        ),
+    )
     parser.add_argument("--work-dir", type=Path, default=None)
     parser.add_argument("--output", type=Path, default=None)
     args: argparse.Namespace = parser.parse_args(argv)
@@ -53,4 +62,5 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions:
         build_models=args.build_models,
         work_dir=args.work_dir,
         output=args.output,
+        baseline_source=args.baseline_source,
     )

scripts/release_performance/models.py

@@ -68,6 +68,7 @@ class RunnerContext:
 class ReleaseComparison:
     baseline_version: str
     candidate_version: str
+    baseline_generator: str
     runs: int
     runner: RunnerContext
     commands: tuple[CommandComparison, ...]
@@ -84,3 +85,4 @@ class ComparisonOptions:
     build_models: int
     work_dir: Path | None
     output: Path | None
+    baseline_source: Path | None = None

tests/e2e/scripts/release_performance/_test_types.py

@@ -10,5 +10,6 @@ class ReleasePerformanceTestCase:
     description: str
     inspection_models: int
     build_models: int
-    expected_return_code: int
+    expected_outcomes: tuple[tuple[int, bool], ...]
     expected_fragments: tuple[str, ...]
+    expected_marked_projects: tuple[str, ...]

tests/e2e/scripts/release_performance/helpers.py

@@ -0,0 +1,35 @@
+"""Fixtures for release performance comparisons with per-side benchmark generators."""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+BASELINE_MARKER: str = "BASELINE_MARKER"
+_MARKING_WRAPPER: str = """
+
+_unmarked_write_pristine_projects = write_pristine_projects
+
+
+def write_pristine_projects(*, root: Path, inspection_models: int, build_models: int) -> Path:
+    pristine: Path = _unmarked_write_pristine_projects(
+        root=root, inspection_models=inspection_models, build_models=build_models
+    )
+    for project in pristine.iterdir():
+        (project / "BASELINE_MARKER").write_text("baseline", encoding="utf-8")
+    return pristine
+"""
+
+
+def write_marked_baseline_source(*, repo_root: Path, destination: Path) -> Path:
+    """Copy this checkout's generators and mark every project the copy generates."""
+
+    for name in ("scripts", "tests"):
+        _ = shutil.copytree(
+            repo_root / name, destination / name, ignore=shutil.ignore_patterns("__pycache__")
+        )
+    benchmark: Path = destination / "scripts" / "release_performance" / "_helpers" / "benchmark.py"
+    _ = benchmark.write_text(
+        benchmark.read_text(encoding="utf-8") + _MARKING_WRAPPER, encoding="utf-8"
+    )
+    return destination