From f6e357876fe1d488b9c8f57f11e95bf9bd37a6f0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:06:50 +0000 Subject: [PATCH] ci(performance): restore the 1.10 cold limit of the fresh ratio Main now runs the default SQL Rules, so both sides of the same-runner fresh comparison pay their cost. Remove the temporary 1.25 wall and 1.41 CPU cold allowance and the cold-only ratio options that existed for it. --- .github/workflows/verify.yml | 11 ------- .../compile_performance_ratio/main/compare.py | 32 ++++--------------- .../main/compile/COMPILE_BENCHMARKS.md | 8 ++--- 3 files changed, 9 insertions(+), 42 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 28bf653589..3f51d3fbb9 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -671,15 +671,8 @@ jobs: max_ratio: "1.10" edit_max_ratio: "1.20" edit_max_cpu_ratio: "1.75" - # Temporary migration allowance; restore 1.10 once the base also runs default Rules. - # Default SQL Rules add a cold cost over select = [] (1.19x to 1.20x wall, 1.32x to - # 1.33x CPU in this job), so only the cold limits scale; warm and edit keep 1.10. - # The compiler-only cache guard at 3,000 models keeps the select = [] budgets. - kind: fresh - label: fresh default Rules max_ratio: "1.10" - cold_max_ratio: "1.25" - cold_max_cpu_ratio: "1.41" - kind: dense label: contracts CPU vs base gate_phase: contracts_cpu_ms @@ -753,16 +746,12 @@ jobs: env: KIND: ${{ matrix.kind }} MAX_RATIO: ${{ matrix.max_ratio }} - COLD_MAX_RATIO: ${{ matrix.cold_max_ratio }} - COLD_MAX_CPU_RATIO: ${{ matrix.cold_max_cpu_ratio }} EDIT_MAX_RATIO: ${{ matrix.edit_max_ratio }} EDIT_MAX_CPU_RATIO: ${{ matrix.edit_max_cpu_ratio }} run: | uv run python -m scripts.compare_compile_performance \ --kind "${KIND}" \ --modes cold warm edit \ - ${COLD_MAX_RATIO:+--cold-max-ratio "$COLD_MAX_RATIO"} \ - ${COLD_MAX_CPU_RATIO:+--cold-max-cpu-ratio "$COLD_MAX_CPU_RATIO"} \ ${EDIT_MAX_RATIO:+--edit-max-ratio "$EDIT_MAX_RATIO"} \ ${EDIT_MAX_CPU_RATIO:+--edit-max-cpu-ratio "$EDIT_MAX_CPU_RATIO"} \ --max-ratio "${MAX_RATIO}" \ diff --git a/scripts/compile_performance_ratio/main/compare.py b/scripts/compile_performance_ratio/main/compare.py index ce2492e0fc..6bb3d0da36 100644 --- a/scripts/compile_performance_ratio/main/compare.py +++ b/scripts/compile_performance_ratio/main/compare.py @@ -16,7 +16,6 @@ from scripts.compile_performance_ratio._helpers.report import append_summary, comparison_markdown from scripts.compile_performance_ratio._helpers.verdict import phase_failures, ratio_failures from scripts.compile_performance_ratio.constants import ( - COLD_MODE, COMPILE_MODES, DEFAULT_MAX_RATIO, DEFAULT_RUNS, @@ -64,20 +63,13 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: modes=modes, compile_args=tuple(args.compile_args), ) - mode_max_ratios: dict[str, float] = { - mode: limit - for mode, limit in ((COLD_MODE, args.cold_max_ratio), (EDIT_MODE, args.edit_max_ratio)) - if limit is not None - } + mode_max_ratios: dict[str, float] = ( + {} if args.edit_max_ratio is None else {EDIT_MODE: args.edit_max_ratio} + ) gate_phases: tuple[str, ...] = tuple(dict.fromkeys(args.gate_phases)) - mode_cpu_max_ratios: dict[str, float] = { - mode: limit - for mode, limit in ( - (COLD_MODE, args.cold_max_cpu_ratio), - (EDIT_MODE, args.edit_max_cpu_ratio), - ) - if limit is not None - } + mode_cpu_max_ratios: dict[str, float] = ( + {} if args.edit_max_cpu_ratio is None else {EDIT_MODE: args.edit_max_cpu_ratio} + ) failures: tuple[str, ...] = ( phase_failures( comparisons=comparisons, @@ -139,18 +131,6 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: ) parser.add_argument("--runs", type=int, default=DEFAULT_RUNS) parser.add_argument("--max-ratio", type=float, default=DEFAULT_MAX_RATIO) - parser.add_argument( - "--cold-max-ratio", - type=float, - default=None, - help="Ratio limit of the cold mode only; defaults to --max-ratio.", - ) - parser.add_argument( - "--cold-max-cpu-ratio", - type=float, - default=None, - help="CPU ratio limit for cold mode only; defaults to the mode's ratio limit.", - ) parser.add_argument( "--edit-max-cpu-ratio", type=float, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md index 5c4260e128..ba7adcfdeb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md @@ -98,11 +98,9 @@ same runner, three alternating runs per mode, and the medians of wall and CPU ti - warm: an unchanged `sqb compile --json` after one untimed run that writes the cache; - edit: `sqb compile --json` after the same one-model edit as the release comparison below. -The fresh comparison writes both projects with the default SQL Rules, and a base older than them -runs none. It allows 1.25x cold wall and 1.41x cold CPU, which covers the cold cost of the default -Rules over `select = []` measured by this job (1.19x and 1.20x wall, 1.32x and 1.33x CPU, two runs of -three alternating pairs). Its warm and edit modes keep 1.10x. The cold allowance is temporary: once -the base also runs default Rules, it returns to 1.10x. +The fresh comparison writes both projects with the default SQL Rules, and every mode keeps the +1.10x limit. Before the base ran them, the default Rules cost 1.19x to 1.20x cold wall and 1.32x to +1.33x cold CPU over `select = []` in this job. Every fresh-process compile and cache guard runs the default SQL Rules (approved 2026-11-01). One guard keeps `select = []` and the previous budgets: the 3,000-model cache guard, which runs in the