diff --git a/edgebench/tasks/ffmpeg_swscale_reimplementation/NOTES.md b/edgebench/tasks/ffmpeg_swscale_reimplementation/NOTES.md index 72471a97..e81df1ed 100644 --- a/edgebench/tasks/ffmpeg_swscale_reimplementation/NOTES.md +++ b/edgebench/tasks/ffmpeg_swscale_reimplementation/NOTES.md @@ -22,7 +22,7 @@ fails (maximize). The rescale is the v1 one (`log_anchor`, anchor 14.155 -> 43). snapshot of v1's build day, at v1's versions. Base: `bases/cpp`. - The reference FFmpeg build, the baseline library, all toolchains and the generated media are byte-identical to the published v1 images. -- `instruction.md`: v1's prompt, verbatim. +- `instruction.md`: v1's prompt, with the development-reference correction documented below. ## Deviations from v1 @@ -63,6 +63,29 @@ for every row, well inside that spread. ## Changes +### 1.0.2 — align public correctness feedback with the scalar baseline + +This builds on the 1.0.1 verifier-stage feedback; the judge files and scoring +remain identical to that version. + +`verify_correctness.py` now compares with the already-shipped +`libswscale_public_baseline.so`, using the same C ABI as `run_dev_bench.py`. +Previously the assembly-enabled CLI rounded differently on some paths, and its +nearest `point` flag failed to parse. This sent a correct candidate back into +unnecessary correctness debugging. Input-media generation may still use the CLI; +the expected output always comes from the scalar library, with separate buffers. +A missing or failed reference remains an error, without a CLI fallback. + +In an isolated amd64 work image, with the same media, baseline and rebuilt Rust +scaffold, the old public checker passed 9/13 for **both** the scalar baseline and +the scaffold. The new checker passed 13/13 for both, including nearest scaling; +a candidate returning a null context still failed. The workload list, per-plane +PSNR thresholds, candidate implementation, judge source and score definition are +unchanged. This fixes development feedback, not historical scores, and does not +establish correctness on unseen inputs or a solver performance gain. The work +image comparison uses an immutable image plus the changed public Python script; +it is not a fresh toolchain rebuild or a new judge benchmark. + ### 1.0.1 — distinguish verifier failure stages Failed submissions now receive a bounded stage description in `result.json`'s summary and @@ -126,7 +149,4 @@ Kept as in v1, so scores stay comparable. Fixing any of them changes scores and exported symbols. Re-implementing FFmpeg code statically is allowed by design. 8. **The submission can tamper with grading.** Its build scripts run as root in the judge container before scoring (the cross-task issue in `tasks/SCHEMA.md`). -9. **The dev tool disagrees with the judge.** `verify_correctness.py` compares against the - assembly-optimized reference, the judge against the C-only baseline; the scaffold, which passes - all 30 hidden workloads, shows 9/13 on the public tool. -10. `language = "C++"` in the task metadata; the deliverable is Rust or Zig. +9. `language = "C++"` in the task metadata; the deliverable is Rust or Zig. diff --git a/edgebench/tasks/ffmpeg_swscale_reimplementation/environment/workspace/instruction.md b/edgebench/tasks/ffmpeg_swscale_reimplementation/environment/workspace/instruction.md index 99450e6a..3ec841c2 100644 --- a/edgebench/tasks/ffmpeg_swscale_reimplementation/environment/workspace/instruction.md +++ b/edgebench/tasks/ffmpeg_swscale_reimplementation/environment/workspace/instruction.md @@ -149,10 +149,13 @@ cat /app/media/manifest.json # Format, size, path metadata -f rawvideo -pix_fmt rgb24 /tmp/gradient_rgb24.raw ``` -**Note:** `verify_correctness.py` is a development-only tool that uses -`/reference/ffmpeg` to generate golden outputs. The reference binary is -deleted before final scoring — the actual verifier compares your output -against the baseline library instead. +**Note:** `verify_correctness.py` compares public workloads against +`/app/libswscale_public_baseline.so`, the C-only reference also used by +`run_dev_bench.py`. It calls the same C API as your candidate, including nearest +scaling. `/reference/ffmpeg` remains useful for generating input media, but its +assembly-optimized conversions can round differently from this scalar reference. +Passing the public workloads does not establish correctness on unseen inputs. +The candidate must implement conversion itself; it may not wrap the reference. ### Study the FFmpeg source: diff --git a/edgebench/tasks/ffmpeg_swscale_reimplementation/environment/workspace/verify_correctness.py b/edgebench/tasks/ffmpeg_swscale_reimplementation/environment/workspace/verify_correctness.py index cc3193b3..72b0725d 100644 --- a/edgebench/tasks/ffmpeg_swscale_reimplementation/environment/workspace/verify_correctness.py +++ b/edgebench/tasks/ffmpeg_swscale_reimplementation/environment/workspace/verify_correctness.py @@ -3,7 +3,7 @@ verify_correctness.py — Agent development tool. Loads the candidate shared library, runs a set of public conversion workloads, -and compares outputs against the reference FFmpeg binary. Reports per-workload +and compares outputs against the public scalar baseline. Reports per-workload PSNR and overall pass/fail. Usage: @@ -47,6 +47,7 @@ ) FFMPEG_BIN = "/reference/ffmpeg" +BASELINE_LIB = "/app/libswscale_public_baseline.so" MEDIA_DIR = Path("/app/media") PSNR_THRESHOLD_CONVERT = 60.0 # format conversion only (same size) @@ -93,53 +94,31 @@ def compute_psnr(data_a: bytes, data_b: bytes) -> float: return 10.0 * math.log10(255.0 * 255.0 / mse) -def generate_reference_output( - src_path: Path, - src_fmt: str, - src_w: int, - src_h: int, - dst_fmt: str, - dst_w: int, - dst_h: int, - algo: int = ALGO_BILINEAR, -) -> bytes: - """Use reference FFmpeg to produce golden output.""" - algo_flag_map = { - ALGO_NEAREST: "point", - ALGO_BILINEAR: "bilinear", - ALGO_BICUBIC: "bicubic", - } - sws_flag = algo_flag_map.get(algo, "bilinear") +def generate_reference_output(reference, workload: dict, src_planes: list[bytes]) -> list[bytes]: + """Use the public C-only baseline, with the same C ABI as the candidate. - with tempfile.NamedTemporaryFile(suffix=".raw", delete=False) as f: - dst_path = f.name + The CLI binary enables assembly and uses different rounding on some paths. + Separate source buffers also prevent a candidate that mutates its input from + changing the reference output. + """ + src_fmt, dst_fmt = workload["src_fmt"], workload["dst_fmt"] + src_w, src_h = workload["src_w"], workload["src_h"] + dst_w, dst_h = workload["dst_w"], workload["dst_h"] + src_data, src_stride, src_bufs = allocate_image(src_fmt, src_w, src_h) + dst_data, dst_stride, dst_bufs = allocate_image(dst_fmt, dst_w, dst_h) + fill_image_from_bytes(src_fmt, src_w, src_h, src_data, src_stride, src_planes) + ctx = reference.swscale_create( + src_w, src_h, src_fmt, dst_w, dst_h, dst_fmt, workload["algo"] + ) + if not ctx: + raise RuntimeError("Scalar reference swscale_create returned NULL") try: - cmd = [ - FFMPEG_BIN, - "-y", - "-f", - "rawvideo", - "-pix_fmt", - src_fmt, - "-s", - f"{src_w}x{src_h}", - "-i", - str(src_path), - "-f", - "rawvideo", - "-pix_fmt", - dst_fmt, - "-s", - f"{dst_w}x{dst_h}", - "-sws_flags", - sws_flag, - dst_path, - ] - subprocess.run(cmd, check=True, capture_output=True) - return Path(dst_path).read_bytes() + ret = reference.swscale_process(ctx, src_data, src_stride, dst_data, dst_stride) + if ret != 0: + raise RuntimeError(f"Scalar reference swscale_process returned {ret}") + return image_to_bytes(dst_fmt, dst_w, dst_h, dst_data, dst_stride) finally: - if os.path.exists(dst_path): - os.unlink(dst_path) + reference.swscale_destroy(ctx) def load_raw_planes(path: Path, fmt: int, w: int, h: int) -> list[bytes]: @@ -296,7 +275,7 @@ def load_raw_planes(path: Path, fmt: int, w: int, h: int) -> list[bytes]: ] -def run_workload(lib, workload: dict) -> dict: +def run_workload(lib, workload: dict, reference=None) -> dict: """Run a single conversion workload and return result dict.""" label = workload["label"] src_fmt = workload["src_fmt"] @@ -346,7 +325,11 @@ def run_workload(lib, workload: dict) -> dict: else: src_planes = load_raw_planes(src_file, src_fmt, src_w, src_h) - # Allocate buffers + if reference is None: + reference = load_swscale_library(BASELINE_LIB) + ref_planes = generate_reference_output(reference, workload, src_planes) + + # Allocate independent candidate buffers src_data, src_stride, src_bufs = allocate_image(src_fmt, src_w, src_h) dst_data, dst_stride, dst_bufs = allocate_image(dst_fmt, dst_w, dst_h) @@ -375,33 +358,7 @@ def run_workload(lib, workload: dict) -> dict: # Extract candidate output candidate_planes = image_to_bytes(dst_fmt, dst_w, dst_h, dst_data, dst_stride) - # Generate reference output - dst_name = FFMPEG_PIXFMT[dst_fmt] - src_name_ffmpeg = FFMPEG_PIXFMT[src_fmt] - - # Write src to temp file for FFmpeg - with tempfile.NamedTemporaryFile(suffix=".raw", delete=False) as f: - for plane in src_planes: - f.write(plane) - tmp_src = f.name - - try: - ref_raw = generate_reference_output( - Path(tmp_src), src_name_ffmpeg, src_w, src_h, dst_name, dst_w, dst_h, algo - ) - finally: - os.unlink(tmp_src) - - # Parse reference into planes - ref_planes = [] ref_desc = PIXFMT_DESCS[dst_fmt] - offset = 0 - for i in range(ref_desc.num_planes): - pw = ref_desc.plane_width(i, dst_w) * ref_desc.planes[i].bpp - ph = ref_desc.plane_height(i, dst_h) - plane_size = pw * ph - ref_planes.append(ref_raw[offset : offset + plane_size]) - offset += plane_size # Compute PSNR per plane is_scaling = (src_w != dst_w) or (src_h != dst_h) @@ -445,12 +402,14 @@ def main(): print("Searched:", find_candidate_lib() or "(nothing found)") sys.exit(1) - if not os.path.exists(FFMPEG_BIN): - print(f"ERROR: Reference FFmpeg not found at {FFMPEG_BIN}") + if not os.path.exists(BASELINE_LIB): + print(f"ERROR: Scalar baseline library not found at {BASELINE_LIB}") sys.exit(1) print(f"Loading candidate: {candidate_path}") lib = load_swscale_library(candidate_path) + reference = load_swscale_library(BASELINE_LIB) + print(f"Scalar reference: {BASELINE_LIB}") results = [] passed = 0 @@ -459,7 +418,7 @@ def main(): for wl in PUBLIC_WORKLOADS: total += 1 try: - result = run_workload(lib, wl) + result = run_workload(lib, wl, reference) except Exception as e: result = {"label": wl["label"], "status": "error", "reason": str(e)} results.append(result) diff --git a/edgebench/tasks/ffmpeg_swscale_reimplementation/instruction.md b/edgebench/tasks/ffmpeg_swscale_reimplementation/instruction.md index f0477aab..64c8dabf 100644 --- a/edgebench/tasks/ffmpeg_swscale_reimplementation/instruction.md +++ b/edgebench/tasks/ffmpeg_swscale_reimplementation/instruction.md @@ -199,10 +199,13 @@ cat /home/workspace/media/manifest.json # Format, size, path metadata -f rawvideo -pix_fmt rgb24 /tmp/gradient_rgb24.raw ``` -**Note:** `verify_correctness.py` is a development-only tool that uses -`/reference/ffmpeg` to generate golden outputs. The reference binary is -deleted before final scoring — the actual verifier compares your output -against the baseline library instead. +**Note:** `verify_correctness.py` compares public workloads against +`/app/libswscale_public_baseline.so`, the C-only reference also used by +`run_dev_bench.py`. It calls the same C API as your candidate, including nearest +scaling. `/reference/ffmpeg` remains useful for generating input media, but its +assembly-optimized conversions can round differently from this scalar reference. +Passing the public workloads does not establish correctness on unseen inputs. +The candidate must implement conversion itself; it may not wrap the reference. ### Study the FFmpeg source: diff --git a/edgebench/tasks/ffmpeg_swscale_reimplementation/task.toml b/edgebench/tasks/ffmpeg_swscale_reimplementation/task.toml index 0bc491c8..6ae1c42f 100644 --- a/edgebench/tasks/ffmpeg_swscale_reimplementation/task.toml +++ b/edgebench/tasks/ffmpeg_swscale_reimplementation/task.toml @@ -4,7 +4,7 @@ name = "Ffmpeg Swscale Reimplementation" category = "Systems & Software Engineering" language = "C++" # as in v1 tasks.jsonl; the deliverable is Rust (or Zig), see NOTES.md description = "Reimplement FFmpeg's libswscale pixel-format conversion and scaling library in Rust, handling multiple pixel formats and scaling algorithms. A correctness-passing scaffold is provided; the agent must optimize for speed via SIMD." -version = "1.0.1" +version = "1.0.2" status = "migrated" [environment] diff --git a/tests/test_ffmpeg_dev_correctness.py b/tests/test_ffmpeg_dev_correctness.py new file mode 100644 index 00000000..db47638b --- /dev/null +++ b/tests/test_ffmpeg_dev_correctness.py @@ -0,0 +1,107 @@ +# Copyright (c) 2026 ByteDance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public checker semantics; real scalar-library qualification runs in its image.""" +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + + +@pytest.fixture +def checker(monkeypatch, tmp_path): + workspace = Path(__file__).resolve().parents[1] / "edgebench/tasks/ffmpeg_swscale_reimplementation/environment/workspace" + monkeypatch.syspath_prepend(str(workspace)) + spec = importlib.util.spec_from_file_location("ffmpeg_dev_correctness", workspace / "verify_correctness.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "MEDIA_DIR", tmp_path) + # Independent bytes, no FFmpeg process or native library needed by unit tests. + (tmp_path / "gradient_4x4_gray8.raw").write_bytes(bytes(range(16))) + monkeypatch.setattr(module.subprocess, "run", lambda *a, **k: pytest.fail("Unexpected CLI reference")) + return module + + +class GrayLibrary: + def __init__(self, *, value=None, mutate=False, result=0, create=True): + self.value = value + self.mutate = mutate + self.result = result + self.create = create + self.destroyed = 0 + self.arguments = [] + + def swscale_create(self, *args): + self.arguments.append(args) + self.width, self.height = args[3:5] + return 1 if self.create else None + + def swscale_process(self, ctx, src, src_stride, dst, dst_stride): + for y in range(self.height): + for x in range(self.width): + if self.mutate: + src[0][y * src_stride[0] + x] = 255 + dst[0][y * dst_stride[0] + x] = ( + src[0][y * src_stride[0] + x] if self.value is None else self.value + ) + return self.result + + def swscale_destroy(self, ctx): + self.destroyed += 1 + + +def workload(checker, algo): + return dict(label="gray identity", src_fmt=checker.PIXFMT_GRAY8, + dst_fmt=checker.PIXFMT_GRAY8, src_w=4, src_h=4, + dst_w=4, dst_h=4, algo=algo) + + +@pytest.mark.parametrize("algo", [0, 1, 2]) +def test_same_pixels_use_scalar_abi_for_each_algorithm(checker, algo): + candidate, reference = GrayLibrary(), GrayLibrary() + result = checker.run_workload(candidate, workload(checker, algo), reference) + assert result["status"] == "pass" + assert result["min_psnr"] == float("inf") + assert reference.arguments == candidate.arguments + assert reference.arguments[0][-1] == algo + assert reference.destroyed == candidate.destroyed == 1 + + +@pytest.mark.parametrize("mutate", [False, True]) +def test_wrong_pixels_are_rejected_even_when_candidate_modifies_input(checker, mutate): + result = checker.run_workload(GrayLibrary(value=255, mutate=mutate), + workload(checker, 0), GrayLibrary()) + assert result["status"] == "fail" + assert result["min_psnr"] < result["threshold"] == 60 + + +@pytest.mark.parametrize("create,result,match,destroyed", [ + (False, 0, "Scalar reference swscale_create", 0), + (True, -1, "Scalar reference swscale_process", 1), +]) +def test_broken_reference_is_an_error_not_a_candidate_mismatch(checker, create, result, match, destroyed): + candidate = GrayLibrary() + reference = GrayLibrary(create=create, result=result) + with pytest.raises(RuntimeError, match=match): + checker.run_workload(candidate, workload(checker, 1), reference) + assert reference.destroyed == destroyed + assert candidate.arguments == [] + + +def test_missing_scalar_baseline_does_not_fall_back_to_cli(checker, monkeypatch): + monkeypatch.setattr(checker, "BASELINE_LIB", "/missing-scalar-baseline.so") + with pytest.raises(OSError): + checker.run_workload(GrayLibrary(), workload(checker, 1))