Integrate FFmpeg development checker and failure-stage feedback

edgebench/tasks/ffmpeg_swscale_reimplementation/NOTES.md

@@ -22,7 +22,7 @@ fails (maximize). The rescale is the v1 one (`log_anchor`, anchor 14.155 -> 43).
   snapshot of v1's build day, at v1's versions. Base: `bases/cpp`.
 - The reference FFmpeg build, the baseline library, all toolchains and the generated media are
   byte-identical to the published v1 images.
-- `instruction.md`: v1's prompt, verbatim.
+- `instruction.md`: v1's prompt, with the development-reference correction documented below.
 
 ## Deviations from v1
 
@@ -63,6 +63,29 @@ for every row, well inside that spread.
 
 ## Changes
 
+### 1.0.2 — align public correctness feedback with the scalar baseline
+
+This builds on the 1.0.1 verifier-stage feedback; the judge files and scoring
+remain identical to that version.
+
+`verify_correctness.py` now compares with the already-shipped
+`libswscale_public_baseline.so`, using the same C ABI as `run_dev_bench.py`.
+Previously the assembly-enabled CLI rounded differently on some paths, and its
+nearest `point` flag failed to parse. This sent a correct candidate back into
+unnecessary correctness debugging. Input-media generation may still use the CLI;
+the expected output always comes from the scalar library, with separate buffers.
+A missing or failed reference remains an error, without a CLI fallback.
+
+In an isolated amd64 work image, with the same media, baseline and rebuilt Rust
+scaffold, the old public checker passed 9/13 for **both** the scalar baseline and
+the scaffold. The new checker passed 13/13 for both, including nearest scaling;
+a candidate returning a null context still failed. The workload list, per-plane
+PSNR thresholds, candidate implementation, judge source and score definition are
+unchanged. This fixes development feedback, not historical scores, and does not
+establish correctness on unseen inputs or a solver performance gain. The work
+image comparison uses an immutable image plus the changed public Python script;
+it is not a fresh toolchain rebuild or a new judge benchmark.
+
 ### 1.0.1 — distinguish verifier failure stages
 
 Failed submissions now receive a bounded stage description in `result.json`'s summary and
@@ -126,7 +149,4 @@ Kept as in v1, so scores stay comparable. Fixing any of them changes scores and
    exported symbols. Re-implementing FFmpeg code statically is allowed by design.
 8. **The submission can tamper with grading.** Its build scripts run as root in the judge
    container before scoring (the cross-task issue in `tasks/SCHEMA.md`).
-9. **The dev tool disagrees with the judge.** `verify_correctness.py` compares against the
-   assembly-optimized reference, the judge against the C-only baseline; the scaffold, which passes
-   all 30 hidden workloads, shows 9/13 on the public tool.
-10. `language = "C++"` in the task metadata; the deliverable is Rust or Zig.
+9. `language = "C++"` in the task metadata; the deliverable is Rust or Zig.

edgebench/tasks/ffmpeg_swscale_reimplementation/environment/workspace/instruction.md

@@ -149,10 +149,13 @@ cat /app/media/manifest.json  # Format, size, path metadata
     -f rawvideo -pix_fmt rgb24 /tmp/gradient_rgb24.raw
 ```
 
-**Note:** `verify_correctness.py` is a development-only tool that uses
-`/reference/ffmpeg` to generate golden outputs. The reference binary is
-deleted before final scoring — the actual verifier compares your output
-against the baseline library instead.
+**Note:** `verify_correctness.py` compares public workloads against
+`/app/libswscale_public_baseline.so`, the C-only reference also used by
+`run_dev_bench.py`. It calls the same C API as your candidate, including nearest
+scaling. `/reference/ffmpeg` remains useful for generating input media, but its
+assembly-optimized conversions can round differently from this scalar reference.
+Passing the public workloads does not establish correctness on unseen inputs.
+The candidate must implement conversion itself; it may not wrap the reference.
 
 ### Study the FFmpeg source:
 

edgebench/tasks/ffmpeg_swscale_reimplementation/environment/workspace/verify_correctness.py

@@ -3,7 +3,7 @@
 verify_correctness.py — Agent development tool.
 
 Loads the candidate shared library, runs a set of public conversion workloads,
-and compares outputs against the reference FFmpeg binary.  Reports per-workload
+and compares outputs against the public scalar baseline. Reports per-workload
 PSNR and overall pass/fail.
 
 Usage:
@@ -47,6 +47,7 @@
 )
 
 FFMPEG_BIN = "/reference/ffmpeg"
+BASELINE_LIB = "/app/libswscale_public_baseline.so"
 MEDIA_DIR = Path("/app/media")
 
 PSNR_THRESHOLD_CONVERT = 60.0  # format conversion only (same size)
@@ -93,53 +94,31 @@ def compute_psnr(data_a: bytes, data_b: bytes) -> float:
     return 10.0 * math.log10(255.0 * 255.0 / mse)
 
 
-def generate_reference_output(
-    src_path: Path,
-    src_fmt: str,
-    src_w: int,
-    src_h: int,
-    dst_fmt: str,
-    dst_w: int,
-    dst_h: int,
-    algo: int = ALGO_BILINEAR,
-) -> bytes:
-    """Use reference FFmpeg to produce golden output."""
-    algo_flag_map = {
-        ALGO_NEAREST: "point",
-        ALGO_BILINEAR: "bilinear",
-        ALGO_BICUBIC: "bicubic",
-    }
-    sws_flag = algo_flag_map.get(algo, "bilinear")
+def generate_reference_output(reference, workload: dict, src_planes: list[bytes]) -> list[bytes]:
+    """Use the public C-only baseline, with the same C ABI as the candidate.
 
-    with tempfile.NamedTemporaryFile(suffix=".raw", delete=False) as f:
-        dst_path = f.name
+    The CLI binary enables assembly and uses different rounding on some paths.
+    Separate source buffers also prevent a candidate that mutates its input from
+    changing the reference output.
+    """
+    src_fmt, dst_fmt = workload["src_fmt"], workload["dst_fmt"]
+    src_w, src_h = workload["src_w"], workload["src_h"]
+    dst_w, dst_h = workload["dst_w"], workload["dst_h"]
+    src_data, src_stride, src_bufs = allocate_image(src_fmt, src_w, src_h)
+    dst_data, dst_stride, dst_bufs = allocate_image(dst_fmt, dst_w, dst_h)
+    fill_image_from_bytes(src_fmt, src_w, src_h, src_data, src_stride, src_planes)
+    ctx = reference.swscale_create(
+        src_w, src_h, src_fmt, dst_w, dst_h, dst_fmt, workload["algo"]
+    )
+    if not ctx:
+        raise RuntimeError("Scalar reference swscale_create returned NULL")
     try:
-        cmd = [
-            FFMPEG_BIN,
-            "-y",
-            "-f",
-            "rawvideo",
-            "-pix_fmt",
-            src_fmt,
-            "-s",
-            f"{src_w}x{src_h}",
-            "-i",
-            str(src_path),
-            "-f",
-            "rawvideo",
-            "-pix_fmt",
-            dst_fmt,
-            "-s",
-            f"{dst_w}x{dst_h}",
-            "-sws_flags",
-            sws_flag,
-            dst_path,
-        ]
-        subprocess.run(cmd, check=True, capture_output=True)
-        return Path(dst_path).read_bytes()
+        ret = reference.swscale_process(ctx, src_data, src_stride, dst_data, dst_stride)
+        if ret != 0:
+            raise RuntimeError(f"Scalar reference swscale_process returned {ret}")
+        return image_to_bytes(dst_fmt, dst_w, dst_h, dst_data, dst_stride)
     finally:
-        if os.path.exists(dst_path):
-            os.unlink(dst_path)
+        reference.swscale_destroy(ctx)
 
 
 def load_raw_planes(path: Path, fmt: int, w: int, h: int) -> list[bytes]:
@@ -296,7 +275,7 @@ def load_raw_planes(path: Path, fmt: int, w: int, h: int) -> list[bytes]:
 ]
 
 
-def run_workload(lib, workload: dict) -> dict:
+def run_workload(lib, workload: dict, reference=None) -> dict:
     """Run a single conversion workload and return result dict."""
     label = workload["label"]
     src_fmt = workload["src_fmt"]
@@ -346,7 +325,11 @@ def run_workload(lib, workload: dict) -> dict:
     else:
         src_planes = load_raw_planes(src_file, src_fmt, src_w, src_h)
 
-    # Allocate buffers
+    if reference is None:
+        reference = load_swscale_library(BASELINE_LIB)
+    ref_planes = generate_reference_output(reference, workload, src_planes)
+
+    # Allocate independent candidate buffers
     src_data, src_stride, src_bufs = allocate_image(src_fmt, src_w, src_h)
     dst_data, dst_stride, dst_bufs = allocate_image(dst_fmt, dst_w, dst_h)
 
@@ -375,33 +358,7 @@ def run_workload(lib, workload: dict) -> dict:
     # Extract candidate output
     candidate_planes = image_to_bytes(dst_fmt, dst_w, dst_h, dst_data, dst_stride)
 
-    # Generate reference output
-    dst_name = FFMPEG_PIXFMT[dst_fmt]
-    src_name_ffmpeg = FFMPEG_PIXFMT[src_fmt]
-
-    # Write src to temp file for FFmpeg
-    with tempfile.NamedTemporaryFile(suffix=".raw", delete=False) as f:
-        for plane in src_planes:
-            f.write(plane)
-        tmp_src = f.name
-
-    try:
-        ref_raw = generate_reference_output(
-            Path(tmp_src), src_name_ffmpeg, src_w, src_h, dst_name, dst_w, dst_h, algo
-        )
-    finally:
-        os.unlink(tmp_src)
-
-    # Parse reference into planes
-    ref_planes = []
     ref_desc = PIXFMT_DESCS[dst_fmt]
-    offset = 0
-    for i in range(ref_desc.num_planes):
-        pw = ref_desc.plane_width(i, dst_w) * ref_desc.planes[i].bpp
-        ph = ref_desc.plane_height(i, dst_h)
-        plane_size = pw * ph
-        ref_planes.append(ref_raw[offset : offset + plane_size])
-        offset += plane_size
 
     # Compute PSNR per plane
     is_scaling = (src_w != dst_w) or (src_h != dst_h)
@@ -445,12 +402,14 @@ def main():
         print("Searched:", find_candidate_lib() or "(nothing found)")
         sys.exit(1)
 
-    if not os.path.exists(FFMPEG_BIN):
-        print(f"ERROR: Reference FFmpeg not found at {FFMPEG_BIN}")
+    if not os.path.exists(BASELINE_LIB):
+        print(f"ERROR: Scalar baseline library not found at {BASELINE_LIB}")
         sys.exit(1)
 
     print(f"Loading candidate: {candidate_path}")
     lib = load_swscale_library(candidate_path)
+    reference = load_swscale_library(BASELINE_LIB)
+    print(f"Scalar reference: {BASELINE_LIB}")
 
     results = []
     passed = 0
@@ -459,7 +418,7 @@ def main():
     for wl in PUBLIC_WORKLOADS:
         total += 1
         try:
-            result = run_workload(lib, wl)
+            result = run_workload(lib, wl, reference)
         except Exception as e:
             result = {"label": wl["label"], "status": "error", "reason": str(e)}
         results.append(result)

edgebench/tasks/ffmpeg_swscale_reimplementation/instruction.md

@@ -199,10 +199,13 @@ cat /home/workspace/media/manifest.json  # Format, size, path metadata
     -f rawvideo -pix_fmt rgb24 /tmp/gradient_rgb24.raw
 ```
 
-**Note:** `verify_correctness.py` is a development-only tool that uses
-`/reference/ffmpeg` to generate golden outputs. The reference binary is
-deleted before final scoring — the actual verifier compares your output
-against the baseline library instead.
+**Note:** `verify_correctness.py` compares public workloads against
+`/app/libswscale_public_baseline.so`, the C-only reference also used by
+`run_dev_bench.py`. It calls the same C API as your candidate, including nearest
+scaling. `/reference/ffmpeg` remains useful for generating input media, but its
+assembly-optimized conversions can round differently from this scalar reference.
+Passing the public workloads does not establish correctness on unseen inputs.
+The candidate must implement conversion itself; it may not wrap the reference.
 
 ### Study the FFmpeg source:
 

edgebench/tasks/ffmpeg_swscale_reimplementation/task.toml

@@ -4,7 +4,7 @@ name = "Ffmpeg Swscale Reimplementation"
 category = "Systems & Software Engineering"
 language = "C++"           # as in v1 tasks.jsonl; the deliverable is Rust (or Zig), see NOTES.md
 description = "Reimplement FFmpeg's libswscale pixel-format conversion and scaling library in Rust, handling multiple pixel formats and scaling algorithms. A correctness-passing scaffold is provided; the agent must optimize for speed via SIMD."
-version = "1.0.1"
+version = "1.0.2"
 status = "migrated"
 
 [environment]

tests/test_ffmpeg_dev_correctness.py

@@ -0,0 +1,107 @@
+# Copyright (c) 2026 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Public checker semantics; real scalar-library qualification runs in its image."""
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture
+def checker(monkeypatch, tmp_path):
+    workspace = Path(__file__).resolve().parents[1] / "edgebench/tasks/ffmpeg_swscale_reimplementation/environment/workspace"
+    monkeypatch.syspath_prepend(str(workspace))
+    spec = importlib.util.spec_from_file_location("ffmpeg_dev_correctness", workspace / "verify_correctness.py")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    monkeypatch.setattr(module, "MEDIA_DIR", tmp_path)
+    # Independent bytes, no FFmpeg process or native library needed by unit tests.
+    (tmp_path / "gradient_4x4_gray8.raw").write_bytes(bytes(range(16)))
+    monkeypatch.setattr(module.subprocess, "run", lambda *a, **k: pytest.fail("Unexpected CLI reference"))
+    return module
+
+
+class GrayLibrary:
+    def __init__(self, *, value=None, mutate=False, result=0, create=True):
+        self.value = value
+        self.mutate = mutate
+        self.result = result
+        self.create = create
+        self.destroyed = 0
+        self.arguments = []
+
+    def swscale_create(self, *args):
+        self.arguments.append(args)
+        self.width, self.height = args[3:5]
+        return 1 if self.create else None
+
+    def swscale_process(self, ctx, src, src_stride, dst, dst_stride):
+        for y in range(self.height):
+            for x in range(self.width):
+                if self.mutate:
+                    src[0][y * src_stride[0] + x] = 255
+                dst[0][y * dst_stride[0] + x] = (
+                    src[0][y * src_stride[0] + x] if self.value is None else self.value
+                )
+        return self.result
+
+    def swscale_destroy(self, ctx):
+        self.destroyed += 1
+
+
+def workload(checker, algo):
+    return dict(label="gray identity", src_fmt=checker.PIXFMT_GRAY8,
+                dst_fmt=checker.PIXFMT_GRAY8, src_w=4, src_h=4,
+                dst_w=4, dst_h=4, algo=algo)
+
+
+@pytest.mark.parametrize("algo", [0, 1, 2])
+def test_same_pixels_use_scalar_abi_for_each_algorithm(checker, algo):
+    candidate, reference = GrayLibrary(), GrayLibrary()
+    result = checker.run_workload(candidate, workload(checker, algo), reference)
+    assert result["status"] == "pass"
+    assert result["min_psnr"] == float("inf")
+    assert reference.arguments == candidate.arguments
+    assert reference.arguments[0][-1] == algo
+    assert reference.destroyed == candidate.destroyed == 1
+
+
+@pytest.mark.parametrize("mutate", [False, True])
+def test_wrong_pixels_are_rejected_even_when_candidate_modifies_input(checker, mutate):
+    result = checker.run_workload(GrayLibrary(value=255, mutate=mutate),
+                                  workload(checker, 0), GrayLibrary())
+    assert result["status"] == "fail"
+    assert result["min_psnr"] < result["threshold"] == 60
+
+
+@pytest.mark.parametrize("create,result,match,destroyed", [
+    (False, 0, "Scalar reference swscale_create", 0),
+    (True, -1, "Scalar reference swscale_process", 1),
+])
+def test_broken_reference_is_an_error_not_a_candidate_mismatch(checker, create, result, match, destroyed):
+    candidate = GrayLibrary()
+    reference = GrayLibrary(create=create, result=result)
+    with pytest.raises(RuntimeError, match=match):
+        checker.run_workload(candidate, workload(checker, 1), reference)
+    assert reference.destroyed == destroyed
+    assert candidate.arguments == []
+
+
+def test_missing_scalar_baseline_does_not_fall_back_to_cli(checker, monkeypatch):
+    monkeypatch.setattr(checker, "BASELINE_LIB", "/missing-scalar-baseline.so")
+    with pytest.raises(OSError):
+        checker.run_workload(GrayLibrary(), workload(checker, 1))