From 4e984893dfbc9db3ad363f9493c6dcc110f9c018 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Fri, 4 Sep 2026 15:44:56 +0200 Subject: [PATCH 1/4] =?UTF-8?q?docs:=20G2b=20verdict=20=E2=80=94=204.8231;?= =?UTF-8?q?=20the=20register-add=20neutralized=20the=20epoch=20gain?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 48k total with +18k classical Tashkeela at 6 epochs: flat vs the 3ep 4.8218 control, 0.25pp worse than news-only G2a 4.5701. Paired bootstrap +2.3717 CI 2.194-2.554; teacher reproduces 2.289. With E6 swap 5.8057 the domain-coverage attribution is negative in BOTH directions; Paper-B lever table gains E5/E6/G2b rows and the framing line is corrected. Verdict read from final_eval.json on the volume; recorded per TODO.substantiate item 04. --- docs/EXPERIMENTS.md | 21 +++++++++++++++++++++ docs/PUBLICATION-NOTES.md | 26 ++++++++++++++++++-------- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/docs/EXPERIMENTS.md b/docs/EXPERIMENTS.md index f3f7f79..6e34ef9 100644 --- a/docs/EXPERIMENTS.md +++ b/docs/EXPERIMENTS.md @@ -297,6 +297,27 @@ All rows passed the CER parity gate at release. Readings: shows a convergence-speed effect distinguishable from noise on the surviving curves; the DER gates remain the only verdicts. +## G2b — label-scale ADD rung (run-011-tashkeela-6ep) — verdict recorded 2026-09-04 + +- **Design** (ml #135): 48,000 total units — the control's news-mix + plus 18,000 additional classical Tashkeela units — trained 6 epochs + (G2a's epoch count). Spec `ara-diac-small-2-6ep-tashkeela`. +- **Status:** COMPLETE — **4.8231** full-set windowed DER (n=1200; + teacher reproduces 2.289; paired bootstrap student−teacher +2.3717, + CI [2.194, 2.554]). From final_eval.json on the volume; run + completed and evaluated by its owners; recorded here per + TODO.substantiate item 04. Labels sha256 b59e2f5677693eab8968208e0 + 0a70590d03ce743fbc43b2acdb0ac9bd508c8fb. +- **Read:** the domain-coverage attribution is now negative in BOTH + directions at matched epochs. Swapping news out (E6, 3ep): 5.8057. + Adding classical units on top (G2b, 6ep): 4.8231 — flat vs the 3ep + 30k control (4.8218) and 0.25pp WORSE than news-only G2a (4.5701): + the Tashkeela add neutralized the epoch gain entirely. The + post-E3 residual is not register-shaped; the only levers that have + moved this rung are optimizer (−2.96pp), fresher teacher labels + (−0.47pp), and epochs (−0.25pp). What remains is GKD (on-policy + exposure, run-012 in flight) — after which the ladder is closed. + ## GKD — on-policy distillation rung (REGISTERED/ARMED 2026-09-03) - **Status:** REGISTERED, launch pending owner ordering (the last diff --git a/docs/PUBLICATION-NOTES.md b/docs/PUBLICATION-NOTES.md index 80bdcb8..713dd40 100644 --- a/docs/PUBLICATION-NOTES.md +++ b/docs/PUBLICATION-NOTES.md @@ -77,11 +77,16 @@ confound (fresh aux head for the final 23% of steps); E6 constant-budget register swap (2026-09-02): 5.8057 — replacing news units with classical Tashkeela at constant total HURT (−0.98pp vs control), the domain-shaped-residual hypothesis's causal test -failing in the swap direction; the add direction (G2b) remains -open.** The E5/E6 pair is the paper's data-vs-architecture exhibit: +failing in the swap direction; G2b label-scale ADD (2026-09-04, +verdict recorded per TODO.substantiate 04): 4.8231 — 48k total +(+18k classical) at 6 epochs is FLAT vs the 3ep control and 0.25pp +worse than news-only G2a (4.5701): the add neutralized the epoch +gain, closing the domain hypothesis negative in BOTH directions.** +The E5/E6/G2b set is the paper's data-vs-architecture exhibit: two levers from the frontier-LLM literature (MTP, register -diversification at constant budget) both regressed on byte-level -student distillation — the levers that moved the rung were optimizer +diversification — swap or add) plus the label-scale hypothesis all +regressed or flat on byte-level student distillation — the levers +that moved the rung were optimizer (E3), fresher teacher labels (E4), and epochs (G2a). ### 9. Muon optimizer A/B on the memory student (E3) — LANDED 2026-08-28 @@ -226,14 +231,19 @@ The decomposition for paper B, every line full-set with brackets: | + Muon (E3) | 5.2945 | — | | + r7 teacher | 4.8218 | — | | + 6 epochs (G2a) | 4.5701 | delta 2.12 [1.91, 2.35] | +| register swap (E6, 3ep) | 5.8057 | negative | +| register add (G2b, 6ep) | 4.8231 | delta 2.37 [2.19, 2.55] | +| MTP-aux (E5, 3ep) | 5.0853 | negative (confound disclosed) | | depth halved (lite, 6ep) | 5.784 | delta 3.25 [3.03, 3.49] | Paper-B framing: levers compose roughly additively (optimizer >> -teacher > epochs), and the E6 causal test came back NEGATIVE — +teacher > epochs), and BOTH register tests came back NEGATIVE — swapping 8k news units for classical-register Tashkeela at constant -30k total scored 5.8057 (−0.98pp vs control), so the residual is -not fixed by register mix at constant budget; the add direction -(G2b, 48k total) is the live test, and capacity appears ONLY as +30k total scored 5.8057 (−0.98pp vs control), and adding 18k +classical units on top at 6 epochs scored 4.8231 (flat vs the 3ep +control, 0.25pp worse than news-only G2a — the add neutralized the +epoch gain): the residual is not register-shaped in either +direction, and capacity appears ONLY as depth — width is load-bearing (both stitch ratios collapsed) while depth trades 1.21pp for 37% of the artifact. The subset-overstatement phenomenon (five instances, up to 3.2x inflation) is the From bfe945269e3facc3c220f46de9c355bd5208794a Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Fri, 4 Sep 2026 16:17:45 +0200 Subject: [PATCH 2/4] =?UTF-8?q?fix(lint):=20ruff=20clean=20on=20main=20?= =?UTF-8?q?=E2=80=94=204=20autofixes=20from=20the=20concurrent=20merges?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/publish_model.py | 2 +- src/gpu/modal_export.py | 4 ++-- tests/test_parity_stage_skip.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/publish_model.py b/scripts/publish_model.py index 2d29690..9f3a4bb 100644 --- a/scripts/publish_model.py +++ b/scripts/publish_model.py @@ -18,8 +18,8 @@ import hashlib import os import re -import subprocess import shutil +import subprocess import sys import tempfile import zipfile diff --git a/src/gpu/modal_export.py b/src/gpu/modal_export.py index 34f3af1..f8c3d10 100644 --- a/src/gpu/modal_export.py +++ b/src/gpu/modal_export.py @@ -191,7 +191,7 @@ app = modal.App("interscript-ml-export", image=IMAGE) -def normalize_precisions(precisions: "str | list[str]") -> list[str]: +def normalize_precisions(precisions: str | list[str]) -> list[str]: """Accept both invocation forms: the parity/margins entrypoints pass a pre-split list, direct ::parity_model-style CLI calls pass a comma string.""" @@ -220,7 +220,7 @@ def _zip_has_parity(zip_path: Path) -> bool: def pending_precisions( - out_dir: Path, mid: str, precisions: "str | list[str]" + out_dir: Path, mid: str, precisions: str | list[str] ) -> list[str]: """Precision stages still to run. A stage is durably complete only when BOTH its margin report exists and its zip carries the parity diff --git a/tests/test_parity_stage_skip.py b/tests/test_parity_stage_skip.py index 3cd3789..660527f 100644 --- a/tests/test_parity_stage_skip.py +++ b/tests/test_parity_stage_skip.py @@ -17,6 +17,7 @@ def _write_zip(path: Path, parity: bool) -> None: import zipfile + import yaml meta = {"id": "m", "precision": "int4"} From c5f60fccb94e85442049201557997a0d575d57da Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Fri, 4 Sep 2026 16:44:52 +0200 Subject: [PATCH 3/4] fix(lint): exclude the vendored evaluator (ADR-0001) + scoring.py autofixes --- pyproject.toml | 3 +++ src/sadeedbench/scoring.py | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 21649c4..16e6e53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,9 @@ markers = [ [tool.ruff] line-length = 100 target-version = "py310" +# ADR-0001: the vendored Misraj evaluator is a provenance artifact — +# excluded, never rewritten or reformatted (same policy as rababa's copy) +extend-exclude = ["src/sadeedbench/vendored_sadeed_evaluator.py"] [tool.ruff.lint] select = ["E", "F", "I", "B", "UP", "SIM", "C4"] diff --git a/src/sadeedbench/scoring.py b/src/sadeedbench/scoring.py index 8e53fa6..2fd5ec1 100644 --- a/src/sadeedbench/scoring.py +++ b/src/sadeedbench/scoring.py @@ -9,8 +9,6 @@ from __future__ import annotations -from typing import Iterable - def score_predictions(preds: list[str], gts: list[str]) -> dict: """Aggregate DER-CE over aligned (pred, gt) paragraph pairs.""" From ea154e37430ef6a455200a29282ef837781becb3 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Fri, 4 Sep 2026 16:45:30 +0200 Subject: [PATCH 4/4] fix(lint): wrap the long strip-haraqat test line --- tests/test_sadeedbench_cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_sadeedbench_cli.py b/tests/test_sadeedbench_cli.py index b127505..91688fe 100644 --- a/tests/test_sadeedbench_cli.py +++ b/tests/test_sadeedbench_cli.py @@ -56,7 +56,8 @@ def test_score_plain_text_lines(tmp_path: Path, capsys) -> None: def test_bootstrap_vs_reference(tmp_path: Path, capsys) -> None: data = _parquet(tmp_path) cand = _preds(tmp_path, "c.jsonl", [{"idx": i, "student": s} for i, s in enumerate(GT)]) - stripped = [g.replace("َ", "").replace("ُ", "").replace("ْ", "").replace("ً", "").replace("ّ", "") for g in GT] + marks = "ًَُّْ" + stripped = ["".join(c for c in g if c not in marks) for g in GT] ref = _preds(tmp_path, "r.jsonl", [{"idx": i, "student": s} for i, s in enumerate(stripped)]) rc = main(["score", "--preds", str(cand), "--data", str(data), "--key", "student", "--vs", str(ref), "--vs-key", "student"])