From 9a33adb2b9ac52bea56fc7e72b6dac3aef490a87 Mon Sep 17 00:00:00 2001
From: Ronald Tse <ronald.tse@gmail.com>
Date: Sat, 5 Sep 2026 09:40:42 +0200
Subject: [PATCH] feat(sadeedbench): --data accepts the HF dataset id

A non-path --data resolves through huggingface_hub.snapshot_download
(local cache, train.parquet split file); Misraj/SadeedDiac-25 is now a
one-command re-scoring target with no manual download.
---
 pyproject.toml                |  1 +
 src/sadeedbench/cli.py        | 23 ++++++++++++++++++-----
 tests/test_sadeedbench_cli.py | 25 +++++++++++++++++++++++--
 3 files changed, 42 insertions(+), 7 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 16e6e53..7c9eee6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -41,6 +41,7 @@ sadeed = [
   "prettytable>=3.9",
   "pandas>=2.0",
   "pyarrow>=14.0",
+  "huggingface_hub>=0.23",
 ]
 dev = [
   "pytest>=8.0",
diff --git a/src/sadeedbench/cli.py b/src/sadeedbench/cli.py
index 16d26df..04b7d6a 100644
--- a/src/sadeedbench/cli.py
+++ b/src/sadeedbench/cli.py
@@ -37,11 +37,24 @@ def _read_preds(path: Path, key: str) -> list[str]:
     return preds
 
 
-def _load_gold(path: Path) -> list[str]:
+def _load_gold(data: str) -> list[str]:
+    """Gold outputs from a local parquet or an HF dataset id.
+
+    A path (existing file) is read directly. Anything else is treated
+    as a Hugging Face dataset id (default split file train.parquet) and
+    resolved through huggingface_hub.snapshot_download with the local
+    cache."""
     import pandas as pd
 
-    table = pd.read_parquet(path)
-    return table["output"].tolist()
+    p = Path(data)
+    if p.exists():
+        return pd.read_parquet(p)["output"].tolist()
+    import huggingface_hub
+
+    snapshot = Path(
+        huggingface_hub.snapshot_download(data, repo_type="dataset")
+    )
+    return pd.read_parquet(snapshot / "train.parquet")["output"].tolist()
 
 
 def main(argv: list[str] | None = None) -> int:
@@ -49,8 +62,8 @@ def main(argv: list[str] | None = None) -> int:
     sub = parser.add_subparsers(dest="cmd", required=True)
     score = sub.add_parser("score", help="score a predictions file")
     score.add_argument("--preds", type=Path, required=True)
-    score.add_argument("--data", type=Path, required=True,
-                       help="SadeedDiac-25 parquet (input/output columns)")
+    score.add_argument("--data", required=True,
+                       help="parquet path or HF dataset id (Misraj/SadeedDiac-25)")
     score.add_argument("--key", default="student",
                        help="JSONL row key carrying the prediction")
     score.add_argument("--vs", type=Path, help="reference predictions file")
diff --git a/tests/test_sadeedbench_cli.py b/tests/test_sadeedbench_cli.py
index 91688fe..1969f19 100644
--- a/tests/test_sadeedbench_cli.py
+++ b/tests/test_sadeedbench_cli.py
@@ -20,8 +20,8 @@
 GT = ["قَوْلُهُ فَحُكْمُهَا", "مُكْتَبَّةٌ جَمِيلَةٌ"]
 
 
-def _parquet(tmp_path: Path) -> Path:
-    p = tmp_path / "bench.parquet"
+def _parquet(tmp_path: Path, name: str = "bench.parquet") -> Path:
+    p = tmp_path / name
     pd.DataFrame({"input": ["قوله فحكمها", "مكتبة جميلة"], "output": GT}).to_parquet(p)
     return p
 
@@ -64,3 +64,24 @@ def test_bootstrap_vs_reference(tmp_path: Path, capsys) -> None:
     out = json.loads(capsys.readouterr().out)
     assert rc == 0
     assert out["vs"]["delta"] < 0  # candidate (perfect) better than stripped reference
+
+
+def test_data_accepts_hf_dataset_id(monkeypatch, tmp_path, capsys) -> None:
+    # --data may name the HF dataset instead of a local parquet; the
+    # loader resolves it through huggingface_hub with a local cache
+    import sadeedbench.cli as cli
+
+    cached = _parquet(tmp_path, name="train.parquet")
+    calls = {}
+
+    def fake_snapshot(repo_id, repo_type):
+        calls["repo_id"] = repo_id
+        return str(cached.parent)
+
+    monkeypatch.setattr("huggingface_hub.snapshot_download", fake_snapshot, raising=False)
+    rc = cli.main(["score", "--preds", str(_preds(tmp_path, "p.jsonl", [
+        {"idx": i, "student": g} for i, g in enumerate(GT)])), "--data", "Misraj/SadeedDiac-25"])
+    assert rc == 0
+    assert calls["repo_id"] == "Misraj/SadeedDiac-25"
+    out = json.loads(capsys.readouterr().out)
+    assert out["der_ce"] == 0.0