From f1faf2b040b51b96a76b179545713adccdc6989a Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Fri, 4 Sep 2026 15:47:35 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20interscript-sadeed-eval=20=E2=80=94=20t?= =?UTF-8?q?he=20DER-CE=20protocol=20as=20a=20standalone=20tool?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third parties can now re-score any model's predictions on the public SadeedDiac-25 benchmark under the campaign's exact convention: windowed DER-CE via the vendored Misraj evaluator (public source, attribution preserved), paired bootstrap vs a reference predictions file, final_preds.jsonl or plain-text input. modal_distill's bootstrap now delegates to the shared pure home (sadeedbench.bootstrap). Validated end-to-end: scoring the G2b run's raw final_preds.jsonl against the benchmark parquet reproduces the published verdict exactly (4.8231 n=1200; delta 2.3717 [2.194, 2.555]). --- pyproject.toml | 9 +- src/gpu/modal_distill.py | 26 +- src/sadeedbench/__init__.py | 4 + src/sadeedbench/bootstrap.py | 47 ++ src/sadeedbench/cli.py | 77 +++ src/sadeedbench/scoring.py | 48 ++ src/sadeedbench/vendored_sadeed_evaluator.py | 523 +++++++++++++++++++ tests/test_sadeedbench_bootstrap.py | 35 ++ tests/test_sadeedbench_cli.py | 65 +++ tests/test_sadeedbench_scoring.py | 36 ++ 10 files changed, 850 insertions(+), 20 deletions(-) create mode 100644 src/sadeedbench/__init__.py create mode 100644 src/sadeedbench/bootstrap.py create mode 100644 src/sadeedbench/cli.py create mode 100644 src/sadeedbench/scoring.py create mode 100644 src/sadeedbench/vendored_sadeed_evaluator.py create mode 100644 tests/test_sadeedbench_bootstrap.py create mode 100644 tests/test_sadeedbench_cli.py create mode 100644 tests/test_sadeedbench_scoring.py diff --git a/pyproject.toml b/pyproject.toml index 13a0867..21649c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,12 @@ publish = [ "huggingface_hub>=0.23", "omegaconf>=2.3", ] +sadeed = [ + "pyarabic>=0.6", + "prettytable>=3.9", + "pandas>=2.0", + "pyarrow>=14.0", +] dev = [ "pytest>=8.0", "pytest-cov>=5.0", @@ -45,10 +51,11 @@ dev = [ [project.scripts] interscript-ml = "src.cli:main" +interscript-sadeed-eval = "sadeedbench.cli:main" [tool.setuptools.packages.find] where = ["src"] -include = ["framework*", "tasks*", "imf*"] +include = ["framework*", "tasks*", "imf*", "sadeedbench*"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/gpu/modal_distill.py b/src/gpu/modal_distill.py index 2339614..ddf5f9e 100644 --- a/src/gpu/modal_distill.py +++ b/src/gpu/modal_distill.py @@ -210,25 +210,13 @@ def _maybe_stitch(spec_id: str, spec: dict, student) -> None: def paired_bootstrap(deltas: list[float], seed: int = 42, n: int = 1000) -> dict: """Sentence-level paired bootstrap over per-item DER deltas (microkimi eval_compare protocol): a point delta without a CI is - not evidence. Deterministic under the fixed default seed.""" - import random - import statistics - - if not deltas: - raise ValueError("empty deltas") - rng = random.Random(seed) - size = len(deltas) - means = sorted( - statistics.fmean(deltas[rng.randrange(size)] for _ in range(size)) - for _ in range(n) - ) - ci = (round(means[int(0.025 * n)], 3), round(means[int(0.975 * n) - 1], 3)) - delta = statistics.fmean(deltas) - return { - "delta": round(delta, 4), - "ci95": ci, - "p_leq0": round(sum(1 for m in means if m <= 0) / n, 4), - } + not evidence. Deterministic under the fixed default seed. + Delegates to sadeedbench.bootstrap (the pure shared home).""" + from sadeedbench.bootstrap import bootstrap_means + + out = bootstrap_means(deltas, seed=seed, n=n) + out["ci95"] = tuple(round(v, 3) for v in out["ci95"]) + return out def _ensure_src_path() -> None: diff --git a/src/sadeedbench/__init__.py b/src/sadeedbench/__init__.py new file mode 100644 index 0000000..acb6856 --- /dev/null +++ b/src/sadeedbench/__init__.py @@ -0,0 +1,4 @@ +"""SadeedDiac-25 scoring as a standalone tool: the campaign's windowed +DER-CE protocol (zero-skip, haraqat-projected, greedy) packaged so +third parties can re-score any model's predictions — including ONNX +artifacts through any IMF runtime — against the published benchmark.""" diff --git a/src/sadeedbench/bootstrap.py b/src/sadeedbench/bootstrap.py new file mode 100644 index 0000000..7f8a1f5 --- /dev/null +++ b/src/sadeedbench/bootstrap.py @@ -0,0 +1,47 @@ +"""Sentence-level paired bootstrap over per-item metric deltas: a +point delta without a CI is not evidence. Deterministic under the +fixed default seed (42, 1,000 resamples, percentile CIs).""" + +from __future__ import annotations + +import random +import statistics +from collections.abc import Iterable + + +def bootstrap_means(deltas: list[float], seed: int = 42, n: int = 1000) -> dict: + """Bootstrap the mean of a delta list (percentile CIs, fixed seed).""" + if not deltas: + raise ValueError("empty deltas") + rng = random.Random(seed) + m = len(deltas) + means = [ + statistics.fsum(rng.choice(deltas) for _ in range(m)) / m for _ in range(n) + ] + means.sort() + lo = means[int(0.025 * n)] + hi = means[min(n - 1, int(0.975 * n))] + delta = statistics.fsum(deltas) / m + p_leq0 = sum(1 for u in means if u <= 0) / n + return { + "delta": round(delta, 4), + "ci95": (round(lo, 4), round(hi, 4)), + "p_leq0": round(p_leq0, 4), + "n": m, + } + + +def bootstrap_delta( + a: Iterable[float | None], + b: Iterable[float | None], + seed: int = 42, + n: int = 1000, +) -> dict: + """Bootstrap the mean of (a - b) over pairwise-aligned items; pairs + with a None on either side are dropped (the aggregate DER call + skips paragraphs with no scorable positions). (candidate, + reference): positive delta = candidate worse.""" + deltas = [x - y for x, y in zip(a, b, strict=True) if x is not None and y is not None] + if not deltas: + raise ValueError("no scorable pairs") + return bootstrap_means(deltas, seed=seed, n=n) diff --git a/src/sadeedbench/cli.py b/src/sadeedbench/cli.py new file mode 100644 index 0000000..16d26df --- /dev/null +++ b/src/sadeedbench/cli.py @@ -0,0 +1,77 @@ +"""interscript-sadeed-eval — score predictions on SadeedDiac-25 under +the campaign's windowed DER-CE convention. + + interscript-sadeed-eval score \\ + --preds final_preds.jsonl --data sadeed-diac-25.parquet \\ + [--key student] [--vs reference.jsonl] [--vs-key teacher] + +Predictions are read as JSONL rows (any key; --key selects, default +"student" — the training harness's final_preds.jsonl writes +idx/src/teacher/student) or as plain text lines. The parquet carries +the benchmark's input/output columns; fetch SadeedDiac-25 from +https://huggingface.co/datasets/Misraj/SadeedDiac-25. Output is JSON +on stdout; --vs adds a paired-bootstrap delta (candidate minus +reference; positive = candidate worse).""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +def _read_preds(path: Path, key: str) -> list[str]: + preds: list[str] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + preds.append(line) + continue + if not isinstance(row, dict) or key not in row: + raise SystemExit(f"row is not a dict with key {key!r}: {line[:60]}") + preds.append(row[key]) + return preds + + +def _load_gold(path: Path) -> list[str]: + import pandas as pd + + table = pd.read_parquet(path) + return table["output"].tolist() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="interscript-sadeed-eval") + sub = parser.add_subparsers(dest="cmd", required=True) + score = sub.add_parser("score", help="score a predictions file") + score.add_argument("--preds", type=Path, required=True) + score.add_argument("--data", type=Path, required=True, + help="SadeedDiac-25 parquet (input/output columns)") + score.add_argument("--key", default="student", + help="JSONL row key carrying the prediction") + score.add_argument("--vs", type=Path, help="reference predictions file") + score.add_argument("--vs-key", default="teacher") + args = parser.parse_args(argv) + + from sadeedbench.bootstrap import bootstrap_delta + from sadeedbench.scoring import per_item_der, score_predictions + + gts = _load_gold(args.data) + preds = _read_preds(args.preds, args.key) + result = score_predictions(preds, gts) + if args.vs: + ref = _read_preds(args.vs, args.vs_key) + result["vs"] = bootstrap_delta( + per_item_der(preds, gts), per_item_der(ref, gts) + ) + json.dump(result, sys.stdout, ensure_ascii=False) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/sadeedbench/scoring.py b/src/sadeedbench/scoring.py new file mode 100644 index 0000000..8e53fa6 --- /dev/null +++ b/src/sadeedbench/scoring.py @@ -0,0 +1,48 @@ +"""Score predictions against SadeedDiac-25 gold under the campaign's +windowed DER-CE convention. + +The convention (docs/paper-a.adoc section 3): every paragraph counts +(zero-skip), haraqat are projected before scoring, and the aggregate is +the Misraj evaluator's Total DER with case endings, reported in +percent. Predictions are whatever a model emitted for the full test +set — same order as the benchmark rows.""" + +from __future__ import annotations + +from typing import Iterable + + +def score_predictions(preds: list[str], gts: list[str]) -> dict: + """Aggregate DER-CE over aligned (pred, gt) paragraph pairs.""" + from sadeedbench.vendored_sadeed_evaluator import ( + ArabicDiacritizationEvaluator, + ) + + if len(preds) != len(gts): + raise ValueError(f"preds/gts length mismatch: {len(preds)} vs {len(gts)}") + _, _, total_der, _, _ = ArabicDiacritizationEvaluator.caculate_errors_on_sentences( + preds, gts, gt_missing_diacritic_is_error=False + ) + return {"der_ce": round(float(total_der), 4), "n": len(preds)} + + +def per_item_der(preds: list[str], gts: list[str]) -> list[float | None]: + """Per-paragraph DER for paired bootstrap; None where the paragraph + carries no scorable positions (ZeroDivisionError in the evaluator's + single-item mode — the aggregate call skips those paragraphs).""" + from sadeedbench.vendored_sadeed_evaluator import ( + ArabicDiacritizationEvaluator, + ) + + ders: list[float | None] = [] + for pred, gt in zip(preds, gts, strict=True): + try: + _, _, d, _, _ = ( + ArabicDiacritizationEvaluator.caculate_errors_on_sentences( + [pred], [gt], gt_missing_diacritic_is_error=False + ) + ) + ders.append(float(d)) + except ZeroDivisionError: + ders.append(None) + return ders diff --git a/src/sadeedbench/vendored_sadeed_evaluator.py b/src/sadeedbench/vendored_sadeed_evaluator.py new file mode 100644 index 0000000..adaf0bf --- /dev/null +++ b/src/sadeedbench/vendored_sadeed_evaluator.py @@ -0,0 +1,523 @@ +"""Vendored Arabic diacritization evaluator from Misraj (Sadeed). + +Source: https://github.com/misraj-ai/Sadeed Eval_code.ipynb (cell 3), +MIT-style released alongside the SadeedDiac-25 benchmark +(https://huggingface.co/datasets/Misraj/SadeedDiac-25). +Unmodified except for this header. + +Metrics: Total DER (= DER with case endings), Morphological DER +(= DER without case endings, word-final letter excluded), plus WERs. +""" + +from pyarabic import araby +from prettytable import PrettyTable +import re +from tqdm import tqdm +import warnings +import pandas as pd + +class ArabicDiacritizationEvaluator: + """ + A utility class for evaluating Arabic diacritization, + """ + STANDARD_ARABIC_LETTER_PATTERN = r"ءآأؤإئابةتثجحخدذرزسشصضطظعغفقكلمنهوىي" + STANDARD_HARAKA_PATTERN = r'ًٌٍَُِّْٓ' #harakat, madda and shadda + TATWEEL_PATTERN = r"ـ" + QURANIC_AND_ISLAMIC_ANNOTATION_SIGNS = r'ؘؙؚٰؐؑؒؓؕؗ٘ۖۗۘۙۚۛۜٱ۝۞ۣ۟۟۠ۡۢۤۥۦۧۨ۩۪ۭٕ۫۬۬ٗٔࣔࣕࣖࣗࣘࣙࣚࣛࣜࣝࣞࣟ࣠࣡࣢ࣰࣱࣲࣳ﴾﴿ﷰﷱﷲﷳﷳﷴﷵﷶﷷﷸﷺﷻ﷼﷽' + NON_ARABIC_SYMOBOLS_IN_ARABIC_BLOCK = r"؀؁؂؃؄؅؎؏ؘؙؚؔؖ؋" + NON_ARABIC_LETTERS_IN_ARABIC_UNICODE_BLOCK = r'ؠػؼؽؾؿٮٯٲٳٴٵٶٷٸٹٺٻټٽپٿڀځڂڃڄڅچڇڈډڊڋڌڍڎڏڐڑڒړڔڕږڗژڙښڛڜڝڞڟڠڡڢڣڤڥڦڧڨکڪګڬڭڮگڰڱڲڳڴڵڶڷڸڹںڻڼڽھڿۀہۂۃۄۅۆۇۈۉۊۋیۍێۏېۑےۓەۮۯۺۻۼ۽۾ۿ' + MODOUD_LETTERS = [araby.ALEF, araby.ALEF_MADDA, araby.ALEF_WASLA,araby.ALEF_MAKSURA,araby.SMALL_ALEF, araby.WAW, araby.SMALL_WAW, araby.YEH, araby.SMALL_YEH] + + NO_HARAKA = '*' + ARABIC_LETTERS_PATTERN = r"["+STANDARD_ARABIC_LETTER_PATTERN+"]+" + + FULLY_EXTENED_ARABIC_WORD = ( + STANDARD_ARABIC_LETTER_PATTERN + + STANDARD_HARAKA_PATTERN + + TATWEEL_PATTERN + + QURANIC_AND_ISLAMIC_ANNOTATION_SIGNS + + NON_ARABIC_SYMOBOLS_IN_ARABIC_BLOCK + + NON_ARABIC_LETTERS_IN_ARABIC_UNICODE_BLOCK) + + + @classmethod + def split_arabic_text(cls, text: str) -> list: + """ + Splits Arabic text into words and symbols based on extended Arabic characters. + """ + det_chars = cls.FULLY_EXTENED_ARABIC_WORD + pattern = f"([{re.escape(det_chars)}]+)" + result = re.split(pattern, text) + result = [word for word in result if word] # Remove empty strings + return result + + @classmethod + def extract_harakat(cls, word: str, shadda_is_letter: bool = False): + """ + Extracts harakat and corresponding letters from a diacritized word. + returns harakat , letters, only one haraka is allowed + """ + harakat = [] + letters = [] + i = -1 + while(word != ""): + char = word[0] + word = word[1:] + if araby.is_haraka(char) or araby.is_shadda(char): + if len(letters)>0: #pass harakat before word + if char == araby.SHADDA: + if shadda_is_letter: + if letters[i].endswith(araby.SHADDA): # there is already a shadda + pass + else: + letters[i] = letters[i] + araby.SHADDA + else: + if (harakat[i] == cls.NO_HARAKA): + harakat[i] = char + else: # not shadda + if (harakat[i] == cls.NO_HARAKA): + harakat[i] = char + elif harakat[i] == araby.SHADDA: + harakat[i] = harakat[i] + char #only allow one haraka after shadda + else: + pass #ignore + else: pass#pass harakat before word + else: # it is letter + letters.append(char) + harakat.append(cls.NO_HARAKA) + i += 1 + + return harakat, letters + + + @classmethod + def has_arabic_letter(cls, str): + result = re.findall(cls.ARABIC_LETTERS_PATTERN, str) + return (not result == None and not result == []) + + @classmethod + def has_al_alta3reef(cls, word: str, must_have_voweles: bool = False): + ''' + checks to see if the word has al alta3reef + this fucntion may give errors if the word starts with something that looks like al alta3reef الزم + @returns true or false + the letter order of the begining of al alt3reef (if there are prefixes this will not + be 0, else it will be 0) or -1 if no al alt3reef was found + + ''' + + harakat, letters = cls.extract_harakat(word) + if len(letters) < 4: + return False, -1 + # + no_harak_or_sukun = [cls.NO_HARAKA , araby.SUKUN] + no_harak_or_fatha = [cls.NO_HARAKA , araby.FATHA] + no_harak_or_kasra = [cls.NO_HARAKA , araby.KASRA] + # check the prefix letters before the word that can take al alt3reef after + # وَ فَ لِ أَ بِ كَ لً سَ + index = 0 + if must_have_voweles: + while (True): + if index >= len(letters): + return False , -1 + if letters[index] == araby.WAW and harakat[index] == araby.FATHA: # وَ واو العطف + index += 1 + elif letters[index] == araby.FEH and harakat[index] == araby.FATHA: #فَ فاء العطف + index += 1 + elif letters[index] == araby.BEH and harakat[index] == araby.KASRA: # بِ باء الجر + index += 1 + elif letters[index] == araby.KAF and harakat[index] == araby.FATHA: # كَ كاف التشبيه + index += 1 + elif letters[index] == araby.ALEF_HAMZA_ABOVE and harakat[index] == araby.FATHA: # أ همزة إستفهام + index += 1 + # لا تأتي مع ال التعريف + # elif letters[index] == araby.SEEN and harakat[index] in no_harak_or_fatha: # سَ سين المستقبل + # index += 1 + else: + break + else: # don't check vowles + while (True): + if index >= len(letters): + return False , -1 + if letters[index] == araby.WAW and harakat[index] in no_harak_or_fatha: # وَ واو العطف + index += 1 + elif letters[index] == araby.FEH and harakat[index] in no_harak_or_fatha: #فَ فاء العطف + index += 1 + elif letters[index] == araby.BEH and harakat[index] in no_harak_or_kasra: # بِ باء الجر + index += 1 + elif letters[index] == araby.KAF and harakat[index] in no_harak_or_fatha: # كَ كاف التشبيه + index += 1 + elif letters[index] == araby.ALEF_HAMZA_ABOVE and harakat[index] in no_harak_or_fatha: # أ همزة إستفهام + index += 1 + # لا تأتي مع ال التعريف + # elif letters[index] == araby.SEEN and harakat[index] in no_harak_or_fatha: # سَ سين المستقبل + # index += 1 + else: + break + + + # there must be letters left because it didn't return + letters = letters[index:] + harakat = harakat[index:] + + if len(letters) < 4: #السَمّ أقل شي حرفين بعد أل التعريف + return False, -1 + + next_letter_haraka = harakat[2] + if letters[0] == araby.ALEF and letters[1] == araby.LAM: + if harakat[0] == cls.NO_HARAKA: + if (letters[2] == araby.ALEF and harakat[1] == araby.KASRA): #حالة الِانْدِلَاعِ عليها كسرة + return True, index + if must_have_voweles: + # الشمسي ما في عاللام حركة والقمري عاللام سكون + if (harakat[1] == cls.NO_HARAKA and araby.SHADDA in next_letter_haraka) or\ + (harakat[1] == araby.SUKUN and not araby.SHADDA in next_letter_haraka): + return True, index + else: + if harakat[1] in no_harak_or_sukun: + return True, index + return False ,-1 + + + #لِلِاسْتِفَادَةِ لَلسَّماء + @classmethod + def has_ll_alta3reef(cls, word: str, must_have_voweles: bool = False): + harakat, letters = cls.extract_harakat(word) + if len(letters) < 4: + return False, -1 + no_harak_or_sukun = [cls.NO_HARAKA , araby.SUKUN] + no_harak_or_fatha = [cls.NO_HARAKA , araby.FATHA] + no_harak_or_kasra_or_fatha = [cls.NO_HARAKA , araby.KASRA, araby.FATHA] + kasra_or_fatha = [araby.KASRA, araby.FATHA] + index = 0 + if must_have_voweles: + while (True): + if index >= len(letters): + return False , -1 + if letters[index] == araby.WAW and harakat[index] == araby.FATHA: # وَ واو العطف + index += 1 + elif letters[index] == araby.FEH and harakat[index] == araby.FATHA: #فَ فاء العطف + index += 1 + elif letters[index] == araby.ALEF_HAMZA_ABOVE and harakat[index] == araby.FATHA: # أ همزة إستفهام + index += 1 + else: + break + else: + while (True): + if index >= len(letters): + return False , -1 + if letters[index] == araby.WAW and harakat[index] in no_harak_or_fatha: # وَ واو العطف + index += 1 + elif letters[index] == araby.FEH and harakat[index] in no_harak_or_fatha: #فَ فاء العطف + index += 1 + elif letters[index] == araby.ALEF_HAMZA_ABOVE and harakat[index] in no_harak_or_fatha: # أ همزة إستفهام + index += 1 + else: + break + letters = letters[index:] + harakat = harakat[index:] + + if len(letters) < 4: #فللاسم) أقل شي حرفين بعد لل) + return False, -1 # استثناء حالة بلل (مبلول) + + next_letter_haraka = harakat[2] + + if letters[0] == araby.LAM and letters[1] == araby.LAM: + if must_have_voweles: + if harakat[0] in kasra_or_fatha: + if (harakat[1] == cls.NO_HARAKA and araby.SHADDA in next_letter_haraka) or\ + (harakat[1] == araby.SUKUN and not araby.SHADDA in next_letter_haraka ) or\ + (letters[2] == araby.ALEF and harakat[1] == araby.KASRA): #حالة لِلِاسْتِفَادَةِ عليها كسرة: + return True, index + else: + if harakat[0] in no_harak_or_kasra_or_fatha: + if harakat[1] in no_harak_or_sukun or\ + (letters[2] == araby.ALEF and harakat[1] == araby.KASRA): #حالة لِلِاسْتِفَادَةِ عليها كسرة + return True, index + + return False ,-1 + + + @classmethod + def is_mad_letter(cls, letter, haraka, prev_haraka) -> bool: + return araby.is_alef(letter) or (letter == araby.WAW and haraka == cls.NO_HARAKA and araby.DAMMA in prev_haraka)\ + or(letter == araby.YEH and haraka == cls.NO_HARAKA and araby.KASRA in prev_haraka) + + @classmethod + def is_fully_diacritized(cls, word: str, count_last_haraka: bool = True) -> bool: + ''' + checksi whether every letter has dicaritic except for moduod letters and al alta3eef + NOTE: this may give worng answer if there must be haraka on waw or yeh + or the word starts with alef lam not al alta3reef + ''' + harakat,letters = cls.extract_harakat(word) + + # check haraka on al alta3reef and ll alta3reef + if len(letters) > 3: + has_al, al_index = cls.has_al_alta3reef(word, must_have_voweles=True) + if has_al: + if al_index + 2 <= len(harakat): # Ensure index is within bounds + harakat = harakat[:al_index] + harakat[al_index + 2:] + letters = letters[:al_index] + letters[al_index + 2:] + else: + has_ll, ll_index = cls.has_ll_alta3reef(word, must_have_voweles=True) + if has_ll: + if ll_index + 2 <= len(harakat): # Ensure index is within bounds + harakat = harakat[:ll_index] + harakat[ll_index + 2:] + letters = letters[:ll_index] + letters[ll_index + 2:] + + test_range = range(len(harakat)) if count_last_haraka else range(len(harakat) - 1) + + if letters and letters[0] in [araby.WAW, araby.YEH]: + if harakat[0] == cls.NO_HARAKA: + return False + + for i in test_range: + if i >= len(letters) or i >= len(harakat): + return False # Handle cases where the indices exceed the length of lists + + if letters[i] in cls.MODOUD_LETTERS: + if i > 0 and cls.is_mad_letter(letters[i], harakat[i], harakat[i - 1]): + continue + else: + if harakat[i] == cls.NO_HARAKA: + return False + if harakat[i] == araby.SHADDA: + return False + + return True + + @classmethod + def caculate_error_on_single_sentence (cls, voweled_sentence, ground_truth_sentence,\ + gt_missing_diacritic_is_error= False): + + voweled_words = cls.split_arabic_text(voweled_sentence.strip()) + gt_words = cls.split_arabic_text(ground_truth_sentence.strip()) + + + word_count = 0 + letter_count = 0 + + not_voweled_words_count =0 + + total_wer = 0 + morph_wer = 0 + total_der = 0 + morph_der = 0 + + if len(voweled_words) != len(gt_words): + raise RuntimeError("sentences words are not the same lenght"+"[" + str(len(voweled_words)) + "] , [" + str(len(gt_words)) + "]"+" :\nsentnece 1: " + voweled_sentence + "\nsentence 2: " + ground_truth_sentence + "\n***********************************") + + for i in range(len(voweled_words)): + v_word = voweled_words[i].strip() + gt_word = gt_words[i].strip() + + if v_word == "" and gt_word == "": + continue + + if not cls.has_arabic_letter(v_word): #not an arabic word + if not v_word == gt_word: + warnings.warn(f"non_arabic word is not the same [{v_word}] , [{gt_word}]") + # skip + continue + + ###### it is arabic word + if not cls.is_fully_diacritized(v_word): + not_voweled_words_count +=1 + + + v_harakat, v_letters = cls.extract_harakat(v_word) + gt_harakat, gt_letters = cls.extract_harakat(gt_word) + word_count +=1 + letter_count += len(gt_letters) + + if v_letters != gt_letters: + raise RuntimeError("words don't match [" + v_word + "] , [" + gt_word + "] in sentence" + voweled_sentence) + + correct_all_but_last_letter = True + correct_last_letter = True + + if gt_missing_diacritic_is_error or cls.is_fully_diacritized(gt_word): # check all letters + # go through all harakat except the last one + for j in range(len(v_harakat) - 1): + if v_harakat[j] != gt_harakat[j]: + total_der += 1 + morph_der += 1 + correct_all_but_last_letter = False + if not correct_all_but_last_letter: + morph_wer +=1 + # check last letter + if v_harakat[-1] != gt_harakat[-1]: + total_der += 1 + correct_last_letter = False + if not(correct_last_letter and correct_all_but_last_letter): + total_wer +=1 + else: + # skip missing for missing gt diacritics if i am not missing it too + + for j in range(len(v_harakat) - 1): + + if not v_harakat[j] == gt_harakat[j] and gt_harakat[j] == cls.NO_HARAKA: + letter_count -= 1 + continue + if v_harakat[j] != gt_harakat[j]: + total_der += 1 + morph_der += 1 + correct_all_but_last_letter = False + if not correct_all_but_last_letter: + morph_wer +=1 + #check last letter + if not v_harakat[-1] == gt_harakat[-1] and gt_harakat[-1] == cls.NO_HARAKA: + letter_count -= 1 + pass # last haraka is missing + else: + if v_harakat[-1] != gt_harakat[-1]: + total_der += 1 + correct_last_letter = False + if not(correct_last_letter and correct_all_but_last_letter): + total_wer +=1 + + return word_count, letter_count, not_voweled_words_count, total_wer, morph_wer, total_der, morph_der + + + @classmethod + def caculate_errors_on_sentences (cls, voweled_sentences, ground_truth_sentences, gt_missing_diacritic_is_error= False) : + if len(voweled_sentences) != len(ground_truth_sentences): + raise RuntimeError("sentences are not the same lenght [" + str(len(voweled_sentences)) + "] , [" + str(len(ground_truth_sentences)) + "]") + Total_WER_count = 0 + Morph_WER_count = 0 + Total_DER_count = 0 + Morph_DER_count = 0 + + total_word_count = 0 + total_letter_count = 0 + total_not_voweled_words_count = 0 + + + for i in tqdm(range(len(voweled_sentences))): + try: + wc, lc, nvw, w_total_wer, w_morph_wer,w_total_der,w_morph_der = cls.caculate_error_on_single_sentence(voweled_sentences[i].strip(), ground_truth_sentences[i].strip(),\ + gt_missing_diacritic_is_error = gt_missing_diacritic_is_error) + except RuntimeError as r: + warnings.warn(f"Skipping example #{i}: because of {r}", RuntimeWarning) + continue + + + total_word_count += wc + total_letter_count += lc + total_not_voweled_words_count += nvw + + Total_WER_count += w_total_wer + Morph_WER_count += w_morph_wer + Total_DER_count += w_total_der + Morph_DER_count += w_morph_der + + + Total_WER = Total_WER_count/total_word_count *100 + Morph_WER = Morph_WER_count/total_word_count *100 + Total_DER = Total_DER_count/total_letter_count *100 + Morph_DER = Morph_DER_count/total_letter_count *100 + NVW = total_not_voweled_words_count/total_word_count * 100 + + return Total_WER, Morph_WER, Total_DER, Morph_DER, NVW + + + @classmethod + def print_pretty_table(cls,Total_WER, Morph_WER, Total_DER, Morph_DER, NVW, gt_missing_diacritic_is_error): + + t = PrettyTable(['Count', 'Value']) + # print("Countng 1: tagger missing is error") + if gt_missing_diacritic_is_error: + t.add_row(["------- Benchmark Diacritization missing is counted ERROR -------" , "--------" ]) + else: + t.add_row(["------- Benchmark Diacritization missing is SKIPPED -------" , "--------" ]) + + t.add_row(["Morpholocial DER" , "{:.4f}".format(Morph_DER)]) + t.add_row(["Total DER" , "{:.4f}".format(Total_DER)]) + t.add_row(["Morphological WER" , "{:.4f}".format(Morph_WER)]) + t.add_row(["Total WER" , "{:.4f}".format(Total_WER)]) + t.add_row(["",""]) + t.add_row(["Not Fully Diac Words Rate" , "{:.4f}".format(NVW)]) + t.add_row(["",""]) + t.add_row(["************************************","********"]) + + print() + print(t) + + + @classmethod + def report_error_on_senenteces(cls, diacritized_sentences, ground_truth_sentences, gt_missing_diacritic_is_error = False): + """ + Calculates and prints diacritization error metrics for a list of sentences. + + This method compares predicted diacritized sentences against ground truth sentences, + computes evaluation metrics (WER and DER), and prints the results in a formatted table. + + Args: + diacritized_sentences (List[str]): List of predicted diacritized sentences. + ground_truth_sentences (List[str]): List of ground truth fully diacritized sentences. + gt_missing_diacritic_is_error (bool, optional): If True, missing diacritics in ground truth + are treated as errors during evaluation, otherwise it is skipped. Defaults to False. + + Returns: + None + """ + print(f"Calculating Diacritization Errors for {len(diacritized_sentences)} sentences") + Total_WER, Morph_WER, Total_DER, Morph_DER, NVW = ArabicDiacritizationEvaluator.caculate_errors_on_sentences(\ + diacritized_sentences, ground_truth_sentences, gt_missing_diacritic_is_error= gt_missing_diacritic_is_error) + cls.print_pretty_table(Total_WER, Morph_WER, Total_DER, Morph_DER, NVW, gt_missing_diacritic_is_error) + + + + @classmethod + def report_errors_on_csv_file (cls, file_path, ground_truth_column_index = 0,\ + predicted_column_index = 1,\ + has_header = True,\ + gt_missing_diacritic_is_error = False): + """ + Calculates diacritization error metrics from a CSV file and prints the results. + + This method reads a CSV file containing ground truth and predicted diacritized sentences, + computes error metrics including Word Error Rate (WER) and Diacritic Error Rate (DER), + and prints a formatted summary of the results. + + Args: + file_path (str): Path to the input CSV file. + ground_truth_column_index (int, optional): Index of the column containing ground truth sentences. + Defaults to 0. + predicted_column_index (int, optional): Index of the column containing predicted diacritized sentences. + Defaults to 1. + has_header (bool, optional): Indicates whether the CSV file has a header row. If True, skips the first row. + Defaults to False. + gt_missing_diacritic_is_error (bool, optional): Whether missing diacritics in ground truth should + be considered errors. Defaults to False. + + Raises: + FileNotFoundError: If the provided CSV file path does not exist. + pd.errors.EmptyDataError: If the file exists but is empty. + Exception: For any other unexpected errors during processing. + + Returns: + None + """ + try: + if has_header: + df = pd.read_csv(file_path, header=0) + else: + df = pd.read_csv(file_path, header = None) + gt_sentences_list = df.iloc[:, ground_truth_column_index].tolist() + diacritized_sentences_list = df.iloc[:, predicted_column_index].tolist() + print(f"Calculating Diacritization Errors for {len(diacritized_sentences_list)} sentences") + + Total_WER, Morph_WER, Total_DER, Morph_DER, NVW = ArabicDiacritizationEvaluator.caculate_errors_on_sentences(\ + diacritized_sentences_list, gt_sentences_list, gt_missing_diacritic_is_error= gt_missing_diacritic_is_error) + cls.print_pretty_table(Total_WER, Morph_WER, Total_DER, Morph_DER, NVW, gt_missing_diacritic_is_error) + + except FileNotFoundError: + print(f"Error: File '{file_path}' not found.") + return + except pd.errors.EmptyDataError: + print(f"Error: File '{file_path}' is empty.") + return + except Exception as e: + print(f"An unexpected error occurred: {e}") + return diff --git a/tests/test_sadeedbench_bootstrap.py b/tests/test_sadeedbench_bootstrap.py new file mode 100644 index 0000000..b7f7808 --- /dev/null +++ b/tests/test_sadeedbench_bootstrap.py @@ -0,0 +1,35 @@ +"""Paired bootstrap lives in its pure home (no modal import) so the +CLI and the training harness share one implementation.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) + +from sadeedbench.bootstrap import bootstrap_delta # noqa: E402 + + +def test_delta_and_ci_exclude_zero() -> None: + # (candidate, reference): positive delta = candidate worse + cand = [2.0] * 50 + ref = [1.0] * 50 + out = bootstrap_delta(cand, ref) + assert abs(out["delta"] - 1.0) < 1e-9 + assert out["ci95"][0] > 0 + assert out["p_leq0"] < 0.001 + + +def test_none_items_dropped_pairwise() -> None: + cand = [2.0, None, 2.0] + ref = [1.0, None, 1.0] + out = bootstrap_delta(cand, ref) + assert out["n"] == 2 + assert abs(out["delta"] - 1.0) < 1e-9 + + +def test_deterministic_seed() -> None: + cand = [2.0, 3.0, 4.0, 5.0] + ref = [1.0, 2.0, 3.0, 4.0] + assert bootstrap_delta(cand, ref) == bootstrap_delta(cand, ref) diff --git a/tests/test_sadeedbench_cli.py b/tests/test_sadeedbench_cli.py new file mode 100644 index 0000000..b127505 --- /dev/null +++ b/tests/test_sadeedbench_cli.py @@ -0,0 +1,65 @@ +"""The CLI: score a predictions file (final_preds.jsonl compatible or +plain text lines) against the benchmark parquet, optionally with a +paired bootstrap vs a reference predictions file.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) + +pytest.importorskip("pyarabic") +pd = pytest.importorskip("pandas") + +from sadeedbench.cli import main # noqa: E402 + +GT = ["قَوْلُهُ فَحُكْمُهَا", "مُكْتَبَّةٌ جَمِيلَةٌ"] + + +def _parquet(tmp_path: Path) -> Path: + p = tmp_path / "bench.parquet" + pd.DataFrame({"input": ["قوله فحكمها", "مكتبة جميلة"], "output": GT}).to_parquet(p) + return p + + +def _preds(tmp_path: Path, name: str, rows: list[dict]) -> Path: + p = tmp_path / name + with p.open("w", encoding="utf-8") as fh: + for r in rows: + fh.write(json.dumps(r, ensure_ascii=False) + "\n") + return p + + +def test_score_final_preds_format(tmp_path: Path, capsys) -> None: + data = _parquet(tmp_path) + preds = _preds(tmp_path, "p.jsonl", [{"idx": i, "student": gt} for i, gt in enumerate(GT)]) + rc = main(["score", "--preds", str(preds), "--data", str(data), "--key", "student"]) + assert rc == 0 + out = json.loads(capsys.readouterr().out) + assert out["der_ce"] == 0.0 + assert out["n"] == 2 + + +def test_score_plain_text_lines(tmp_path: Path, capsys) -> None: + data = _parquet(tmp_path) + preds = tmp_path / "plain.txt" + preds.write_text("\n".join(GT) + "\n", encoding="utf-8") + rc = main(["score", "--preds", str(preds), "--data", str(data)]) + out = json.loads(capsys.readouterr().out) + assert rc == 0 and out["der_ce"] == 0.0 + + +def test_bootstrap_vs_reference(tmp_path: Path, capsys) -> None: + data = _parquet(tmp_path) + cand = _preds(tmp_path, "c.jsonl", [{"idx": i, "student": s} for i, s in enumerate(GT)]) + stripped = [g.replace("َ", "").replace("ُ", "").replace("ْ", "").replace("ً", "").replace("ّ", "") for g in GT] + ref = _preds(tmp_path, "r.jsonl", [{"idx": i, "student": s} for i, s in enumerate(stripped)]) + rc = main(["score", "--preds", str(cand), "--data", str(data), "--key", "student", + "--vs", str(ref), "--vs-key", "student"]) + out = json.loads(capsys.readouterr().out) + assert rc == 0 + assert out["vs"]["delta"] < 0 # candidate (perfect) better than stripped reference diff --git a/tests/test_sadeedbench_scoring.py b/tests/test_sadeedbench_scoring.py new file mode 100644 index 0000000..4e90d28 --- /dev/null +++ b/tests/test_sadeedbench_scoring.py @@ -0,0 +1,36 @@ +"""The protocol as a tool: score any predictions file on SadeedDiac-25 +under the campaign's windowed DER-CE convention, with paired +bootstrap against a reference predictions file.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) + +pytest.importorskip("pyarabic") + +from sadeedbench.scoring import score_predictions # noqa: E402 + +GT = ["قَوْلُهُ فَحُكْمُهَا"] * 4 + + +def test_perfect_predictions_score_zero() -> None: + assert score_predictions(GT, GT)["der_ce"] == 0.0 + + +def test_stripped_predictions_score_high() -> None: + # bare text (all haraqat removed) is the collapse constant + out = score_predictions(["قوله فحكمها"] * 4, GT) + assert out["der_ce"] > 50.0 + assert out["n"] == 4 + + +def test_partial_predictions_score_between() -> None: + perfect = score_predictions(GT, GT)["der_ce"] + stripped = score_predictions(["قوله فحكمها"] * 4, GT)["der_ce"] + half = score_predictions([GT[0], "قوله فحكمها", GT[2], "قوله فحكمها"], GT) + assert perfect < half["der_ce"] < stripped