From ec6d5e42b39fd82aa94694b05acc26ceb1367995 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 7 Sep 2026 15:48:22 +0000 Subject: [PATCH 1/2] expand eval corpus to 20 cases with stratified harness stats Add a third wave of bounded extract/rename/split/implement/test/prose cases and report pass@1, pass@end, and first-failure rates by tool and category. Categories label the committed corpus; they are not a task classifier. Co-authored-by: jmjava --- docs/evaluation-protocol.md | 21 +- scripts/run_harness.py | 5 +- src/local_coding_slm/eval/cases.py | 94 ++- src/local_coding_slm/eval/cases_extended.py | 6 + src/local_coding_slm/eval/cases_more.py | 611 ++++++++++++++++++++ src/local_coding_slm/eval/harness.py | 23 +- src/local_coding_slm/eval/score.py | 1 + src/local_coding_slm/eval/stats.py | 47 ++ src/local_coding_slm/eval/stub_ollama.py | 13 +- src/local_coding_slm/eval/taxonomy.py | 35 ++ tests/test_eval_mcp_orchestrate.py | 2 +- tests/test_eval_score.py | 49 +- tests/test_eval_stats.py | 76 +++ 13 files changed, 968 insertions(+), 15 deletions(-) create mode 100644 src/local_coding_slm/eval/cases_more.py create mode 100644 src/local_coding_slm/eval/stats.py create mode 100644 src/local_coding_slm/eval/taxonomy.py create mode 100644 tests/test_eval_stats.py diff --git a/docs/evaluation-protocol.md b/docs/evaluation-protocol.md index 90dc0a2..f8e5cbf 100644 --- a/docs/evaluation-protocol.md +++ b/docs/evaluation-protocol.md @@ -63,6 +63,23 @@ The same scorer runs against: | `implement_clamp` | `local_code` | Implement `clamp` from a spec, no starter file | | `explain_clamp` | `local_explain` | Prose: names the function and bounds; mentions clipping | | `review_login` | `local_review` | Prose first-pass: flags None and missing auth. **Not** an apply | +| `extract_dataclass` | `local_refactor` | Move `Person` + `as_person` into `person_model.py`; `tagged()` oracle | +| `extract_dataclass_vague` | `local_refactor` | **Same checker**, vaguer prompt (`Move the person type…`) | +| `rename_three_files` | `local_refactor` | `fetch` → `load` across `http_client.py` / `service.py` / `main.py` | +| `split_settings` | `local_refactor` | Move retries/timeout constants into `settings.py` | +| `implement_slug` | `local_code` | `slugify('Hello World') == 'hello-world'` | +| `implement_median` | `local_code` | Odd-length `median([1, 3, 2]) == 2` | +| `test_clamp_execute` | `local_generate_tests` | Generated clamp tests are imported and executed | +| `test_median_execute` | `local_generate_tests` | Generated median tests are imported and executed | +| `explain_mean` | `local_explain` | Prose: names `mean` and that it divides the sum | +| `review_divzero` | `local_review` | Prose first-pass: flags a missing zero-denominator check | + +That is **20 cases**. Harness JSON (`scripts/run_harness.py --out`) adds +`by_tool` and `by_category` rates (`pass@1`, `pass@end`, `escalated`, +`first_failure`) so a paper can stratify without dumping transcripts. +Categories (`extract`, `prompt_contract`, `rename`, `split`, `implement`, +`tests`, `explain`, `review`) are labels on the committed corpus, **not** +a task classifier. Known-fail fixtures are part of the corpus. They prove the scorer can tell layers apart: @@ -196,8 +213,10 @@ live script. It now uses the shared fence extractor. Prefer After repeated live runs, a paper may claim: - Layer-conditional rates on this corpus (format vs structure vs behavior). +- Tool- and category-stratified `pass@1` / `pass@end` on the 20-case corpus. - That a vaguer prompt raises structure failures on the same oracle - (`whitespace_extract` vs `whitespace_extract_vague`). + (`whitespace_extract` vs `whitespace_extract_vague`; `extract_dataclass` vs + `extract_dataclass_vague`). - That shape-only test generation overstates success relative to executed tests (`test_add_execute`; A6 now uses this checker). - That keep-vs-delegate and accept/rewrite/reject are enforceable as a diff --git a/scripts/run_harness.py b/scripts/run_harness.py index 132ea1a..cd5c934 100644 --- a/scripts/run_harness.py +++ b/scripts/run_harness.py @@ -20,7 +20,8 @@ run_campaign, run_orchestrated_campaign, ) -from local_coding_slm.eval.record import summarize, write_jsonl # noqa: E402 +from local_coding_slm.eval.record import write_jsonl # noqa: E402 +from local_coding_slm.eval.stats import enrich_summary # noqa: E402 def main() -> None: @@ -93,7 +94,7 @@ def main() -> None: dest.mkdir(parents=True, exist_ok=True) write_jsonl(str(dest / "attempts.jsonl"), rows) (dest / "summary.json").write_text( - json.dumps(summarize(rows), indent=2) + "\n", + json.dumps(enrich_summary(rows), indent=2) + "\n", encoding="utf-8", ) print(f"wrote {dest / 'attempts.jsonl'}") diff --git a/src/local_coding_slm/eval/cases.py b/src/local_coding_slm/eval/cases.py index 6510a82..b291858 100644 --- a/src/local_coding_slm/eval/cases.py +++ b/src/local_coding_slm/eval/cases.py @@ -281,6 +281,7 @@ def _run_generated_tests(merged: dict[str, str]) -> None: style="Keep the code minimal and preserve type hints.", required_top_level=("_normalize_whitespace", "normalize_user"), behavior=WHITESPACE_CHECKS, + category="extract", ), EvalCase( id="whitespace_extract_vague", @@ -290,6 +291,7 @@ def _run_generated_tests(merged: dict[str, str]) -> None: style="Keep the code minimal and preserve type hints.", required_top_level=("_normalize_whitespace", "normalize_user"), behavior=WHITESPACE_CHECKS, + category="prompt_contract", ), EvalCase( id="multi_file_rename", @@ -306,6 +308,7 @@ def _run_generated_tests(merged: dict[str, str]) -> None: BehaviorCheck("use", "total", ([1, 2, 3],), 6), BehaviorCheck("use", "total", ([],), 0), ), + category="rename", ), EvalCase( id="test_add_execute", @@ -315,11 +318,27 @@ def _run_generated_tests(merged: dict[str, str]) -> None: style="pytest", extra_structure=_has_test_functions, behavior_fn=_run_generated_tests, + category="tests", ), ) SEED_CASE_IDS: tuple[str, ...] = tuple(case.id for case in CASES) -CASES = CASES + EXTENDED_CASES + +from local_coding_slm.eval.cases_more import ( # noqa: E402 + EXTRACT_DATACLASS_NESTED, + EXPLAIN_MEAN_VAGUE, + IMPLEMENT_MEDIAN_FIRST, + IMPLEMENT_SLUG_NO_HYPHEN, + MORE_CASES, + MORE_GOLDEN, + REVIEW_DIVZERO_LGTM, + RENAME_THREE_PARTIAL, + SPLIT_SETTINGS_MONOLITH, + TEST_CLAMP_SHAPE_ONLY, + TEST_MEDIAN_SHAPE_ONLY, +) + +CASES = CASES + EXTENDED_CASES + MORE_CASES CASES_BY_ID = {case.id: case for case in CASES} GOLDEN_FOR_CASE = { @@ -328,6 +347,7 @@ def _run_generated_tests(merged: dict[str, str]) -> None: "multi_file_rename": MULTI_FILE_GOLDEN, "test_add_execute": TEST_ADD_GOLDEN, **EXTENDED_GOLDEN, + **MORE_GOLDEN, } @@ -454,4 +474,76 @@ class Fixture: False, "structure", ), + Fixture("extract_dataclass_golden", "extract_dataclass", GOLDEN_FOR_CASE["extract_dataclass"], True), + Fixture( + "extract_dataclass_nested", + "extract_dataclass", + EXTRACT_DATACLASS_NESTED, + False, + "structure", + ), + Fixture("rename_three_golden", "rename_three_files", GOLDEN_FOR_CASE["rename_three_files"], True), + Fixture( + "rename_three_partial", + "rename_three_files", + RENAME_THREE_PARTIAL, + False, + "format", + ), + Fixture("split_settings_golden", "split_settings", GOLDEN_FOR_CASE["split_settings"], True), + Fixture( + "split_settings_monolith", + "split_settings", + SPLIT_SETTINGS_MONOLITH, + False, + "format", + ), + Fixture("implement_slug_golden", "implement_slug", GOLDEN_FOR_CASE["implement_slug"], True), + Fixture( + "implement_slug_no_hyphen", + "implement_slug", + IMPLEMENT_SLUG_NO_HYPHEN, + False, + "behavior", + ), + Fixture("implement_median_golden", "implement_median", GOLDEN_FOR_CASE["implement_median"], True), + Fixture( + "implement_median_first", + "implement_median", + IMPLEMENT_MEDIAN_FIRST, + False, + "behavior", + ), + Fixture("test_clamp_golden", "test_clamp_execute", GOLDEN_FOR_CASE["test_clamp_execute"], True), + Fixture( + "test_clamp_wrong_assert", + "test_clamp_execute", + TEST_CLAMP_SHAPE_ONLY, + False, + "behavior", + ), + Fixture("test_median_golden", "test_median_execute", GOLDEN_FOR_CASE["test_median_execute"], True), + Fixture( + "test_median_wrong_assert", + "test_median_execute", + TEST_MEDIAN_SHAPE_ONLY, + False, + "behavior", + ), + Fixture("explain_mean_golden", "explain_mean", GOLDEN_FOR_CASE["explain_mean"], True), + Fixture( + "explain_mean_vague", + "explain_mean", + EXPLAIN_MEAN_VAGUE, + False, + "structure", + ), + Fixture("review_divzero_golden", "review_divzero", GOLDEN_FOR_CASE["review_divzero"], True), + Fixture( + "review_divzero_lgtm", + "review_divzero", + REVIEW_DIVZERO_LGTM, + False, + "structure", + ), ) diff --git a/src/local_coding_slm/eval/cases_extended.py b/src/local_coding_slm/eval/cases_extended.py index 37390a4..d3decdd 100644 --- a/src/local_coding_slm/eval/cases_extended.py +++ b/src/local_coding_slm/eval/cases_extended.py @@ -460,6 +460,7 @@ def _review_flags_auth(merged: dict[str, str]) -> None: BehaviorCheck("report", "summarize", ([], 0, 5), 0.0), ), max_tokens=1200, + category="extract", ), EvalCase( id="extract_shared_parser", @@ -477,6 +478,7 @@ def _review_flags_auth(merged: dict[str, str]) -> None: BehaviorCheck("csv_orders", "order_qty", (" widget , 3 ",), 3), ), max_tokens=1200, + category="extract", ), EvalCase( id="split_pipeline", @@ -491,6 +493,7 @@ def _review_flags_auth(merged: dict[str, str]) -> None: BehaviorCheck("pipeline", "run", ("",), "0"), ), max_tokens=1200, + category="split", ), EvalCase( id="implement_clamp", @@ -504,6 +507,7 @@ def _review_flags_auth(merged: dict[str, str]) -> None: BehaviorCheck("clamp", "clamp", (-2, 0, 5), 0), BehaviorCheck("clamp", "clamp", (9, 0, 5), 5), ), + category="implement", ), EvalCase( id="explain_clamp", @@ -513,6 +517,7 @@ def _review_flags_auth(merged: dict[str, str]) -> None: expect_fences=False, required_phrases=("clamp", "lo", "hi"), behavior_fn=_explain_mentions_clip, + category="explain", ), EvalCase( id="review_login", @@ -522,6 +527,7 @@ def _review_flags_auth(merged: dict[str, str]) -> None: expect_fences=False, required_phrases=("none",), behavior_fn=_review_flags_auth, + category="review", ), ) diff --git a/src/local_coding_slm/eval/cases_more.py b/src/local_coding_slm/eval/cases_more.py new file mode 100644 index 0000000..47bc098 --- /dev/null +++ b/src/local_coding_slm/eval/cases_more.py @@ -0,0 +1,611 @@ +"""Third-wave corpus: more extract/rename/split/implement/test/prose cases. + +Imported last from cases.py so test helpers exist. Golden strings are +fixtures for the scorer and stub Ollama, not live model dumps. +""" + +from __future__ import annotations + +import ast + +from local_coding_slm.eval.score import BehaviorCheck, EvalCase + +# --- extract Person dataclass (precise + vague, same checker) --- + +PEOPLE_SOURCE = '''\ +from dataclasses import dataclass + + +@dataclass +class Person: + name: str + age: int + + +def as_person(name: str, age: int) -> Person: + return Person(name=name, age=age) + + +def label(person: Person) -> str: + return f"{person.name}:{person.age}" + + +def tagged(name: str, age: int) -> str: + return label(as_person(name, age)) +''' + +EXTRACT_DATACLASS_TASK = ( + "Extract the Person dataclass and as_person helper from people.py " + "into a new file person_model.py. people.py should import them so " + "tagged() still returns name:age. Return fenced files person_model.py " + "and people.py, no prose." +) + +EXTRACT_DATACLASS_TASK_VAGUE = "Move the person type into its own module." + +EXTRACT_DATACLASS_GOLDEN = '''\ +```python +# person_model.py +from dataclasses import dataclass + + +@dataclass +class Person: + name: str + age: int + + +def as_person(name: str, age: int) -> Person: + return Person(name=name, age=age) +``` + +```python +# people.py +from person_model import Person, as_person + + +def label(person: Person) -> str: + return f"{person.name}:{person.age}" + + +def tagged(name: str, age: int) -> str: + return label(as_person(name, age)) +``` +''' + +EXTRACT_DATACLASS_NESTED = '''\ +```python +# person_model.py +def unused() -> None: + return None +``` + +```python +# people.py +from dataclasses import dataclass + + +def wrap() -> None: + @dataclass + class Person: + name: str + age: int + + def as_person(name: str, age: int) -> Person: + return Person(name=name, age=age) + + def label(person: Person) -> str: + return f"{person.name}:{person.age}" + + def tagged(name: str, age: int) -> str: + return label(as_person(name, age)) +``` +''' + + +def _top_level_names(tree: ast.AST) -> set[str]: + return ( + { + node.name + for node in tree.body + if isinstance(node, (ast.FunctionDef, ast.ClassDef)) + } + if isinstance(tree, ast.Module) + else set() + ) + + +def _person_must_move(by_path: dict[str, str]) -> str | None: + try: + model = ast.parse(by_path.get("person_model.py", "")) + people = ast.parse(by_path.get("people.py", "")) + except SyntaxError as exc: + return f"unparseable Python: {exc.msg}" + if "Person" not in _top_level_names(model): + return "person_model.py is missing module-level Person" + if "as_person" not in _top_level_names(model): + return "person_model.py is missing as_person" + if "Person" in _top_level_names(people): + return "Person is still defined in people.py" + people_src = by_path.get("people.py", "") + if "from person_model import" not in people_src: + return "people.py must import from person_model" + return None + + +# --- rename fetch → load across three files --- + +HTTP_SOURCE = "def fetch(url: str) -> str:\n return url.upper()\n" + +SERVICE_SOURCE = ( + "from http_client import fetch\n\n" + "def load_item(url: str) -> str:\n" + " return fetch(url)\n" +) + +MAIN_SOURCE = ( + "from service import load_item\n\n" + "def run(url: str) -> str:\n" + " return load_item(url)\n" +) + +RENAME_THREE_TASK = ( + "Rename fetch to load in http_client.py, service.py, and main.py. " + "Keep load_item and run working. Do not keep fetch as a wrapper. " + "Return three fenced Python files with those path comments, no prose." +) + +RENAME_THREE_GOLDEN = '''\ +```python +# http_client.py +def load(url: str) -> str: + return url.upper() +``` + +```python +# service.py +from http_client import load + + +def load_item(url: str) -> str: + return load(url) +``` + +```python +# main.py +from service import load_item + + +def run(url: str) -> str: + return load_item(url) +``` +''' + +RENAME_THREE_PARTIAL = '''\ +```python +# http_client.py +def load(url: str) -> str: + return url.upper() +``` +''' + + +def _fetch_must_drop(by_path: dict[str, str]) -> str | None: + try: + http = ast.parse(by_path.get("http_client.py", "")) + except SyntaxError as exc: + return f"http_client.py is not parseable: {exc.msg}" + names = _top_level_names(http) + if "fetch" in names: + return "http_client.py still defines fetch; expected a rename to load" + if "load" not in names: + return "http_client.py is missing load" + service = by_path.get("service.py", "") + if "from http_client import load" not in service: + return "service.py must import load from http_client" + if "from http_client import fetch" in service: + return "service.py still imports fetch" + return None + + +# --- split retries/timeout into settings.py --- + +WORKER_SOURCE = '''\ +def fetch_with_policy(url: str) -> str: + retries = 3 + timeout = 30 + return f"{url}:{retries}:{timeout}" +''' + +SPLIT_SETTINGS_TASK = ( + "Move retries and timeout constants from worker.py into settings.py. " + "worker.py should import them. fetch_with_policy must still return " + "url:retries:timeout. Return fenced files settings.py and worker.py." +) + +SPLIT_SETTINGS_GOLDEN = '''\ +```python +# settings.py +retries = 3 +timeout = 30 +``` + +```python +# worker.py +from settings import retries, timeout + + +def fetch_with_policy(url: str) -> str: + return f"{url}:{retries}:{timeout}" +``` +''' + +SPLIT_SETTINGS_MONOLITH = '''\ +```python +# worker.py +def fetch_with_policy(url: str) -> str: + retries = 3 + timeout = 30 + return f"{url}:{retries}:{timeout}" +``` +''' + + +def _settings_split(by_path: dict[str, str]) -> str | None: + missing = [path for path in ("settings.py", "worker.py") if path not in by_path] + if missing: + return "missing " + ", ".join(missing) + worker = by_path["worker.py"] + if "retries = 3" in worker or "timeout = 30" in worker: + return "retries/timeout still assigned in worker.py" + if "from settings import" not in worker: + return "worker.py must import from settings" + return None + + +# --- implement slugify --- + +IMPLEMENT_SLUG_TASK = ( + "Write a Python function slugify(text) in slug.py. Spaces become hyphens " + "and the result is lowercase. slugify('Hello World') must equal " + "'hello-world'. Return one fenced file with a path comment." +) + +IMPLEMENT_SLUG_GOLDEN = '''\ +```python +# slug.py +def slugify(text: str) -> str: + return "-".join(text.lower().split()) +``` +''' + +IMPLEMENT_SLUG_NO_HYPHEN = '''\ +```python +# slug.py +def slugify(text: str) -> str: + return text.lower() +``` +''' + +# --- implement median --- + +IMPLEMENT_MEDIAN_TASK = ( + "Write a Python function median(values) in stats_fn.py for an odd-length " + "list of ints. median([1, 3, 2]) must equal 2. Return one fenced file " + "with a path comment." +) + +IMPLEMENT_MEDIAN_GOLDEN = '''\ +```python +# stats_fn.py +def median(values: list[int]) -> int: + ordered = sorted(values) + return ordered[len(ordered) // 2] +``` +''' + +IMPLEMENT_MEDIAN_FIRST = '''\ +```python +# stats_fn.py +def median(values: list[int]) -> int: + return values[0] +``` +''' + +# --- generated tests that must execute --- + +CLAMP_IMPL = '''\ +def clamp(value: int, lo: int, hi: int) -> int: + return max(lo, min(hi, value)) +''' + +MEDIAN_IMPL = '''\ +def median(values: list[int]) -> int: + ordered = sorted(values) + return ordered[len(ordered) // 2] +''' + +TEST_CLAMP_TASK = ( + "Write pytest tests that execute clamp for in-range and out-of-range " + "values. Return a fenced file named test_clamp.py. Include every import. " + "Do not change clamp.py." +) + +TEST_CLAMP_GOLDEN = '''\ +```python +# test_clamp.py +from clamp import clamp + + +def test_clamp_inside() -> None: + assert clamp(3, 0, 5) == 3 + + +def test_clamp_below() -> None: + assert clamp(-2, 0, 5) == 0 + + +def test_clamp_above() -> None: + assert clamp(9, 0, 5) == 5 +``` +''' + +TEST_CLAMP_SHAPE_ONLY = '''\ +```python +# test_clamp.py +from clamp import clamp + +def test_clamp(): + assert clamp(3, 0, 5) == 99 +``` +''' + +TEST_MEDIAN_TASK = ( + "Write pytest tests that execute median for odd-length integer lists. " + "Return a fenced file named test_stats_fn.py. Include every import. " + "Do not change stats_fn.py." +) + +TEST_MEDIAN_GOLDEN = '''\ +```python +# test_stats_fn.py +from stats_fn import median + + +def test_median_middle() -> None: + assert median([1, 3, 2]) == 2 + + +def test_median_single() -> None: + assert median([7]) == 7 +``` +''' + +TEST_MEDIAN_SHAPE_ONLY = '''\ +```python +# test_stats_fn.py +from stats_fn import median + +def test_median(): + assert median([1, 3, 2]) == 1 +``` +''' + + +def _has_test_functions(by_path: dict[str, str]) -> str | None: + from local_coding_slm.eval.cases import _has_test_functions as shared + + return shared(by_path) + + +def _run_generated_tests(merged: dict[str, str]) -> None: + from local_coding_slm.eval.cases import _run_generated_tests as shared + + shared(merged) + + +# --- explain mean (prose) --- + +MEAN_SOURCE = '''\ +def mean(values: list[int]) -> float: + return sum(values) / len(values) +''' + +EXPLAIN_MEAN_TASK = ( + "Explain what mean does in mean.py in two or three sentences. Name the " + "function and that it divides the sum by the length. Do not rewrite the code." +) + +EXPLAIN_MEAN_GOLDEN = ( + "mean(values) adds the numbers and divides that sum by the length of the " + "list, which is the arithmetic average." +) + +EXPLAIN_MEAN_VAGUE = "This helper is useful in several places." + + +def _explain_mentions_average(merged: dict[str, str]) -> None: + text = merged["_prose"].lower() + if "average" not in text and "sum" not in text and "divid" not in text: + raise AssertionError("explanation never mentions average, sum, or divide") + + +# --- review missing zero check (prose) --- + +DIV_SOURCE = '''\ +def ratio(a: int, b: int) -> float: + return a / b +''' + +REVIEW_DIVZERO_TASK = ( + "First-pass review of ratio() in div.py. Flag the missing zero-denominator " + "check. Do not rewrite the function." +) + +REVIEW_DIVZERO_GOLDEN = ( + "ratio() divides a by b with no zero-denominator check, so b == 0 raises. " + "Guard the divisor before dividing." +) + +REVIEW_DIVZERO_LGTM = "Looks good to me. No issues." + + +def _review_flags_zero(merged: dict[str, str]) -> None: + text = merged["_prose"].lower() + if "zero" not in text and "denominator" not in text and "divis" not in text: + raise AssertionError("review did not flag a zero/denominator/division issue") + + +MORE_CASES: tuple[EvalCase, ...] = ( + EvalCase( + id="extract_dataclass", + tool="local_refactor", + task=EXTRACT_DATACLASS_TASK, + files=({"path": "people.py", "content": PEOPLE_SOURCE},), + required_paths=("person_model.py", "people.py"), + required_top_level=("as_person", "label", "tagged"), + extra_structure=_person_must_move, + behavior=( + BehaviorCheck("people", "tagged", ("Ada", 36), "Ada:36"), + ), + max_tokens=1200, + category="extract", + ), + EvalCase( + id="extract_dataclass_vague", + tool="local_refactor", + task=EXTRACT_DATACLASS_TASK_VAGUE, + files=({"path": "people.py", "content": PEOPLE_SOURCE},), + required_paths=("person_model.py", "people.py"), + required_top_level=("as_person", "label", "tagged"), + extra_structure=_person_must_move, + behavior=( + BehaviorCheck("people", "tagged", ("Ada", 36), "Ada:36"), + ), + max_tokens=1200, + category="prompt_contract", + ), + EvalCase( + id="rename_three_files", + tool="local_refactor", + task=RENAME_THREE_TASK, + files=( + {"path": "http_client.py", "content": HTTP_SOURCE}, + {"path": "service.py", "content": SERVICE_SOURCE}, + {"path": "main.py", "content": MAIN_SOURCE}, + ), + required_paths=("http_client.py", "service.py", "main.py"), + required_top_level=("load", "load_item", "run"), + extra_structure=_fetch_must_drop, + behavior=(BehaviorCheck("main", "run", ("ab",), "AB"),), + max_tokens=1200, + category="rename", + ), + EvalCase( + id="split_settings", + tool="local_refactor", + task=SPLIT_SETTINGS_TASK, + files=({"path": "worker.py", "content": WORKER_SOURCE},), + required_paths=("settings.py", "worker.py"), + required_top_level=("fetch_with_policy",), + extra_structure=_settings_split, + behavior=(BehaviorCheck("worker", "fetch_with_policy", ("x",), "x:3:30"),), + max_tokens=1200, + category="split", + ), + EvalCase( + id="implement_slug", + tool="local_code", + task=IMPLEMENT_SLUG_TASK, + files=(), + required_paths=("slug.py",), + required_top_level=("slugify",), + behavior=( + BehaviorCheck("slug", "slugify", ("Hello World",), "hello-world"), + BehaviorCheck("slug", "slugify", ("Ada",), "ada"), + ), + category="implement", + ), + EvalCase( + id="implement_median", + tool="local_code", + task=IMPLEMENT_MEDIAN_TASK, + files=(), + required_paths=("stats_fn.py",), + required_top_level=("median",), + behavior=( + BehaviorCheck("stats_fn", "median", ([1, 3, 2],), 2), + BehaviorCheck("stats_fn", "median", ([7],), 7), + ), + category="implement", + ), + EvalCase( + id="test_clamp_execute", + tool="local_generate_tests", + task=TEST_CLAMP_TASK, + files=({"path": "clamp.py", "content": CLAMP_IMPL},), + style="pytest", + extra_structure=_has_test_functions, + behavior_fn=_run_generated_tests, + category="tests", + ), + EvalCase( + id="test_median_execute", + tool="local_generate_tests", + task=TEST_MEDIAN_TASK, + files=({"path": "stats_fn.py", "content": MEDIAN_IMPL},), + style="pytest", + extra_structure=_has_test_functions, + behavior_fn=_run_generated_tests, + category="tests", + ), + EvalCase( + id="explain_mean", + tool="local_explain", + task=EXPLAIN_MEAN_TASK, + files=({"path": "mean.py", "content": MEAN_SOURCE},), + expect_fences=False, + required_phrases=("mean", "sum"), + behavior_fn=_explain_mentions_average, + category="explain", + ), + EvalCase( + id="review_divzero", + tool="local_review", + task=REVIEW_DIVZERO_TASK, + files=({"path": "div.py", "content": DIV_SOURCE},), + expect_fences=False, + required_phrases=("zero",), + behavior_fn=_review_flags_zero, + category="review", + ), +) + +MORE_GOLDEN = { + "extract_dataclass": EXTRACT_DATACLASS_GOLDEN, + "extract_dataclass_vague": EXTRACT_DATACLASS_GOLDEN, + "rename_three_files": RENAME_THREE_GOLDEN, + "split_settings": SPLIT_SETTINGS_GOLDEN, + "implement_slug": IMPLEMENT_SLUG_GOLDEN, + "implement_median": IMPLEMENT_MEDIAN_GOLDEN, + "test_clamp_execute": TEST_CLAMP_GOLDEN, + "test_median_execute": TEST_MEDIAN_GOLDEN, + "explain_mean": EXPLAIN_MEAN_GOLDEN, + "review_divzero": REVIEW_DIVZERO_GOLDEN, +} + +MORE_OBSERVED = { + "extract_dataclass": EXTRACT_DATACLASS_NESTED, + "rename_three_files": RENAME_THREE_PARTIAL, + "split_settings": SPLIT_SETTINGS_MONOLITH, + "implement_slug": IMPLEMENT_SLUG_NO_HYPHEN, + "implement_median": IMPLEMENT_MEDIAN_FIRST, + "test_median_execute": TEST_MEDIAN_SHAPE_ONLY, + "explain_mean": EXPLAIN_MEAN_VAGUE, + "review_divzero": REVIEW_DIVZERO_LGTM, +} + +MORE_PERSISTENT_FAST = { + "extract_dataclass_vague": EXTRACT_DATACLASS_NESTED, + "test_clamp_execute": TEST_CLAMP_SHAPE_ONLY, +} diff --git a/src/local_coding_slm/eval/harness.py b/src/local_coding_slm/eval/harness.py index 72c08d7..05a4ed1 100644 --- a/src/local_coding_slm/eval/harness.py +++ b/src/local_coding_slm/eval/harness.py @@ -18,8 +18,9 @@ run_job, ) from local_coding_slm.eval.policy import AttemptPlan, next_plan -from local_coding_slm.eval.record import AttemptRecord, summarize +from local_coding_slm.eval.record import AttemptRecord from local_coding_slm.eval.score import score_candidate +from local_coding_slm.eval.stats import enrich_summary ROOT = Path(__file__).resolve().parents[3] @@ -275,7 +276,7 @@ async def _one_attempt( def format_summary(rows: list[AttemptRecord]) -> str: - stats = summarize(rows) + stats = enrich_summary(rows) lines = [ f"backend={rows[0].backend if rows else '?'} profile={rows[0].profile if rows else '?'}", f"cases={stats['cases']} attempts={stats['attempts']}", @@ -286,6 +287,24 @@ def format_summary(rows: list[AttemptRecord]) -> str: f"mcp_ms p50={stats['mcp_ms_p50']:.1f} max={stats['mcp_ms_max']:.1f}", f"first_failure={stats['first_failure']}", ] + by_tool = stats.get("by_tool") or {} + if by_tool: + lines.append("by_tool:") + for name, item in by_tool.items(): + lines.append( + f" {name}: n={item['cases']} pass@1={item['pass_at_1']:.2f} " + f"pass@end={item['pass_end']:.2f} escalated={item['escalated']:.2f} " + f"first_failure={item['first_failure']}" + ) + by_category = stats.get("by_category") or {} + if by_category: + lines.append("by_category:") + for name, item in by_category.items(): + lines.append( + f" {name}: n={item['cases']} pass@1={item['pass_at_1']:.2f} " + f"pass@end={item['pass_end']:.2f} escalated={item['escalated']:.2f} " + f"first_failure={item['first_failure']}" + ) for item in stats["cases_detail"]: lines.append( f" {item['job']}: pass_at={item['pass_at']} " diff --git a/src/local_coding_slm/eval/score.py b/src/local_coding_slm/eval/score.py index ea064f7..cad1e6b 100644 --- a/src/local_coding_slm/eval/score.py +++ b/src/local_coding_slm/eval/score.py @@ -84,6 +84,7 @@ class EvalCase: expect_fences: bool = True required_phrases: tuple[str, ...] = () max_tokens: int = 700 + category: str = "other" def score_candidate(text: str, case: EvalCase) -> EvalResult: diff --git a/src/local_coding_slm/eval/stats.py b/src/local_coding_slm/eval/stats.py new file mode 100644 index 0000000..d71e6f1 --- /dev/null +++ b/src/local_coding_slm/eval/stats.py @@ -0,0 +1,47 @@ +"""Stratified summary for harness JSON. Looks up tool/category from the corpus.""" + +from __future__ import annotations + +from collections import defaultdict + +from local_coding_slm.eval.record import AttemptRecord, summarize +from local_coding_slm.eval.taxonomy import category_of + + +def enrich_summary(rows: list[AttemptRecord]) -> dict[str, object]: + """Add by_tool and by_category rates. Same pass@1 / pass@end rules as summarize.""" + stats = summarize(rows) + by_job: dict[str, list[AttemptRecord]] = {} + for row in rows: + by_job.setdefault(row.job, []).append(row) + + def _group(key_fn) -> dict[str, dict[str, object]]: + buckets: dict[str, list[str]] = defaultdict(list) + for job, group in by_job.items(): + buckets[key_fn(group[0])].append(job) + out: dict[str, dict[str, object]] = {} + for key, jobs in sorted(buckets.items()): + groups = [by_job[job] for job in jobs] + n = len(groups) + pass_at_1 = sum(1 for g in groups if any(r.passed and r.attempt == 1 for r in g)) + pass_end = sum(1 for g in groups if any(r.passed for r in g)) + escalated = sum(1 for g in groups if any(r.model == "strong" for r in g)) + fails: dict[str, int] = {} + for group in groups: + for row in group: + if row.first_failure: + fails[row.first_failure] = fails.get(row.first_failure, 0) + 1 + out[key] = { + "cases": n, + "pass_at_1": pass_at_1 / n if n else 0.0, + "pass_end": pass_end / n if n else 0.0, + "escalated": escalated / n if n else 0.0, + "first_failure": fails, + } + return out + + from local_coding_slm.eval.cases import CASES_BY_ID + + stats["by_tool"] = _group(lambda row: CASES_BY_ID[row.case_id].tool) + stats["by_category"] = _group(lambda row: category_of(row.case_id)) + return stats diff --git a/src/local_coding_slm/eval/stub_ollama.py b/src/local_coding_slm/eval/stub_ollama.py index ed2424c..77cd97d 100644 --- a/src/local_coding_slm/eval/stub_ollama.py +++ b/src/local_coding_slm/eval/stub_ollama.py @@ -18,6 +18,7 @@ WHITESPACE_NO_FENCE, ) from local_coding_slm.eval.cases_extended import OBSERVED_FIRST +from local_coding_slm.eval.cases_more import MORE_OBSERVED, MORE_PERSISTENT_FAST from local_coding_slm.ollama_client import DEFAULT_FAST_MODEL, DEFAULT_STRONG_MODEL @@ -36,14 +37,18 @@ def scripted_content(case_id: str, model_choice: str, visit: int, profile: str) if profile != "observed": raise ValueError(f"unknown stub profile {profile!r}") # These stay wrong for every fast call so the policy must escalate to strong. - if model_choice == "fast" and case_id == "whitespace_extract_vague": - return WHITESPACE_NESTED - if model_choice == "fast" and case_id == "test_add_execute": - return TEST_ADD_SHAPE_ONLY + persistent_fast = { + "whitespace_extract_vague": WHITESPACE_NESTED, + "test_add_execute": TEST_ADD_SHAPE_ONLY, + **MORE_PERSISTENT_FAST, + } + if model_choice == "fast" and case_id in persistent_fast: + return persistent_fast[case_id] first_fail = { "whitespace_extract": WHITESPACE_NO_FENCE, "multi_file_rename": MULTI_FILE_PARTIAL, **OBSERVED_FIRST, + **MORE_OBSERVED, } if model_choice == "fast" and visit == 1 and case_id in first_fail: return first_fail[case_id] diff --git a/src/local_coding_slm/eval/taxonomy.py b/src/local_coding_slm/eval/taxonomy.py new file mode 100644 index 0000000..1bf4955 --- /dev/null +++ b/src/local_coding_slm/eval/taxonomy.py @@ -0,0 +1,35 @@ +"""Case categories for stratified harness stats. Not a task classifier.""" + +from __future__ import annotations + +CATEGORIES: dict[str, str] = { + "whitespace_extract": "extract", + "whitespace_extract_vague": "prompt_contract", + "multi_file_rename": "rename", + "test_add_execute": "tests", + "move_function_imports": "extract", + "extract_shared_parser": "extract", + "split_pipeline": "split", + "implement_clamp": "implement", + "explain_clamp": "explain", + "review_login": "review", + "extract_dataclass": "extract", + "extract_dataclass_vague": "prompt_contract", + "rename_three_files": "rename", + "split_settings": "split", + "implement_slug": "implement", + "implement_median": "implement", + "test_clamp_execute": "tests", + "test_median_execute": "tests", + "explain_mean": "explain", + "review_divzero": "review", +} + + +def category_of(case_id: str) -> str: + from local_coding_slm.eval.cases import CASES_BY_ID + + case = CASES_BY_ID.get(case_id) + if case is not None: + return case.category + return CATEGORIES.get(case_id, "other") diff --git a/tests/test_eval_mcp_orchestrate.py b/tests/test_eval_mcp_orchestrate.py index c95c1a1..2441b87 100644 --- a/tests/test_eval_mcp_orchestrate.py +++ b/tests/test_eval_mcp_orchestrate.py @@ -18,7 +18,7 @@ async def test_golden_corpus_pass_at_one(self) -> None: strong_ms=1, ) stats = summarize(rows) - self.assertGreaterEqual(stats["cases"], 10) + self.assertGreaterEqual(stats["cases"], 20) self.assertEqual(stats["pass_at_1"], 1.0) self.assertEqual(stats["pass_end"], 1.0) diff --git a/tests/test_eval_score.py b/tests/test_eval_score.py index 94388a6..5733521 100644 --- a/tests/test_eval_score.py +++ b/tests/test_eval_score.py @@ -4,14 +4,15 @@ import unittest -from local_coding_slm.eval.cases import CASES_BY_ID, FIXTURES, SEED_CASE_IDS +from local_coding_slm.eval.cases import CASES, CASES_BY_ID, FIXTURES, SEED_CASE_IDS from local_coding_slm.eval.score import score_candidate class FixtureCorpusTests(unittest.TestCase): def test_every_fixture_matches_expected_layer(self) -> None: - self.assertGreaterEqual(len(FIXTURES), 16) + self.assertGreaterEqual(len(FIXTURES), 40) self.assertGreaterEqual(len(SEED_CASE_IDS), 4) + self.assertGreaterEqual(len(CASES), 20) for fixture in FIXTURES: with self.subTest(fixture=fixture.name): case = CASES_BY_ID[fixture.case_id] @@ -51,8 +52,6 @@ def test_vague_and_precise_share_the_same_checker(self) -> None: self.assertNotEqual(precise.task, vague.task) def test_corpus_covers_all_generation_tools(self) -> None: - from local_coding_slm.eval.cases import CASES - tools = {case.tool for case in CASES} self.assertTrue( { @@ -88,6 +87,48 @@ def test_move_partial_is_format_not_behavior(self) -> None: self.assertEqual(result.layer("format").status, "fail") self.assertEqual(result.layer("behavior").status, "skip") + def test_third_wave_vague_shares_checker(self) -> None: + precise = CASES_BY_ID["extract_dataclass"] + vague = CASES_BY_ID["extract_dataclass_vague"] + self.assertEqual(precise.required_top_level, vague.required_top_level) + self.assertEqual(precise.behavior, vague.behavior) + self.assertEqual(precise.extra_structure, vague.extra_structure) + self.assertNotEqual(precise.task, vague.task) + + def test_categories_cover_the_corpus(self) -> None: + from local_coding_slm.eval.taxonomy import CATEGORIES, category_of + + ids = [case.id for case in CASES] + self.assertEqual(len(ids), len(set(ids))) + self.assertEqual(set(ids), set(CATEGORIES)) + for case in CASES: + self.assertEqual(case.category, CATEGORIES[case.id], case.id) + self.assertEqual(category_of(case.id), case.category) + self.assertNotEqual(case.category, "other") + + def test_tasks_are_unique_for_stub_matching(self) -> None: + from local_coding_slm.eval.stub_ollama import infer_case_id + + tasks = [case.task for case in CASES] + self.assertEqual(len(tasks), len(set(tasks))) + for case in CASES: + self.assertEqual(infer_case_id(case.task), case.id, case.id) + + def test_slug_no_hyphen_is_behavior(self) -> None: + case = CASES_BY_ID["implement_slug"] + bad = next(item for item in FIXTURES if item.name == "implement_slug_no_hyphen") + result = score_candidate(bad.text, case) + self.assertEqual(result.layer("structure").status, "pass") + self.assertEqual(result.layer("behavior").status, "fail") + + def test_nested_person_is_structure(self) -> None: + case = CASES_BY_ID["extract_dataclass"] + nested = next(item for item in FIXTURES if item.name == "extract_dataclass_nested") + result = score_candidate(nested.text, case) + self.assertEqual(result.layer("format").status, "pass") + self.assertEqual(result.layer("structure").status, "fail") + self.assertEqual(result.layer("behavior").status, "skip") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_eval_stats.py b/tests/test_eval_stats.py new file mode 100644 index 0000000..12363b3 --- /dev/null +++ b/tests/test_eval_stats.py @@ -0,0 +1,76 @@ +"""Stratified pass@1 / pass@end by tool and category.""" + +from __future__ import annotations + +import unittest + +from local_coding_slm.eval.record import AttemptRecord +from local_coding_slm.eval.stats import enrich_summary + + +def _rec(**kwargs: object) -> AttemptRecord: + base: dict[str, object] = { + "job": "whitespace_extract#1", + "case_id": "whitespace_extract", + "attempt": 1, + "model": "fast", + "suffix": "", + "passed": True, + "first_failure": None, + "mcp_ms": 1.0, + "score_ms": 0.2, + "elapsed_ms": 1.2, + "response_chars": 10, + "backend": "stub", + "profile": "golden", + "layers": {"transport": "pass", "format": "pass", "structure": "pass", "behavior": "pass"}, + } + base.update(kwargs) + return AttemptRecord(**base) # type: ignore[arg-type] + + +class EnrichSummaryTests(unittest.TestCase): + def test_by_tool_and_category(self) -> None: + rows = [ + _rec(), + _rec( + job="implement_clamp#1", + case_id="implement_clamp", + passed=False, + first_failure="behavior", + layers={"transport": "pass", "format": "pass", "structure": "pass", "behavior": "fail"}, + ), + _rec( + job="implement_clamp#1", + case_id="implement_clamp", + attempt=2, + model="strong", + passed=True, + first_failure=None, + ), + _rec( + job="explain_clamp#1", + case_id="explain_clamp", + passed=True, + ), + ] + stats = enrich_summary(rows) + self.assertEqual(stats["cases"], 3) + self.assertAlmostEqual(stats["pass_at_1"], 2 / 3) + self.assertEqual(stats["pass_end"], 1.0) + by_tool = stats["by_tool"] + self.assertEqual(by_tool["local_refactor"]["cases"], 1) + self.assertEqual(by_tool["local_refactor"]["pass_at_1"], 1.0) + self.assertEqual(by_tool["local_code"]["cases"], 1) + self.assertEqual(by_tool["local_code"]["pass_at_1"], 0.0) + self.assertEqual(by_tool["local_code"]["pass_end"], 1.0) + self.assertEqual(by_tool["local_code"]["escalated"], 1.0) + by_category = stats["by_category"] + self.assertEqual(by_category["extract"]["pass_at_1"], 1.0) + self.assertEqual(by_category["implement"]["pass_at_1"], 0.0) + self.assertEqual(by_category["explain"]["pass_at_1"], 1.0) + self.assertIn("behavior", by_tool["local_code"]["first_failure"]) + + +if __name__ == "__main__": + unittest.main() From 5a065921b18e1e40289414ef4a224429dfc97506 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 7 Sep 2026 15:49:40 +0000 Subject: [PATCH 2/2] record 20-case stub harness rates by tool and category Dated Cloud Agent note for the expanded corpus. Stub observed campaign is pass@1 0.00 / pass@end 1.00 with layer mix; not live GPU timings. Co-authored-by: jmjava --- README.md | 2 + docs/cloud-corpus-stats-2026-09-07.md | 67 +++++++++++++++++++++++++++ docs/phase3-log.md | 3 ++ docs/roadmap.md | 2 +- 4 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 docs/cloud-corpus-stats-2026-09-07.md diff --git a/README.md b/README.md index 27d301e..b05651d 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ does bounded, mechanical generation on a private GPU host running the workstation. - **[docs/cloud-orchestrator-results-2026-09-07.md](docs/cloud-orchestrator-results-2026-09-07.md)** — dated Cloud Agent run of the stub corpus and apply gate (no GPU). +- **[docs/cloud-corpus-stats-2026-09-07.md](docs/cloud-corpus-stats-2026-09-07.md)** + — dated 20-case stub harness with `by_tool` / `by_category` rates (no GPU). - **[docs/security-scan-results-2026-09-06.md](docs/security-scan-results-2026-09-06.md)** — dated Gitleaks, GitHub alert, tracked-tree, and deployment-safety results. - **[examples/](examples/)** — public-safe client config templates. Copy them diff --git a/docs/cloud-corpus-stats-2026-09-07.md b/docs/cloud-corpus-stats-2026-09-07.md new file mode 100644 index 0000000..8c22bd7 --- /dev/null +++ b/docs/cloud-corpus-stats-2026-09-07.md @@ -0,0 +1,67 @@ +# Cloud Agent 20-case corpus stats — 2026-09-07 + +Dated note from a Cursor Cloud Agent. This VM has **no GPU and no Ollama**. +The worker is the loopback stub. These rates measure the scorer, stub +failover policy, and stdio MCP loop. They are **not** live +`qwen3.5:9b` / `devstral-small-2` quality. + +Commands (all passed): + +```bash +PYTHONPATH=src .venv/bin/python -m unittest discover -s tests -v +.venv/bin/python scripts/run_eval.py +.venv/bin/python scripts/run_orchestration.py +.venv/bin/python scripts/run_harness.py --backend stub --profile golden --fast-ms 1 --strong-ms 1 +.venv/bin/python scripts/run_harness.py --backend stub --profile observed --fast-ms 1 --strong-ms 1 --out eval-runs/observed-20 +.venv/bin/python scripts/run_harness.py --backend stub --profile golden --orchestrate --fast-ms 1 --strong-ms 1 +``` + +## Unit suite + +**109 tests passed.** Fixture scoring now covers 20 cases / 41 fixtures. +`by_tool` and `by_category` rates are tested without MCP. + +## Stub MCP harness (not GPU time) + +Golden, 20 jobs, 20 attempts: **pass@1 = 1.00**, escalated = 0.00. + +Observed, 20 jobs, 42 attempts: **pass@1 = 0.00**, **pass@end = 1.00**, +escalated = 0.40. First failures: format 6, structure 10, behavior 6. + +| Tool | n | pass@1 | pass@end | escalated | first_failure | +| --- | ---: | ---: | ---: | ---: | --- | +| `local_refactor` | 10 | 0.00 | 1.00 | 0.20 | format 6, structure 6 | +| `local_code` | 3 | 0.00 | 1.00 | 1.00 | behavior 3 | +| `local_generate_tests` | 3 | 0.00 | 1.00 | 1.00 | behavior 3 | +| `local_explain` | 2 | 0.00 | 1.00 | 0.00 | structure 2 | +| `local_review` | 2 | 0.00 | 1.00 | 0.00 | structure 2 | + +| Category | n | pass@1 | pass@end | escalated | first_failure | +| --- | ---: | ---: | ---: | ---: | --- | +| extract | 4 | 0.00 | 1.00 | 0.00 | format 2, structure 2 | +| prompt_contract | 2 | 0.00 | 1.00 | 1.00 | structure 4 | +| rename | 2 | 0.00 | 1.00 | 0.00 | format 2 | +| split | 2 | 0.00 | 1.00 | 0.00 | format 2 | +| implement | 3 | 0.00 | 1.00 | 1.00 | behavior 3 | +| tests | 3 | 0.00 | 1.00 | 1.00 | behavior 3 | +| explain | 2 | 0.00 | 1.00 | 0.00 | structure 2 | +| review | 2 | 0.00 | 1.00 | 0.00 | structure 2 | + +Vague-prompt cases (`whitespace_extract_vague`, `extract_dataclass_vague`) +stay wrong on every fast call, so they escalate to strong (pass@3). +Implement and test cases fail behavior on the first fast reply and +escalate. Format/structure-only cases repair on fast (pass@2). + +Categories label the committed corpus. They are **not** a learned +task classifier. + +## Apply gate + +Unchanged 13-job scripted loop: 9 delegated, 11 applied, 2 held. +Keep jobs did not call `local_*`. + +## What this note is not + +- Live tok/s or pass@1 on the workstation tags +- Proof that desktop Cursor's tool picker chose `local_refactor` +- A claim that stub `mcp_ms` is model latency diff --git a/docs/phase3-log.md b/docs/phase3-log.md index d088f92..4ffb55c 100644 --- a/docs/phase3-log.md +++ b/docs/phase3-log.md @@ -8,6 +8,8 @@ Reproducible commands, complete 2026-09-06 observations, and limitations: Scoring method: [evaluation-protocol.md](evaluation-protocol.md). Cloud-safe stub + apply-gate run: [cloud-orchestrator-results-2026-09-07.md](cloud-orchestrator-results-2026-09-07.md). +20-case corpus + stratified stub rates: +[cloud-corpus-stats-2026-09-07.md](cloud-corpus-stats-2026-09-07.md). | Date | Tool | Model | Task | Result | Notes | | --- | --- | --- | --- | --- | --- | @@ -16,6 +18,7 @@ Cloud-safe stub + apply-gate run: | 2026-09-06 | local_refactor | fast | extract module-level whitespace helper | accepted | Real Ollama; generated module parsed and preserved behavior across 3 executed cases; warm run 6.7s | | 2026-09-06 | local_refactor | strong | extract module-level whitespace helper | accepted | Real Ollama; generated module parsed and preserved behavior across 3 executed cases; warm run 25.6s; model reported 46% CPU / 54% GPU at 16K context | | 2026-09-07 | all local_* eval tools | stub | 10-case corpus + apply gate | accepted / rewritten / rejected per job | Cloud Agent; no GPU; A6 now executes tests | +| 2026-09-07 | all local_* eval tools | stub | 20-case corpus; by_tool / by_category rates | pass@end 1.00 (observed); pass@1 0.00 | Cloud Agent; no GPU; 42 attempts; escalated 0.40 | Columns: diff --git a/docs/roadmap.md b/docs/roadmap.md index f9cd403..1ad0939 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -10,7 +10,7 @@ LAN addresses, or SKUs here. | --- | --- | --- | | 1 — inference host | Done on the workstation lab | Ollama + starter tags on a private GPU | | 2 — MCP bridge | Done | `local-coding-slm` stdio tools; Cursor / Copilot / Claude adapters | -| 3 — measure | Protocol + expanded corpus + MCP apply gate + CI; live rates still informal | Layered scoring; scripted premium routing/review; no auto-classifier yet | +| 3 — measure | Protocol + 20-case corpus + stratified stub stats + apply gate + CI; live rates still informal | Layered scoring; scripted premium routing/review; no auto-classifier yet | | T12 second NVIDIA host | Blocked on host power | WSL GPU via SSH; see [examples/downstairs-wsl-gpu.md](../examples/downstairs-wsl-gpu.md) | Finish Phase 3 measurement (layered live rates on the committed corpus)