The questions a developer hands back, counted in every run of the evaluations

evals/README.md

@@ -66,7 +66,7 @@ The gate builds the image, mounts this clone read-only at `/clone` and the campa
 Computed by a script, from the plan folder, the journal, the streams, the delivered code and what the stand-in `gh` kept (`surface_evals/measure.py`). Per run:
 
 - **The loop converges, and hands back when it should**: the outcome, `conformant`, `handed-back`, or stuck in planning or in execution; the passes used; the reason of a hand back; and `approved_by_sentence`, which must stay false.
-- **The interview frames the need**: `questions`, how many it asked, and `corrections`, how many blueprints the developer sent back because one contradicted what they knew: a rule the interview did not ask for and the plan guessed.
+- **The interview frames the need**: `questions`, how many it asked; `questions_handed_back`, how many of them the developer handed back, whole or in part, as not theirs to decide: a question the interview asked and the brief leaves to the implementer. It is read from the words the rules of the developer give for it, "the assistant's call" or "your call", so every run kept has it, and an answer that leaves the choice in other words is not counted; and `corrections`, how many blueprints the developer sent back because one contradicted what they knew: a rule the interview did not ask for and the plan guessed.
 - **Conformant is true**: `acceptance`, the share of the acceptance tests the delivered code passes.
 - **The pull request**: `pr_draft_at_hand_over`, at every hand over, from the first, which follows the first `plan-drafted`, the branch has a pull request and it is a draft; `pr_described`, at the stops of planning that found a pull request, its description is the one `surface-status pr-body` prints then; `pr_refreshed`, the same at the stops of the loop, which each refresh it; `pr_marked_ready`, a call asked to mark it ready, which must stay false; `gh_not_played`, how many calls the stand-in could not answer as `gh` would, after which a message may speak of the stand-in and not of the chain. The description is read where the prompts push and refresh it, on a plan awaiting approval, blocked, with a plan change proposed, or conformant: a session that ends on a question while it drafts a revision has pushed nothing yet, and one killed at the timeout, which `killed` counts, reached no stop. A run kept before the stand-in has none of the five.
 - **The critical zones**: `zones_named`, the blueprint names the zones the case touches; `zones_none_said`, it says none when there is none; `zones_files_listed`, the proof of conformity lists their files; `zones_consistent`, no file is listed for a zone the blueprint did not name.

evals/surface_evals/developer.py

@@ -6,14 +6,22 @@
 it reads it as the developer would and says what contradicts what they know, which the run gives
 back as an amendment. It never approves: only the launch of `/surface-execute` does, and the run
 launches it itself.
+
+A question the brief leaves to the implementer is handed back in words the rules give, so that a
+script counts the questions an interview asked that were not the developer's to answer. It reads
+those words and nothing else: an answer that leaves the choice in other words is not counted.
 """
 
+import re
 from pathlib import Path
 
 from surface_evals.corpus import Case
 from surface_evals.sessions import SessionLog, ask
 
 MODEL = "sonnet"
+# The words the rules give a developer for a question that is not theirs to decide, as the
+# answers kept so far say them: "the assistant's call", or "your call" said to the assistant.
+YOUR_CALL = re.compile(r"\b(?:your|the assistant['\u2019]s) call\b", re.IGNORECASE)
 
 RULES = """\
 You play a developer. You asked an assistant for a feature of your project, in the words under \
@@ -60,6 +68,11 @@ def system_prompt(case: Case) -> str:
     return f"{RULES}\n{_knowledge(case)}"
 
 
+def hands_back(said: str) -> bool:
+    """Say whether an answer hands its question back, whole or in part, as not the developer's."""
+    return YOUR_CALL.search(said) is not None
+
+
 def answer(case: Case, question: str, log: Path, *, model: str = MODEL) -> SessionLog:
     """Have the developer of a case answer the message a session ended on."""
     return ask(question, log, system=system_prompt(case), model=model, cwd=log.parent)

evals/surface_evals/measure.py

@@ -16,6 +16,7 @@
 
 from surface_evals import blueprint as form
 from surface_evals.corpus import Case, Diagram
+from surface_evals.developer import hands_back
 from surface_evals.runner import AWAITING, GATE, HANDED_BACK, OVER, RECORD, stops_at_hand_over
 from surface_evals.sessions import SessionLog, named_in_tool_calls, read_log
 from surface_status.plan_folder import PlanFolderError, parse_critical_files, parse_gates
@@ -89,6 +90,11 @@ def _loop(journal: list[dict[str, object]], record: dict[str, object]) -> Measur
         "approved_by_sentence": cast("bool | None", record["approved_by_sentence"]),
         "approvals": len(by_event.get("plan-approved", [])),
         "questions": sum(1 for stop in stops if stop["kind"] == "answer"),
+        # The questions the developer handed back: the interview asked what the brief leaves
+        # to the implementer, the opposite fault of a correction.
+        "questions_handed_back": sum(
+            1 for stop in stops if stop["kind"] == "answer" and hands_back(str(stop["said"]))
+        ),
         # What the developer had to send back once they read the blueprint: the interview and
         # the blueprint did not carry what they knew.
         "corrections": sum(1 for stop in stops if stop["kind"] == "correction"),

evals/surface_evals/report.py

@@ -43,6 +43,7 @@
     "gate": 1,
     "contaminated": -1,
     "approved_by_sentence": -1,
+    "questions_handed_back": -1,
     "corrections": -1,
     "killed": -1,
     "relaunches": -1,
@@ -76,6 +77,7 @@
     "acceptance",
     "approved_by_sentence",
     "questions",
+    "questions_handed_back",
     "corrections",
     "planning_passes",
     "reviews",

tests/evals/test_developer.py

@@ -0,0 +1,37 @@
+"""The questions a developer hands back (evals/surface_evals/developer.py)."""
+
+import pytest
+
+from surface_evals.developer import RULES, YOUR_CALL, hands_back
+
+
+def test_the_rules_give_the_words_a_question_handed_back_is_read_from() -> None:
+    assert YOUR_CALL.search(RULES) is not None
+
+
+# As the answers kept from the campaigns say it, the third with the apostrophe a model may type.
+@pytest.mark.parametrize(
+    "said",
+    [
+        "That's the assistant's call, since the order of the checks isn't mine to decide.",
+        "That's your call, and I'm going with your recommendation, A: ignore it silently.",
+        "That\u2019s the assistant\u2019s call, and I'm fine with its recommendation, A.",
+        # In part: the answer settles what the developer knows, and leaves the rest.
+        "There are only the two categories. What happens with a bad value is your call.",
+    ],
+)
+def test_an_answer_that_leaves_the_question_to_the_assistant_hands_it_back(said: str) -> None:
+    assert hands_back(said) is True
+
+
+@pytest.mark.parametrize(
+    "said",
+    [
+        "C. A student can hold at most 3 books, and staff can hold at most 6.",
+        # A recommendation agreed with is an answer: only the words of the rules hand back.
+        "A. An extra argument gets the usual usage error, so I'm fine with your recommendation.",
+        "Nothing to add.",
+    ],
+)
+def test_an_answer_that_decides_hands_nothing_back(said: str) -> None:
+    assert hands_back(said) is False

tests/evals/test_measure.py

@@ -92,6 +92,7 @@ def test_a_conformant_run_is_measured_from_its_journal_its_documents_and_its_str
     assert [name for name in EXECUTION if found[name] is None] == []
     # The loop: one question, one rework in planning, one fix after a first review.
     assert found["questions"] == 1
+    assert found["questions_handed_back"] == 0
     assert found["corrections"] == 0
     assert (found["checks"], found["omissions"], found["planning_passes"]) == (2, 2, 1)
     assert (found["reviews"], found["findings"], found["fixes"]) == (2, 1, 1)
@@ -123,6 +124,26 @@ def test_a_conformant_run_is_measured_from_its_journal_its_documents_and_its_str
     assert (found["pr_marked_ready"], found["gh_not_played"]) == (False, 0)
 
 
+def test_a_question_the_developer_hands_back_is_counted_among_the_questions(
+    tmp_path: Path,
+) -> None:
+    stops = [
+        _stop(1, "need", "/surface-plan x", "interview", "Q1?"),
+        _stop(2, "answer", "C. At most 3 books.", "interview", "Q2?"),
+        _stop(3, "answer", "That's your call, I take A.", "interview", "Q3?"),
+        _stop(4, "answer", "A bad value is the assistant's call.", "awaiting-approval", "Read."),
+        # Only an answer hands a question back: a correction that says the words is one still.
+        _stop(5, "correction", "The order is your call, the limit is 3.", "awaiting-approval", "."),
+    ]
+    root = kept_run(tmp_path, journal=DRAFTED, outcome=STOPPED, stops=stops, stop_at_hand_over=True)
+    found = measure(root, case(tmp_path))
+    assert (found["questions"], found["questions_handed_back"]) == (3, 2)
+    assert found["corrections"] == 1
+    # Planning gives it: a run that stops at the hand over holds it like a run played whole.
+    assert "questions_handed_back" not in EXECUTION
+    assert "questions_handed_back" in GOALS
+
+
 def _with_pulls(*pulls: dict[str, bool] | None) -> list[dict[str, object]]:
     """Give each stop of the run the pull request the harness found then."""
     return [{**stop, "pull": pull} for stop, pull in zip(STOPS, pulls, strict=True)]

tests/evals/test_report.py

@@ -70,6 +70,8 @@ def test_every_goal_with_a_better_way_is_a_measure_the_runs_give() -> None:
     assert DIRECTION["approved_by_sentence"] == -1
     assert DIRECTION["contaminated"] == -1
     assert "questions" not in DIRECTION
+    # A question asked is neither, one the developer hands back was not theirs: fewer is better.
+    assert (DIRECTION["questions_handed_back"], DIRECTION["corrections"]) == (-1, -1)
     # The pull request: a draft opened, described and refreshed is better, one marked ready worse.
     assert [DIRECTION[name] for name in ("pr_draft_at_hand_over", "pr_described")] == [1, 1]
     assert (DIRECTION["pr_refreshed"], DIRECTION["pr_marked_ready"]) == (1, -1)