Re-cut the prose-register eval keys the subject disagrees with every run

evals/asserts/detection.mjs

@@ -1,28 +1,88 @@
 import { normalize } from './heuristics.mjs';
 
+// A subject quote shorter than this would sit inside several key quotes at
+// once ("just", "the code"), so it never counts on its own.
+const MIN_OVERLAP = 20;
+
+// Both strings are verbatim runs of the same document, so they refer to the
+// same passage when one contains the other or the end of one is the start of
+// the other. Returns the length of the shared run (0 when none counts) and
+// whether the subject quote holds the whole key. A prefix-only match called
+// det-03 a miss when the subject began its quote one word into the key and
+// ran past the key's end.
+export function overlap(keyQuote, subjectQuote) {
+  const key = normalize(keyQuote);
+  const quote = normalize(subjectQuote);
+  const min = Math.min(MIN_OVERLAP, key.length);
+  if (quote.length < min) return { length: 0, whole: false };
+  if (quote.includes(key)) return { length: key.length, whole: true };
+  if (key.includes(quote)) return { length: quote.length, whole: false };
+  for (let len = Math.min(key.length, quote.length); len >= min; len--) {
+    if (key.endsWith(quote.slice(0, len)) || quote.endsWith(key.slice(0, len))) {
+      return { length: len, whole: false };
+    }
+  }
+  return { length: 0, whole: false };
+}
+
+export const quotesOverlap = (keyQuote, subjectQuote) => overlap(keyQuote, subjectQuote).length > 0;
+
+// A violation one character wide (det-03's em-dash) can be quoted from either
+// side, and a quote that stops at the dash shares too little with the key for
+// overlap to count. A target may name an anchor inside its quote that settles
+// the match on its own.
+function matches(target, quote) {
+  if (target.anchor && quote.includes(normalize(target.anchor))) {
+    return { length: normalize(target.quote).length, whole: true };
+  }
+  return overlap(target.quote, quote);
+}
+
+// One subject quote is one finding. It is credited to every target it holds
+// whole and, when it holds none, to the single target it overlaps most; a
+// quote that runs from one violation into the opening of the next is not a
+// find of both.
+function credited(targets, quote) {
+  const scored = targets.map((target) => ({ target, ...matches(target, quote) }));
+  const whole = scored.filter((s) => s.whole).map((s) => s.target);
+  if (whole.length) return whole;
+  const best = scored.filter((s) => s.length > 0).sort((a, b) => b.length - a.length)[0];
+  return best ? [best.target] : [];
+}
+
+function subjectQuotes(output) {
+  const quotes = [];
+  for (const line of output.split('\n')) {
+    const match = line.match(/^-\s*QUOTE:\s*(.*)$/i);
+    if (!match) continue;
+    const [text] = match[1].split(/\|\s*RULE:/i);
+    const quote = normalize(text).replace(/^"|"$/g, '').trim();
+    if (quote) quotes.push(quote);
+  }
+  return quotes;
+}
+
 export default function assertDetection(output, context) {
-  const { violations, traps } = context.vars;
-  const violationLines = output
-    .split('\n')
-    .filter((line) => /^-\s*QUOTE:/i.test(line))
-    .join('\n');
-  const normalized = normalize(violationLines);
-
-  const missed = violations.filter(
-    (violation) => !normalized.includes(normalize(violation.quote).slice(0, 60)),
-  );
-  const trapHits = traps.filter(
-    (trap) => normalized.includes(normalize(trap.quote).slice(0, 40)),
-  );
+  const { violations, traps, min_recall: minRecall = 1 } = context.vars;
+  const hit = new Set();
+  for (const quote of subjectQuotes(output)) {
+    for (const target of credited([...violations, ...traps], quote)) hit.add(target);
+  }
+
+  const missed = violations.filter((violation) => !hit.has(violation));
+  const trapHits = traps.filter((trap) => hit.has(trap));
 
   const found = violations.length - missed.length;
-  const pass = missed.length === 0 && trapHits.length === 0;
+  const recall = found / violations.length;
+  // A case whose violation list is not exhaustive sets min_recall below 1 and
+  // records recall as its score; a flagged trap fails at any floor.
+  const pass = recall >= minRecall && trapHits.length === 0;
   const score = Math.max(0, (found - trapHits.length) / violations.length);
-  const reason = pass
-    ? `${found}/${violations.length} violations, 0 traps`
-    : `${found}/${violations.length} violations, ${trapHits.length} trap(s) flagged` +
-      (missed.length ? `; missed: ${missed.map((m) => m.quote.slice(0, 40)).join(' | ')}` : '') +
-      (trapHits.length ? `; traps: ${trapHits.map((t) => t.quote.slice(0, 40)).join(' | ')}` : '');
+  const floor = minRecall < 1 ? `, floor ${minRecall}` : '';
+  const reason =
+    `${found}/${violations.length} violations, ${trapHits.length} trap(s) flagged${floor}` +
+    (missed.length ? `; missed: ${missed.map((m) => m.quote.slice(0, 40)).join(' | ')}` : '') +
+    (trapHits.length ? `; traps: ${trapHits.map((t) => t.quote.slice(0, 40)).join(' | ')}` : '');
 
   return { pass, score, reason };
 }

evals/bin/check-gate.mjs

@@ -50,10 +50,19 @@ const isHardFailure = (row) =>
     (meta(row).arguable !== true &&
       components(row).some((c) => !c.pass && c.assertion?.metric === 'choice')));
 
+// The rule judgment is recorded but never counts: a passage breaks more than
+// one rule, and the judge rejects a true rule the key did not name. A row
+// whose only failed asserts are informational passes for the floor.
+const INFORMATIONAL_METRICS = new Set(['rule']);
+const isInformational = (component) => INFORMATIONAL_METRICS.has(component.assertion?.metric);
+const countsAsPassed = (row) =>
+  row.success ||
+  (components(row).length > 0 && components(row).every((c) => c.pass || isInformational(c)));
+
 // Precedence: argv, then the skill's own `min_pass_rate` in evals.json, then
-// the default. A skill whose cases are soft by design (prose-register's
-// detection cases fail by construction) can carry a lower floor than one
-// whose cases all have a single right answer.
+// the default. A skill whose keys are contested or whose detection cases
+// record recall instead of failing (prose-register) can carry a lower floor
+// than one whose cases all have a single right answer.
 //
 // Only the named skill's file is parsed (the generator locates it the same
 // way), so a sibling's broken evals.json cannot fail this gate; an unreadable
@@ -80,10 +89,14 @@ if (!(minRate > 0 && minRate <= 1)) {
 }
 
 const hardFailures = rows.filter(isHardFailure);
-const passed = rows.filter((row) => row.success).length;
+const passed = rows.filter(countsAsPassed).length;
 const passRate = passed / rows.length;
 
-console.log(`${passed}/${rows.length} passed (rate ${(passRate * 100).toFixed(1)}%, floor ${(minRate * 100).toFixed(0)}%)`);
+const informational = rows.flatMap((row) => components(row).filter(isInformational));
+const informationalNote = informational.length
+  ? `; rule ${informational.filter((c) => c.pass).length}/${informational.length} informational`
+  : '';
+console.log(`${passed}/${rows.length} passed (rate ${(passRate * 100).toFixed(1)}%, floor ${(minRate * 100).toFixed(0)}%)${informationalNote}`);
 
 let failed = false;
 if (hardFailures.length > 0) {

evals/lib/load-evals.mjs

@@ -1,5 +1,6 @@
 import { readFileSync, readdirSync, existsSync, lstatSync } from 'node:fs';
 import path from 'node:path';
+import { normalize } from '../asserts/heuristics.mjs';
 
 export const SUPPORTED_TYPES = new Set([
   'discrimination',
@@ -107,6 +108,29 @@ export function validateData(data) {
       }
     }
 
+    // Alternatives the rule judge accepts besides expected_rule; only
+    // discrimination cases have a stated rule to judge.
+    if (item.accepted_rules !== undefined) {
+      if (!item.type.startsWith('discrimination')) {
+        throw new Error(`${item.id}.accepted_rules applies only to discrimination cases`);
+      }
+      const rules = item.accepted_rules;
+      if (!Array.isArray(rules) || rules.length === 0 || rules.some((rule) => typeof rule !== 'string' || rule === '')) {
+        throw new Error(`${item.id}.accepted_rules must be a non-empty array of rule strings`);
+      }
+    }
+
+    // Below 1 the case records recall as its score instead of failing on a
+    // non-exhaustive violation list; only detection has recall to floor.
+    if (item.min_recall !== undefined) {
+      if (item.type !== 'detection') {
+        throw new Error(`${item.id}.min_recall applies only to detection cases`);
+      }
+      if (typeof item.min_recall !== 'number' || !(item.min_recall >= 0 && item.min_recall <= 1)) {
+        throw new Error(`${item.id}.min_recall must be a number in [0, 1]`);
+      }
+    }
+
     if (item.type === 'detection') {
       requireField(item.prompt, `${item.id}.prompt`);
       requireField(item.input_document, `${item.id}.input_document`);
@@ -123,6 +147,19 @@ export function validateData(data) {
       for (const trap of item.traps) {
         requireField(trap.quote, `${item.id}.traps[].quote`);
       }
+      // The grader matches by text overlap, so a quote the document does not
+      // contain verbatim (an ellipsis, a paraphrase) can never be found or
+      // tripped.
+      const document = normalize(item.input_document);
+      for (const { quote, anchor } of [...item.violations, ...item.traps]) {
+        if (!document.includes(normalize(quote))) {
+          throw new Error(`${item.id} quote is not in input_document verbatim: "${quote.slice(0, 40)}"`);
+        }
+        // An anchor settles a match on its own, so it must be part of the quote it stands for.
+        if (anchor !== undefined && (typeof anchor !== 'string' || anchor === '' || !normalize(quote).includes(normalize(anchor)))) {
+          throw new Error(`${item.id} anchor must be a non-empty substring of its quote: "${String(anchor).slice(0, 40)}"`);
+        }
+      }
     }
   }
 

evals/lib/prompts.mjs

@@ -112,17 +112,24 @@ Notes:
 ${item.grading_note || 'None.'}`;
 }
 
+// The judge sees only the reference rule and the alternatives the case lists;
+// grading_note stays human-facing. Passed as free text, a note that asked for
+// a precise two-rule answer was read as a requirement and failed the answers
+// that met it.
 export function buildRuleRubric(item) {
+  const accepted = item.accepted_rules?.length
+    ? item.accepted_rules.map((rule) => `- ${rule}`).join('\n')
+    : 'None.';
   return `Grade only the line in the output that begins with "RULE:". Ignore which option was chosen and anything else in the output.
 
-The RULE line passes if it names the same principle as the reference rule in any wording, names the part of a compound reference rule that decides this case, or names an alternative the author's notes accept. It fails if it names a different principle, is missing, or is too vague to tell this rule apart from the skill's other rules.
+The RULE line passes if it names the same principle as the reference rule in any wording, names the part of a compound reference rule that decides this case, or names one of the accepted alternative rules. It fails if it names a different principle, is missing, or is too vague to tell this rule apart from the skill's other rules.
 
 Reference rule:
 ${item.expected_rule ?? item.expected_rule_for_worst}
 
 Rule text from the skill:
 ${item.rule_quote || 'None.'}
 
-Author's notes (may list accepted alternative rules):
-${item.grading_note || 'None.'}`;
+Accepted alternative rules (any one of these also passes):
+${accepted}`;
 }

evals/test/asserts.test.mjs

@@ -1,7 +1,12 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
 import assertDiscrimination from '../asserts/discrimination.mjs';
-import assertDetection from '../asserts/detection.mjs';
+import assertDetection, { quotesOverlap } from '../asserts/detection.mjs';
+import { findEvalFiles, loadEvals } from '../lib/load-evals.mjs';
+
+const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
 
 const discVars = {
   letter_to_key: { A: 'generic_comment', B: 'no_comment' },
@@ -102,3 +107,132 @@ test('detection ignores lines outside the QUOTE format', () => {
   const result = assertDetection(output, { vars: detVars });
   assert.equal(result.pass, true, 'prose mention of a trap outside violation lines must not count');
 });
+
+// det-03's miss: the key begins "same artifact — one rewrite", the subject
+// quoted from "artifact — one rewrite" onward and past the key's end.
+test('detection counts a quote that starts inside the key and runs past it', () => {
+  const vars = {
+    violations: [{ quote: 'same artifact — one rewrite by strangers', rule: 'No em-dashes.' }],
+    traps: [],
+  };
+  const output = '- QUOTE: "artifact — one rewrite by strangers, one by a machine" | RULE: em-dash';
+  const result = assertDetection(output, { vars });
+  assert.equal(result.pass, true, result.reason);
+  assert.equal(result.score, 1);
+});
+
+// det-03's second miss: the subject quoted the text on the left of the dash,
+// 15 characters of overlap with the key, under the floor.
+const dashQuotedFromTheLeft = '- QUOTE: "Same company, same invoice, same artifact —" | RULE: em-dash';
+
+test('detection counts a quote from the far side of a one-character violation when the key names an anchor', () => {
+  const vars = {
+    violations: [{ quote: 'same artifact — one rewrite by strangers', anchor: 'artifact —', rule: 'No em-dashes.' }],
+    traps: [],
+  };
+  const result = assertDetection(dashQuotedFromTheLeft, { vars });
+  assert.equal(result.pass, true, result.reason);
+});
+
+test('detection without an anchor misses a quote that overlaps the key by less than the floor', () => {
+  const vars = {
+    violations: [{ quote: 'same artifact — one rewrite by strangers', rule: 'No em-dashes.' }],
+    traps: [],
+  };
+  const result = assertDetection(dashQuotedFromTheLeft, { vars });
+  assert.equal(result.pass, false);
+  assert.match(result.reason, /0\/1 violations/);
+});
+
+test('detection counts a quote that wraps the key in context on both sides', () => {
+  const output = '- QUOTE: "Then: Larger chunks use more memory. Smaller chunks use more CPU. And so on." | RULE: generic';
+  const result = assertDetection(output, { vars: { ...detVars, violations: detVars.violations.slice(0, 1) } });
+  assert.equal(result.pass, true, result.reason);
+});
+
+test('detection does not count a fragment shorter than the overlap floor', () => {
+  const output = '- QUOTE: "more memory" | RULE: generic tradeoff';
+  const result = assertDetection(output, { vars: { ...detVars, violations: detVars.violations.slice(0, 1) } });
+  assert.equal(result.pass, false);
+  assert.match(result.reason, /0\/1 violations/);
+});
+
+test('detection flags a trap quoted from its middle, not only from its start', () => {
+  const output = [
+    '- QUOTE: "Larger chunks use more memory. Smaller chunks use more CPU." | RULE: generic tradeoff',
+    '- QUOTE: "Parse all records and drop the header before writing." | RULE: narration',
+    '- QUOTE: "returns a view of the buffer" | RULE: jargon',
+  ].join('\n');
+  const result = assertDetection(output, { vars: { ...detVars, traps: [{ quote: '`binary_part/3` returns a view of the buffer' }] } });
+  assert.equal(result.pass, false);
+  assert.match(result.reason, /1 trap/);
+});
+
+test('detection with min_recall 0 passes on partial recall and records recall as the score', () => {
+  const output = '- QUOTE: "Larger chunks use more memory. Smaller chunks use more CPU." | RULE: generic tradeoff';
+  const result = assertDetection(output, { vars: { ...detVars, min_recall: 0 } });
+  assert.equal(result.pass, true, result.reason);
+  assert.equal(result.score, 0.5);
+  assert.match(result.reason, /1\/2 violations, 0 trap\(s\) flagged, floor 0/);
+});
+
+test('detection with min_recall 0 still fails on a trap hit', () => {
+  const output = '- QUOTE: "`binary_part/3` returns a view" | RULE: needless jargon';
+  const result = assertDetection(output, { vars: { ...detVars, min_recall: 0 } });
+  assert.equal(result.pass, false);
+  assert.match(result.reason, /1 trap/);
+});
+
+test('detection with a fractional min_recall gates at that floor', () => {
+  const output = '- QUOTE: "Larger chunks use more memory. Smaller chunks use more CPU." | RULE: generic tradeoff';
+  assert.equal(assertDetection(output, { vars: { ...detVars, min_recall: 0.5 } }).pass, true);
+  assert.equal(assertDetection(output, { vars: { ...detVars, min_recall: 0.6 } }).pass, false);
+});
+
+const adjacentVars = {
+  violations: [
+    { quote: 'The creator of Bun spent the tokens. It took eleven days. A model wrote the commits.', rule: 'drumbeat' },
+    { quote: 'Developers did not take it well. Soon it will run on that rewrite. We are users now.', rule: 'drumbeat' },
+  ],
+  traps: [],
+};
+
+// Seen on a CI run: one subject line quoting the first paragraph and the
+// opening sentence of the next was credited to both violations.
+test('detection credits a quote that runs from one key into the next to the key it holds whole', () => {
+  const output = '- QUOTE: "The creator of Bun spent the tokens. It took eleven days. A model wrote the commits. Developers did not take it well." | RULE: drumbeat';
+  const result = assertDetection(output, { vars: adjacentVars });
+  assert.equal(result.pass, false);
+  assert.match(result.reason, /1\/2 violations/);
+});
+
+test('detection credits a fragment straddling two keys to the one it overlaps most', () => {
+  const output = '- QUOTE: "It took eleven days. A model wrote the commits. Developers did not take" | RULE: drumbeat';
+  const result = assertDetection(output, { vars: adjacentVars });
+  assert.match(result.reason, /1\/2 violations.*missed: Developers/);
+});
+
+test('detection credits a quote holding two whole keys to both', () => {
+  const output = `- QUOTE: "${adjacentVars.violations[0].quote} ${adjacentVars.violations[1].quote}" | RULE: drumbeat`;
+  const result = assertDetection(output, { vars: adjacentVars });
+  assert.equal(result.pass, true, result.reason);
+});
+
+// Overlap matching is symmetric enough that a violation quote sharing a run of
+// text with a trap would flag the trap on a correct answer; keep the fixtures
+// free of that.
+test('no detection case has a violation quote that would itself trip one of its traps', () => {
+  for (const file of findEvalFiles(REPO_ROOT)) {
+    const data = loadEvals(file);
+    for (const item of data.cases.filter((c) => c.type === 'detection')) {
+      for (const violation of item.violations) {
+        for (const trap of item.traps) {
+          assert.ok(
+            !quotesOverlap(trap.quote, violation.quote),
+            `${data.skill}/${item.id}: violation "${violation.quote.slice(0, 40)}" overlaps trap "${trap.quote.slice(0, 40)}"`,
+          );
+        }
+      }
+    }
+  }
+});