feat: emit behavior.* change codes from probe failures (#61)

docs/AGENT_EXECUTION.md

@@ -40,9 +40,14 @@ Positioning (use consistently):
 | Probes in CLI/CI | Compare/Action probe gates | #58 | P0 |
 | Unified `eval` | Missing | #76 | P0 |
 | Final-state verifier | Missing | #105 | P1 |
+<<<<<<< HEAD
+| pass@k / pass^k | Library trials only | #106 | P1 |
+| `behavior.*` codes | Emitted from probe/stability results | #61 | P1 |
+=======
 | pass@k / pass^k | First-class in StabilityReport | #106 | P1 |
 | `behavior.*` codes | Deferred | #61 | P1 |
 | Compatibility scorecard | Scorecard + explanations in compare | #77 | P1 |
+>>>>>>> origin/main
 | Semantic collision / rename / embeddings | Token Jaccard only | #79, #80, #117 | P1/P2 |
 | No-tool / workflows | Library harness (PRs) | #107, #108 | P1 |
 | Trace replay | Missing | #83, #84 | P1 |

docs/change-codes.md

@@ -38,19 +38,37 @@ Severities **`breaking`** and **`critical`** fail CI (`compare` exits `1`).
 
 ## Behavioral / model-backed signals (`behavior.*`)
 
-Structural `compare` does **not** yet emit `behavior.*` change codes.
-Model-backed probe work (#44–#47) ships separately as:
-
-- Library APIs: `evaluate_probes_with_model`, `compute_probe_metrics`,
-  `run_probe_trials` (see [probes.md](probes.md))
-- Report helpers: `render_probe_metrics_*` / `render_stability_*` in `report.py`
-
-Those results are **not** folded into `CompatibilityReport.changes` today, so
-they do not affect compare exit codes by themselves.
-
-The reserved `behavior.*` namespace (for example
-`behavior.tool_selection_failed`, `behavior.arguments_invalid`,
-`behavior.unstable_probe`) is **deferred**. Tracking issue:
-[#61](https://github.com/askmy-stack/tool-semantics/issues/61). Until that
-lands, do not invent ad-hoc behavioral codes in the diff engine; extend probe
-metrics instead or implement #61 with an update to this catalog in the same PR.
+Model-backed probe outcomes can be mapped to first-class `Change` entries via
+`tool_semantics.behavior_codes` (#61). Attach them to a structural
+`CompatibilityReport` with `merge_behavior_changes` so release policy treats
+selection/arg failures like other breaking codes.
+
+| Code | Typical severity | Meaning |
+| --- | --- | --- |
+| `behavior.tool_selection_failed` | breaking | Model selected the wrong tool (or a forbidden one) |
+| `behavior.arguments_invalid` | breaking | Model arguments failed schema / expected-argument checks |
+| `behavior.risk_expectation_failed` | breaking | Selected tool violated `max_risk` |
+| `behavior.confirmation_expectation_failed` | warning | Confirmation expectation was not met |
+| `behavior.unstable_probe` | warning | Selections/args varied across stability trials |
+| `behavior.deterministic_failure` | breaking | Same failing outcome on every stability trial |
+| `behavior.missing_data` | info | Model returned no usable tool call (never breaking alone) |
+| `behavior.evaluation_failed` | warning | Runner/provider error during evaluation |
+
+`probe` JSON reports include a `behavior_changes` array in model and stability
+modes. Missing-data and skipped (unapproved) probes do **not** emit breaking
+codes.
+
+Library:
+
+```python
+from tool_semantics.behavior_codes import (
+    changes_from_model_probe_report,
+    changes_from_stability_report,
+    merge_behavior_changes,
+)
+
+behavior = changes_from_model_probe_report(model_report)
+combined = merge_behavior_changes(structural_report, behavior)
+```
+
+Related APIs: `evaluate_probes_with_model`, `run_probe_trials` ([probes.md](probes.md)).

docs/probes.md

@@ -32,6 +32,11 @@ tool-semantics probe .tool-semantics/v1.json \
 
 Exit codes: `0` pass, `1` probe / stability failure, `2` input / config error.
 
+Model and stability JSON reports include a `behavior_changes` array of
+`behavior.*` codes (see [change-codes.md](change-codes.md)). Use
+`merge_behavior_changes` to fold them into a structural `CompatibilityReport`
+for policy gates.
+
 Probe files may be JSON or YAML: a list of probes, or `{ "probes": [ … ] }`.
 
 ## Compare / CI probe gate

src/tool_semantics/behavior_codes.py

@@ -0,0 +1,155 @@
+"""Emit behavior.* Change entries from model-backed probe outcomes (#61)."""
+
+from __future__ import annotations
+
+from tool_semantics.diff import Change, CompatibilityReport, Severity
+from tool_semantics.probes import (
+    ModelProbeOutcome,
+    ModelProbeReport,
+    ModelProbeResult,
+    StabilityReport,
+)
+
+# Stable catalog — keep in sync with docs/change-codes.md.
+BEHAVIOR_TOOL_SELECTION_FAILED = "behavior.tool_selection_failed"
+BEHAVIOR_ARGUMENTS_INVALID = "behavior.arguments_invalid"
+BEHAVIOR_UNSTABLE_PROBE = "behavior.unstable_probe"
+BEHAVIOR_DETERMINISTIC_FAILURE = "behavior.deterministic_failure"
+BEHAVIOR_RISK_EXPECTATION_FAILED = "behavior.risk_expectation_failed"
+BEHAVIOR_CONFIRMATION_EXPECTATION_FAILED = "behavior.confirmation_expectation_failed"
+BEHAVIOR_MISSING_DATA = "behavior.missing_data"
+BEHAVIOR_EVALUATION_FAILED = "behavior.evaluation_failed"
+
+
+def changes_from_model_probe_result(result: ModelProbeResult) -> list[Change]:
+    """Map one model probe result to zero or more behavior.* changes.
+
+    SKIPPED and successful outcomes emit nothing. MISSING_DATA emits info only
+    (never breaking) so absent model calls do not false-fail CI.
+    """
+    if result.outcome == ModelProbeOutcome.SKIPPED:
+        return []
+    if result.outcome == ModelProbeOutcome.MISSING_DATA:
+        return [
+            Change(
+                severity=Severity.INFO,
+                code=BEHAVIOR_MISSING_DATA,
+                subject=result.probe_id,
+                message=(
+                    f"Probe '{result.probe_id}' returned missing model data "
+                    f"(no tool call or incomplete response): {result.message}"
+                ),
+            )
+        ]
+    if result.outcome == ModelProbeOutcome.FAILED_EVALUATION:
+        return [
+            Change(
+                severity=Severity.WARNING,
+                code=BEHAVIOR_EVALUATION_FAILED,
+                subject=result.probe_id,
+                message=f"Probe '{result.probe_id}' evaluation failed: {result.message}",
+            )
+        ]
+
+    changes: list[Change] = []
+    if result.tool_selection_correct is False:
+        changes.append(
+            Change(
+                severity=Severity.BREAKING,
+                code=BEHAVIOR_TOOL_SELECTION_FAILED,
+                subject=result.probe_id,
+                message=(
+                    f"Probe '{result.probe_id}' selected wrong tool "
+                    f"('{result.selected_tool or 'none'}'): {result.message}"
+                ),
+            )
+        )
+    if result.arguments_valid is False:
+        changes.append(
+            Change(
+                severity=Severity.BREAKING,
+                code=BEHAVIOR_ARGUMENTS_INVALID,
+                subject=result.probe_id,
+                message=(
+                    f"Probe '{result.probe_id}' produced invalid arguments "
+                    f"{result.arguments}: {result.message}"
+                ),
+            )
+        )
+    if result.risk_compliant is False:
+        changes.append(
+            Change(
+                severity=Severity.BREAKING,
+                code=BEHAVIOR_RISK_EXPECTATION_FAILED,
+                subject=result.probe_id,
+                message=(
+                    f"Probe '{result.probe_id}' violated risk expectation "
+                    f"(selected '{result.selected_tool or 'none'}')."
+                ),
+            )
+        )
+    if result.confirmation_compliant is False:
+        changes.append(
+            Change(
+                severity=Severity.WARNING,
+                code=BEHAVIOR_CONFIRMATION_EXPECTATION_FAILED,
+                subject=result.probe_id,
+                message=(
+                    f"Probe '{result.probe_id}' violated confirmation expectation "
+                    f"(selected '{result.selected_tool or 'none'}')."
+                ),
+            )
+        )
+    return changes
+
+
+def changes_from_model_probe_report(report: ModelProbeReport) -> list[Change]:
+    out: list[Change] = []
+    for result in report.results:
+        out.extend(changes_from_model_probe_result(result))
+    return out
+
+
+def changes_from_stability_report(report: StabilityReport) -> list[Change]:
+    """Map stability summaries to distinct unstable vs deterministic codes (#47/#61)."""
+    changes: list[Change] = []
+    for summary in report.summaries:
+        if summary.deterministic_failure:
+            changes.append(
+                Change(
+                    severity=Severity.BREAKING,
+                    code=BEHAVIOR_DETERMINISTIC_FAILURE,
+                    subject=summary.probe_id,
+                    message=(
+                        f"Probe '{summary.probe_id}' failed deterministically across "
+                        f"{report.trial_count} trials: {summary.message}"
+                    ),
+                )
+            )
+        elif summary.unstable:
+            changes.append(
+                Change(
+                    severity=Severity.WARNING,
+                    code=BEHAVIOR_UNSTABLE_PROBE,
+                    subject=summary.probe_id,
+                    message=(
+                        f"Probe '{summary.probe_id}' is unstable across "
+                        f"{report.trial_count} trials "
+                        f"(stability={summary.stability_score:.2f}): {summary.message}"
+                    ),
+                )
+            )
+    return changes
+
+
+def merge_behavior_changes(
+    report: CompatibilityReport, behavior_changes: list[Change]
+) -> CompatibilityReport:
+    """Attach probe-derived changes onto a structural compatibility report."""
+    if not behavior_changes:
+        return report
+    return CompatibilityReport(
+        baseline=report.baseline,
+        candidate=report.candidate,
+        changes=[*report.changes, *behavior_changes],
+    )

src/tool_semantics/cli.py

@@ -10,6 +10,10 @@
 from rich.table import Table
 
 from tool_semantics import __version__
+from tool_semantics.behavior_codes import (
+    changes_from_model_probe_report,
+    changes_from_stability_report,
+)
 from tool_semantics.config import apply_ignore_rules, load_config
 from tool_semantics.diff import compare_snapshots
 from tool_semantics.mcp_capture import (
@@ -595,7 +599,12 @@ def probe(
         )
         if json_output is not None:
             json_output.parent.mkdir(parents=True, exist_ok=True)
-            json_output.write_text(render_stability_json(stability), encoding="utf-8")
+            payload = json.loads(render_stability_json(stability))
+            payload["behavior_changes"] = [
+                change.model_dump(mode="json")
+                for change in changes_from_stability_report(stability)
+            ]
+            json_output.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
         if markdown_output is not None:
             markdown_output.parent.mkdir(parents=True, exist_ok=True)
             markdown_output.write_text(render_stability_markdown(stability), encoding="utf-8")
@@ -632,13 +641,17 @@ def probe(
         )
     console.print(table)
     console.print(f"Result: [bold]{'PASS' if model_report.passed else 'FAIL'}[/bold]")
+    behavior_changes = changes_from_model_probe_report(model_report)
+    if behavior_changes:
+        console.print(f"Behavior changes: {len(behavior_changes)}")
     if json_output is not None:
         json_output.parent.mkdir(parents=True, exist_ok=True)
         payload = {
             "mode": "model",
             "passed": model_report.passed,
             "opt_in": model_report.opt_in,
             "results": [item.model_dump(mode="json") for item in model_report.results],
+            "behavior_changes": [change.model_dump(mode="json") for change in behavior_changes],
         }
         json_output.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
     if markdown_output is not None: