From 57a70f8922d9b0359efe53b39c911eecd12fd05c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 16 Sep 2026 12:56:12 +0000 Subject: [PATCH] feat: emit behavior.* change codes from probe failures (#61) Map model-backed probe and stability outcomes to stable behavior.* Change codes with documented severities. Missing-data is info-only; unstable vs deterministic failures use distinct codes. Probe JSON reports include behavior_changes; merge_behavior_changes attaches them to CompatibilityReport. Co-authored-by: Abhinaysai Kamineni --- docs/AGENT_EXECUTION.md | 2 +- docs/change-codes.md | 50 +++++--- docs/probes.md | 5 + src/tool_semantics/behavior_codes.py | 155 +++++++++++++++++++++++ src/tool_semantics/cli.py | 15 ++- tests/test_behavior_codes.py | 178 +++++++++++++++++++++++++++ 6 files changed, 387 insertions(+), 18 deletions(-) create mode 100644 src/tool_semantics/behavior_codes.py create mode 100644 tests/test_behavior_codes.py diff --git a/docs/AGENT_EXECUTION.md b/docs/AGENT_EXECUTION.md index c9b350d..42297a3 100644 --- a/docs/AGENT_EXECUTION.md +++ b/docs/AGENT_EXECUTION.md @@ -40,7 +40,7 @@ Positioning (use consistently): | Unified `eval` | Missing | #76 | P0 | | Final-state verifier | Missing | #105 | P1 | | pass@k / pass^k | Library trials only | #106 | P1 | -| `behavior.*` codes | Deferred | #61 | P1 | +| `behavior.*` codes | Emitted from probe/stability results | #61 | P1 | | Semantic collision / rename / embeddings | Token Jaccard only | #79, #80, #117 | P1/P2 | | No-tool / workflows | Missing | #107, #108 | P1 | | Trace replay | Missing | #83, #84 | P1 | diff --git a/docs/change-codes.md b/docs/change-codes.md index 7e50c8f..ec6e416 100644 --- a/docs/change-codes.md +++ b/docs/change-codes.md @@ -33,19 +33,37 @@ Severities **`breaking`** and **`critical`** fail CI (`compare` exits `1`). ## Behavioral / model-backed signals (`behavior.*`) -Structural `compare` does **not** yet emit `behavior.*` change codes. -Model-backed probe work (#44–#47) ships separately as: - -- Library APIs: `evaluate_probes_with_model`, `compute_probe_metrics`, - `run_probe_trials` (see [probes.md](probes.md)) -- Report helpers: `render_probe_metrics_*` / `render_stability_*` in `report.py` - -Those results are **not** folded into `CompatibilityReport.changes` today, so -they do not affect compare exit codes by themselves. - -The reserved `behavior.*` namespace (for example -`behavior.tool_selection_failed`, `behavior.arguments_invalid`, -`behavior.unstable_probe`) is **deferred**. Tracking issue: -[#61](https://github.com/askmy-stack/tool-semantics/issues/61). Until that -lands, do not invent ad-hoc behavioral codes in the diff engine; extend probe -metrics instead or implement #61 with an update to this catalog in the same PR. +Model-backed probe outcomes can be mapped to first-class `Change` entries via +`tool_semantics.behavior_codes` (#61). Attach them to a structural +`CompatibilityReport` with `merge_behavior_changes` so release policy treats +selection/arg failures like other breaking codes. + +| Code | Typical severity | Meaning | +| --- | --- | --- | +| `behavior.tool_selection_failed` | breaking | Model selected the wrong tool (or a forbidden one) | +| `behavior.arguments_invalid` | breaking | Model arguments failed schema / expected-argument checks | +| `behavior.risk_expectation_failed` | breaking | Selected tool violated `max_risk` | +| `behavior.confirmation_expectation_failed` | warning | Confirmation expectation was not met | +| `behavior.unstable_probe` | warning | Selections/args varied across stability trials | +| `behavior.deterministic_failure` | breaking | Same failing outcome on every stability trial | +| `behavior.missing_data` | info | Model returned no usable tool call (never breaking alone) | +| `behavior.evaluation_failed` | warning | Runner/provider error during evaluation | + +`probe` JSON reports include a `behavior_changes` array in model and stability +modes. Missing-data and skipped (unapproved) probes do **not** emit breaking +codes. + +Library: + +```python +from tool_semantics.behavior_codes import ( + changes_from_model_probe_report, + changes_from_stability_report, + merge_behavior_changes, +) + +behavior = changes_from_model_probe_report(model_report) +combined = merge_behavior_changes(structural_report, behavior) +``` + +Related APIs: `evaluate_probes_with_model`, `run_probe_trials` ([probes.md](probes.md)). diff --git a/docs/probes.md b/docs/probes.md index a999a86..ea41085 100644 --- a/docs/probes.md +++ b/docs/probes.md @@ -29,6 +29,11 @@ tool-semantics probe .tool-semantics/v1.json \ Exit codes: `0` pass, `1` probe / stability failure, `2` input / config error. +Model and stability JSON reports include a `behavior_changes` array of +`behavior.*` codes (see [change-codes.md](change-codes.md)). Use +`merge_behavior_changes` to fold them into a structural `CompatibilityReport` +for policy gates. + Probe files may be JSON or YAML: a list of probes, or `{ "probes": [ … ] }`. ## Approval workflow diff --git a/src/tool_semantics/behavior_codes.py b/src/tool_semantics/behavior_codes.py new file mode 100644 index 0000000..2bab7b1 --- /dev/null +++ b/src/tool_semantics/behavior_codes.py @@ -0,0 +1,155 @@ +"""Emit behavior.* Change entries from model-backed probe outcomes (#61).""" + +from __future__ import annotations + +from tool_semantics.diff import Change, CompatibilityReport, Severity +from tool_semantics.probes import ( + ModelProbeOutcome, + ModelProbeReport, + ModelProbeResult, + StabilityReport, +) + +# Stable catalog — keep in sync with docs/change-codes.md. +BEHAVIOR_TOOL_SELECTION_FAILED = "behavior.tool_selection_failed" +BEHAVIOR_ARGUMENTS_INVALID = "behavior.arguments_invalid" +BEHAVIOR_UNSTABLE_PROBE = "behavior.unstable_probe" +BEHAVIOR_DETERMINISTIC_FAILURE = "behavior.deterministic_failure" +BEHAVIOR_RISK_EXPECTATION_FAILED = "behavior.risk_expectation_failed" +BEHAVIOR_CONFIRMATION_EXPECTATION_FAILED = "behavior.confirmation_expectation_failed" +BEHAVIOR_MISSING_DATA = "behavior.missing_data" +BEHAVIOR_EVALUATION_FAILED = "behavior.evaluation_failed" + + +def changes_from_model_probe_result(result: ModelProbeResult) -> list[Change]: + """Map one model probe result to zero or more behavior.* changes. + + SKIPPED and successful outcomes emit nothing. MISSING_DATA emits info only + (never breaking) so absent model calls do not false-fail CI. + """ + if result.outcome == ModelProbeOutcome.SKIPPED: + return [] + if result.outcome == ModelProbeOutcome.MISSING_DATA: + return [ + Change( + severity=Severity.INFO, + code=BEHAVIOR_MISSING_DATA, + subject=result.probe_id, + message=( + f"Probe '{result.probe_id}' returned missing model data " + f"(no tool call or incomplete response): {result.message}" + ), + ) + ] + if result.outcome == ModelProbeOutcome.FAILED_EVALUATION: + return [ + Change( + severity=Severity.WARNING, + code=BEHAVIOR_EVALUATION_FAILED, + subject=result.probe_id, + message=f"Probe '{result.probe_id}' evaluation failed: {result.message}", + ) + ] + + changes: list[Change] = [] + if result.tool_selection_correct is False: + changes.append( + Change( + severity=Severity.BREAKING, + code=BEHAVIOR_TOOL_SELECTION_FAILED, + subject=result.probe_id, + message=( + f"Probe '{result.probe_id}' selected wrong tool " + f"('{result.selected_tool or 'none'}'): {result.message}" + ), + ) + ) + if result.arguments_valid is False: + changes.append( + Change( + severity=Severity.BREAKING, + code=BEHAVIOR_ARGUMENTS_INVALID, + subject=result.probe_id, + message=( + f"Probe '{result.probe_id}' produced invalid arguments " + f"{result.arguments}: {result.message}" + ), + ) + ) + if result.risk_compliant is False: + changes.append( + Change( + severity=Severity.BREAKING, + code=BEHAVIOR_RISK_EXPECTATION_FAILED, + subject=result.probe_id, + message=( + f"Probe '{result.probe_id}' violated risk expectation " + f"(selected '{result.selected_tool or 'none'}')." + ), + ) + ) + if result.confirmation_compliant is False: + changes.append( + Change( + severity=Severity.WARNING, + code=BEHAVIOR_CONFIRMATION_EXPECTATION_FAILED, + subject=result.probe_id, + message=( + f"Probe '{result.probe_id}' violated confirmation expectation " + f"(selected '{result.selected_tool or 'none'}')." + ), + ) + ) + return changes + + +def changes_from_model_probe_report(report: ModelProbeReport) -> list[Change]: + out: list[Change] = [] + for result in report.results: + out.extend(changes_from_model_probe_result(result)) + return out + + +def changes_from_stability_report(report: StabilityReport) -> list[Change]: + """Map stability summaries to distinct unstable vs deterministic codes (#47/#61).""" + changes: list[Change] = [] + for summary in report.summaries: + if summary.deterministic_failure: + changes.append( + Change( + severity=Severity.BREAKING, + code=BEHAVIOR_DETERMINISTIC_FAILURE, + subject=summary.probe_id, + message=( + f"Probe '{summary.probe_id}' failed deterministically across " + f"{report.trial_count} trials: {summary.message}" + ), + ) + ) + elif summary.unstable: + changes.append( + Change( + severity=Severity.WARNING, + code=BEHAVIOR_UNSTABLE_PROBE, + subject=summary.probe_id, + message=( + f"Probe '{summary.probe_id}' is unstable across " + f"{report.trial_count} trials " + f"(stability={summary.stability_score:.2f}): {summary.message}" + ), + ) + ) + return changes + + +def merge_behavior_changes( + report: CompatibilityReport, behavior_changes: list[Change] +) -> CompatibilityReport: + """Attach probe-derived changes onto a structural compatibility report.""" + if not behavior_changes: + return report + return CompatibilityReport( + baseline=report.baseline, + candidate=report.candidate, + changes=[*report.changes, *behavior_changes], + ) diff --git a/src/tool_semantics/cli.py b/src/tool_semantics/cli.py index 51a515c..1ac1643 100644 --- a/src/tool_semantics/cli.py +++ b/src/tool_semantics/cli.py @@ -10,6 +10,10 @@ from rich.table import Table from tool_semantics import __version__ +from tool_semantics.behavior_codes import ( + changes_from_model_probe_report, + changes_from_stability_report, +) from tool_semantics.config import apply_ignore_rules, load_config from tool_semantics.diff import compare_snapshots from tool_semantics.mcp_capture import ( @@ -534,7 +538,12 @@ def probe( ) if json_output is not None: json_output.parent.mkdir(parents=True, exist_ok=True) - json_output.write_text(render_stability_json(stability), encoding="utf-8") + payload = json.loads(render_stability_json(stability)) + payload["behavior_changes"] = [ + change.model_dump(mode="json") + for change in changes_from_stability_report(stability) + ] + json_output.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") if markdown_output is not None: markdown_output.parent.mkdir(parents=True, exist_ok=True) markdown_output.write_text(render_stability_markdown(stability), encoding="utf-8") @@ -571,6 +580,9 @@ def probe( ) console.print(table) console.print(f"Result: [bold]{'PASS' if model_report.passed else 'FAIL'}[/bold]") + behavior_changes = changes_from_model_probe_report(model_report) + if behavior_changes: + console.print(f"Behavior changes: {len(behavior_changes)}") if json_output is not None: json_output.parent.mkdir(parents=True, exist_ok=True) payload = { @@ -578,6 +590,7 @@ def probe( "passed": model_report.passed, "opt_in": model_report.opt_in, "results": [item.model_dump(mode="json") for item in model_report.results], + "behavior_changes": [change.model_dump(mode="json") for change in behavior_changes], } json_output.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") if markdown_output is not None: diff --git a/tests/test_behavior_codes.py b/tests/test_behavior_codes.py new file mode 100644 index 0000000..7b59770 --- /dev/null +++ b/tests/test_behavior_codes.py @@ -0,0 +1,178 @@ +"""Tests for behavior.* change codes (#61).""" + +from __future__ import annotations + +from tool_semantics.behavior_codes import ( + BEHAVIOR_ARGUMENTS_INVALID, + BEHAVIOR_DETERMINISTIC_FAILURE, + BEHAVIOR_MISSING_DATA, + BEHAVIOR_TOOL_SELECTION_FAILED, + BEHAVIOR_UNSTABLE_PROBE, + changes_from_model_probe_report, + changes_from_stability_report, + merge_behavior_changes, +) +from tool_semantics.diff import Change, CompatibilityReport, Severity +from tool_semantics.models import InterfaceSnapshot, RiskLevel, ToolContract, ToolParameter +from tool_semantics.probes import ( + ModelProbeOutcome, + ModelProbeReport, + ModelProbeResult, + Probe, + run_probe_trials, +) +from tool_semantics.runner import FakeModelRunner, ModelCompletion, RunnerMetadata, ToolCallRequest + + +def _snapshot() -> InterfaceSnapshot: + return InterfaceSnapshot( + server_name="demo", + tools=[ + ToolContract( + name="search_issues", + description="Search", + parameters=[ToolParameter(name="query", schema={"type": "string"}, required=True)], + risk=RiskLevel.READ_ONLY, + ), + ToolContract( + name="create_issue", + description="Create", + parameters=[ToolParameter(name="title", schema={"type": "string"}, required=True)], + risk=RiskLevel.EXTERNAL_WRITE, + ), + ], + ) + + +def test_model_selection_and_args_emit_breaking_codes() -> None: + report = ModelProbeReport( + results=[ + ModelProbeResult( + probe_id="bad-select", + passed=False, + message="wrong", + selected_tool="create_issue", + arguments={"title": "x"}, + outcome=ModelProbeOutcome.OK, + tool_selection_correct=False, + arguments_valid=True, + ), + ModelProbeResult( + probe_id="bad-args", + passed=False, + message="args", + selected_tool="search_issues", + arguments={}, + outcome=ModelProbeOutcome.OK, + tool_selection_correct=True, + arguments_valid=False, + ), + ] + ) + changes = changes_from_model_probe_report(report) + codes = {change.code for change in changes} + assert BEHAVIOR_TOOL_SELECTION_FAILED in codes + assert BEHAVIOR_ARGUMENTS_INVALID in codes + assert all(change.severity == Severity.BREAKING for change in changes) + + +def test_missing_data_is_info_not_breaking() -> None: + report = ModelProbeReport( + results=[ + ModelProbeResult( + probe_id="empty", + passed=False, + message="no call", + outcome=ModelProbeOutcome.MISSING_DATA, + tool_selection_correct=False, + ) + ] + ) + changes = changes_from_model_probe_report(report) + assert len(changes) == 1 + assert changes[0].code == BEHAVIOR_MISSING_DATA + assert changes[0].severity == Severity.INFO + + +def test_skipped_emits_nothing() -> None: + report = ModelProbeReport( + results=[ + ModelProbeResult( + probe_id="skip", + passed=False, + message="unapproved", + outcome=ModelProbeOutcome.SKIPPED, + ) + ] + ) + assert changes_from_model_probe_report(report) == [] + + +def test_stability_unstable_vs_deterministic_codes() -> None: + snap = _snapshot() + unstable_runner = FakeModelRunner( + [ + ModelCompletion( + tool_calls=[ToolCallRequest(name="search_issues", arguments={"query": "a"})], + metadata=RunnerMetadata(provider="fake", model="m"), + ), + ModelCompletion( + tool_calls=[ToolCallRequest(name="create_issue", arguments={"title": "x"})], + metadata=RunnerMetadata(provider="fake", model="m"), + ), + ] + ) + unstable = run_probe_trials( + snap, + [ + Probe( + id="u", + intent="search", + expected_tool="search_issues", + required_params=["query"], + approved=True, + ) + ], + unstable_runner, + trial_count=2, + ) + unstable_changes = changes_from_stability_report(unstable) + assert any(change.code == BEHAVIOR_UNSTABLE_PROBE for change in unstable_changes) + assert all(change.severity == Severity.WARNING for change in unstable_changes) + + fail_runner = FakeModelRunner( + [ + ModelCompletion( + tool_calls=[ToolCallRequest(name="create_issue", arguments={"title": "x"})], + metadata=RunnerMetadata(provider="fake", model="m"), + ) + for _ in range(2) + ] + ) + failed = run_probe_trials( + snap, + [Probe(id="d", intent="search", expected_tool="search_issues", approved=True)], + fail_runner, + trial_count=2, + ) + det = changes_from_stability_report(failed) + assert any(change.code == BEHAVIOR_DETERMINISTIC_FAILURE for change in det) + assert det[0].severity == Severity.BREAKING + + +def test_merge_behavior_changes_affects_compatibility() -> None: + structural = CompatibilityReport(baseline="a", candidate="b", changes=[]) + assert structural.is_compatible + merged = merge_behavior_changes( + structural, + [ + Change( + severity=Severity.BREAKING, + code=BEHAVIOR_TOOL_SELECTION_FAILED, + subject="p", + message="fail", + ) + ], + ) + assert not merged.is_compatible + assert len(merged.changes) == 1