From 8aa7468ac0a668e6cf1c924fcc04e5f228b22b77 Mon Sep 17 00:00:00 2001 From: Dat Date: Sun, 26 Jul 2026 16:27:26 +0700 Subject: [PATCH 01/15] feat(jobs): add scope_result and withheld_env to Job (S4 schema) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the Job record with the delegation security-posture fields for slice S4, keeping schema_version at 3 (additive optional fields; v1/v2 records still load with them absent, D7): - scope_result: Optional[ScopeResultDict] — diff-scope assertion outcome. None means no allowlist was declared (enforcement off), distinct from a declared scope that passed. status is ok/violated/undetermined. - withheld_env: Optional[list[str]] — NAMES (never values) of credential env vars withheld from the delegate. None means the scrub did not run. delegation_verdict folds a scope violation OR an undetermined scope into the existing 'failed' state (fail closed) rather than adding a fifth verdict, so the four-state contract and its tests are unchanged; an absent scope skips the gate entirely. runtime_status now surfaces both fields. --- src/crossagent/jobs.py | 50 +++++++++++++++++++++++ tests/test_jobs.py | 93 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+) diff --git a/src/crossagent/jobs.py b/src/crossagent/jobs.py index 0ab3697..a7fdb6e 100644 --- a/src/crossagent/jobs.py +++ b/src/crossagent/jobs.py @@ -44,6 +44,33 @@ class CheckResultDict(TypedDict): stderr_tail: str +# Outcome of the diff-scope assertion (slice S4). ``ok`` — every path the +# delegate modified was inside the declared allowlist; ``violated`` — it wrote +# outside its declared scope; ``undetermined`` — crossagent could not establish +# what changed (e.g. the cwd is not a git repo). ``undetermined`` is FAIL-CLOSED, +# never a pass: a scope check that silently passes when it cannot see the changes +# grants false assurance. +ScopeStatus = Literal["ok", "violated", "undetermined"] + + +class ScopeResultDict(TypedDict): + """Persisted outcome of the diff-scope assertion (slice S4). + + ``declared`` is the caller-supplied allowlist; ``violating_paths`` lists the + repo-relative paths the delegate modified outside it (empty unless + ``status == "violated"``); ``detail`` is a human-readable summary. + + A ``Job.scope_result`` of ``None`` means *no scope was declared* — scope + enforcement was off — which stays structurally distinct from a scope that + was declared and satisfied (D7: absent is never the same as ``ok``). + """ + + declared: list[str] + status: ScopeStatus + violating_paths: list[str] + detail: str + + # The delegation verdict keeps "the delegate finished" separate from "the work # was verified" (D5). See :func:`delegation_verdict`. DelegationVerdict = Literal["verified", "failed", "unverified", "incomplete"] @@ -191,6 +218,16 @@ class Job: # verification (this field) are deliberately separate fields so "finished" # can never be read as "verified" (D5). check_result: Optional[CheckResultDict] = None + # --- Delegation security posture (schema v3, slice S4) --------------- + # ``scope_result`` is the diff-scope assertion outcome. ``None`` means no + # allowlist was declared (enforcement off) — distinct from a declared scope + # that passed (D7). A declared scope that was violated OR could not be + # determined fails the delegation (fail closed); see delegation_verdict. + scope_result: Optional[ScopeResultDict] = None + # Names — never values — of credential-bearing env vars withheld from the + # advisor child. ``None`` means the scrub did not run (a pre-S4 record); an + # empty list means it ran and withheld nothing (D7: absent != empty). + withheld_env: Optional[list[str]] = None def delegation_verdict(job: Job) -> DelegationVerdict: @@ -206,11 +243,22 @@ def delegation_verdict(job: Job) -> DelegationVerdict: - ``unverified`` — the delegate finished cleanly but no check was run. This is NOT a pass: a missing gate is never green. - ``verified`` — the delegate finished cleanly and the check exited 0. + + Slice S4 folds the diff-scope assertion into this same verdict rather than + adding a fifth state: a delegate that wrote outside its declared allowlist, + or whose adherence to that allowlist could not be determined, has not + produced trustworthy work — that is semantically a *failed* delegation, so + ``scope_result.status`` other than ``ok`` yields ``failed`` (fail closed). + When no scope was declared (``scope_result is None``) this gate is skipped, + preserving the pre-S4 four-state behaviour and its tests unchanged. """ if not is_terminal(job.status): return "incomplete" if job.status != JobState.SUCCEEDED: return "failed" + scope = job.scope_result + if scope is not None and scope.get("status") != "ok": + return "failed" check = job.check_result if check is None: return "unverified" @@ -544,6 +592,8 @@ def runtime_status(job: Job) -> dict[str, Any]: "cost_source": job.cost_source, "model_reported": job.model_reported, "check_result": job.check_result, + "scope_result": job.scope_result, + "withheld_env": job.withheld_env, "delegation_verdict": delegation_verdict(job), } diff --git a/tests/test_jobs.py b/tests/test_jobs.py index 5168468..d4b1b87 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -1746,3 +1746,96 @@ def test_v2_record_loads_with_check_result_none(tmp_path): loaded = load_state(job_dir) assert loaded.check_result is None assert delegation_verdict(loaded) == "unverified" + + +# ========================================================================= +# Delegation verdict interaction with the diff-scope assertion (slice S4). +# A declared scope that was violated OR could not be determined fails the +# delegation (fail closed) without adding a fifth verdict state. +# ========================================================================= + + +def _scope(status, violating=()): + return { + "declared": ["src"], + "status": status, + "violating_paths": list(violating), + "detail": "", + } + + +def _succeeded_scope(scope_status, *, check_exit=None): + check = ( + None + if check_exit is None + else { + "command": "pytest", + "exit_code": check_exit, + "stdout_tail": "", + "stderr_tail": "", + } + ) + return Job( + job_id="job_v", + status=JobState.SUCCEEDED, + scope_result=_scope(scope_status), + check_result=check, + ) + + +def test_delegation_verdict_scope_violation_fails_even_with_passing_check(): + """A scope violation dominates: a passing check cannot green a delegation + that wrote outside its declared allowlist.""" + job = _succeeded_scope("violated", check_exit=0) + assert delegation_verdict(job) == "failed" + + +def test_delegation_verdict_undetermined_scope_is_failed_not_pass(): + """Fail closed: an undetermined scope (couldn't tell what changed) is never + verified, even with a passing check.""" + job = _succeeded_scope("undetermined", check_exit=0) + assert delegation_verdict(job) == "failed" + + +def test_delegation_verdict_scope_ok_defers_to_check_gate(): + """An in-bounds scope does not by itself verify: the check gate still runs. + Scope ok + passing check -> verified; scope ok + no check -> unverified.""" + assert delegation_verdict(_succeeded_scope("ok", check_exit=0)) == "verified" + assert delegation_verdict(_succeeded_scope("ok", check_exit=None)) == "unverified" + + +def test_delegation_verdict_scope_ok_but_check_fails_is_failed(): + assert delegation_verdict(_succeeded_scope("ok", check_exit=1)) == "failed" + + +def test_delegation_verdict_absent_scope_preserves_pre_s4_behaviour(): + """No scope declared (scope_result None) skips the gate entirely.""" + job = Job(job_id="job_v", status=JobState.SUCCEEDED, scope_result=None) + assert delegation_verdict(job) == "unverified" + + +def test_runtime_status_surfaces_scope_and_withheld_env(): + job = Job( + job_id="job_v", + status=JobState.SUCCEEDED, + scope_result=_scope("violated", violating=["evil.py"]), + withheld_env=["AWS_SECRET_ACCESS_KEY"], + ) + result = runtime_status(job) + assert result["scope_result"]["status"] == "violated" + assert result["withheld_env"] == ["AWS_SECRET_ACCESS_KEY"] + assert result["delegation_verdict"] == "failed" + + +def test_v3_scope_result_round_trips_on_disk(tmp_path): + job_dir = create_job_dir(tmp_path, "job_scope_rt") + job = Job( + job_id="job_scope_rt", + status=JobState.SUCCEEDED, + scope_result=_scope("violated", violating=["evil.py"]), + withheld_env=["GITHUB_TOKEN"], + ) + save_state(job_dir, job) + loaded = load_state(job_dir) + assert loaded.scope_result == _scope("violated", violating=["evil.py"]) + assert loaded.withheld_env == ["GITHUB_TOKEN"] From 782758fc33c0147b645c878a4cc271568497199e Mon Sep 17 00:00:00 2001 From: Dat Date: Sun, 26 Jul 2026 16:27:35 +0700 Subject: [PATCH 02/15] feat(security): withhold credential env vars from the delegate (S4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A delegate is partially untrusted (research finding [10]); it must not receive the caller's ambient secrets, or a prompt-injected delegate with a cloud key in its environment is a direct exfiltration path. credentials.scrub_env withholds every variable whose NAME matches a curated credential pattern (SECRET/TOKEN/PASSWORD/API_KEY/ACCESS_KEY/...) by default; withheld_names reports the withheld names for the audit trail. Both share one predicate so the launch env and the record can never disagree. Only names are ever recorded — a name is not a secret, its value is. --- src/crossagent/credentials.py | 76 +++++++++++++++++++++++++ tests/test_credentials.py | 104 ++++++++++++++++++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 src/crossagent/credentials.py create mode 100644 tests/test_credentials.py diff --git a/src/crossagent/credentials.py b/src/crossagent/credentials.py new file mode 100644 index 0000000..0469bd7 --- /dev/null +++ b/src/crossagent/credentials.py @@ -0,0 +1,76 @@ +"""Credential scrubbing for the advisor child environment (slice S4). + +A delegate is a partially-untrusted actor (research finding [10]: two-tier +delegation has a documented path from untrusted repo text to a committed +backdoor). crossagent must therefore not hand the delegate the caller's ambient +secrets — a compromised or prompt-injected delegate with a cloud key in its +environment is a direct exfiltration path. + +By default every environment variable whose NAME matches a credential pattern is +withheld from the child. A caller that genuinely needs one passed through (for +example the advisor's own API key) opts in explicitly with ``--pass-env NAME``. + +Only variable NAMES are ever recorded (in the audit log or the job record). A +name such as ``AWS_SECRET_ACCESS_KEY`` is not itself a secret, but its VALUE is +and must never appear in ``events.jsonl``, the redacted command, or any job +record field. +""" + +from __future__ import annotations + +from collections.abc import Mapping + +# Substrings (matched case-insensitively against the variable NAME) that mark a +# variable as credential-bearing. Curated to catch the common secret shapes +# without over-matching benign names: ``ACCESS_KEY`` matches +# ``AWS_ACCESS_KEY_ID`` but a bare ``KEY`` (which would also hit +# ``KEYBOARD_LAYOUT``) is deliberately not listed. +_CREDENTIAL_SUBSTRINGS: tuple[str, ...] = ( + "SECRET", + "TOKEN", + "PASSWORD", + "PASSWD", + "CREDENTIAL", + "API_KEY", + "APIKEY", + "ACCESS_KEY", + "PRIVATE_KEY", +) + + +def is_credential_name(name: str) -> bool: + """Return True when *name* looks like a credential-bearing env var.""" + upper = name.upper() + return any(marker in upper for marker in _CREDENTIAL_SUBSTRINGS) + + +def withheld_names( + env: Mapping[str, str], pass_through: list[str] | None = None +) -> list[str]: + """Return the sorted NAMES of credential vars that would be withheld. + + A name listed in *pass_through* is never withheld. Values are never + returned — only names, which are safe to record. + """ + allow = set(pass_through or []) + return sorted( + name for name in env if name not in allow and is_credential_name(name) + ) + + +def scrub_env( + env: Mapping[str, str], *, pass_through: list[str] | None = None +) -> dict[str, str]: + """Return a copy of *env* with credential-bearing variables removed. + + A variable is kept only when its name is explicitly allow-listed in + *pass_through* or does not match any credential pattern. Uses the same + :func:`is_credential_name` predicate as :func:`withheld_names`, so the two + can never disagree about what was scrubbed. + """ + allow = set(pass_through or []) + return { + name: value + for name, value in env.items() + if name in allow or not is_credential_name(name) + } diff --git a/tests/test_credentials.py b/tests/test_credentials.py new file mode 100644 index 0000000..c1ab4a4 --- /dev/null +++ b/tests/test_credentials.py @@ -0,0 +1,104 @@ +"""Tests for credential env scrubbing (slice S4). + +A delegate is partially untrusted, so the caller's ambient secrets must not be +handed to it. ``scrub_env`` withholds credential-bearing variables by default; +``withheld_names`` reports the withheld NAMES (never values) for the audit +trail; ``--pass-env`` is the explicit opt-in exception. +""" + +from __future__ import annotations + +from crossagent import credentials as credentials_mod +from crossagent.credentials import is_credential_name, scrub_env, withheld_names + + +def test_is_credential_name_matches_common_secret_shapes(): + for name in ( + "AWS_SECRET_ACCESS_KEY", + "AWS_ACCESS_KEY_ID", + "GITHUB_TOKEN", + "DB_PASSWORD", + "MYSQL_PASSWD", + "STRIPE_API_KEY", + "OPENAI_APIKEY", + "GCP_CREDENTIALS", + "SSH_PRIVATE_KEY", + "SESSION_SECRET", + ): + assert is_credential_name(name), name + + +def test_is_credential_name_does_not_over_match_benign_names(): + for name in ("PATH", "HOME", "LANG", "KEYBOARD_LAYOUT", "SHELL", "TERM"): + assert not is_credential_name(name), name + + +def test_is_credential_name_is_case_insensitive(): + assert is_credential_name("aws_secret_access_key") + assert is_credential_name("Github_Token") + + +def test_scrub_env_removes_secrets_but_keeps_benign_vars(): + env = { + "PATH": "/usr/bin", + "HOME": "/home/dat", + "AWS_SECRET_ACCESS_KEY": "super-secret-value", + "GITHUB_TOKEN": "ghp_secret", + } + scrubbed = scrub_env(env) + assert scrubbed == {"PATH": "/usr/bin", "HOME": "/home/dat"} + + +def test_scrub_env_secret_value_never_appears_in_output(): + env = {"DB_PASSWORD": "hunter2", "PATH": "/usr/bin"} + scrubbed = scrub_env(env) + assert "hunter2" not in scrubbed.values() + assert "DB_PASSWORD" not in scrubbed + + +def test_scrub_env_pass_through_allowlist_keeps_named_var(): + env = {"ANTHROPIC_API_KEY": "sk-ant-xxx", "AWS_SECRET_ACCESS_KEY": "aws"} + scrubbed = scrub_env(env, pass_through=["ANTHROPIC_API_KEY"]) + assert scrubbed == {"ANTHROPIC_API_KEY": "sk-ant-xxx"} + + +def test_scrub_env_does_not_mutate_input(): + env = {"GITHUB_TOKEN": "x", "PATH": "/usr/bin"} + scrub_env(env) + assert "GITHUB_TOKEN" in env # original untouched + + +def test_withheld_names_returns_sorted_names_only(): + env = { + "GITHUB_TOKEN": "x", + "AWS_SECRET_ACCESS_KEY": "y", + "PATH": "/usr/bin", + } + assert withheld_names(env) == ["AWS_SECRET_ACCESS_KEY", "GITHUB_TOKEN"] + + +def test_withheld_names_excludes_pass_through(): + env = {"GITHUB_TOKEN": "x", "ANTHROPIC_API_KEY": "y"} + assert withheld_names(env, pass_through=["ANTHROPIC_API_KEY"]) == ["GITHUB_TOKEN"] + + +def test_withheld_names_empty_when_nothing_secret(): + assert withheld_names({"PATH": "/usr/bin", "HOME": "/home/dat"}) == [] + + +def test_scrub_and_withheld_agree_on_the_same_predicate(): + """The launch env and the audit record must never disagree about what was + scrubbed — both go through ``is_credential_name``.""" + env = {"PATH": "/usr/bin", "API_KEY": "k", "SESSION_TOKEN": "t"} + scrubbed = set(scrub_env(env)) + withheld = set(withheld_names(env)) + # Every name is either kept or withheld, never both, never neither. + assert scrubbed | withheld == set(env) + assert scrubbed & withheld == set() + + +def test_credential_substrings_are_uppercase_constants(): + # Guard against a lowercase entry silently never matching (predicate uppers). + assert all( + marker == marker.upper() for marker in credentials_mod._CREDENTIAL_SUBSTRINGS + ) From 8a662756476c19aed6ff066d2b99c3e66070976d Mon Sep 17 00:00:00 2001 From: Dat Date: Sun, 26 Jul 2026 16:27:42 +0700 Subject: [PATCH 03/15] feat(security): add git diff-scope assertion for delegated writes (S4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a delegate finishes, assert every path it modified lies inside the caller-declared allowlist. Modified paths are determined from git (argument list, shell=False — matching check.py), diffed against a baseline captured BEFORE the delegate ran so a pre-existing dirty tree is not misattributed; paths dirty both before and after are content-hashed so a further edit or a revert is still attributed. Paths are compared by resolved real location so symlink and .. traversal cannot smuggle a write outside the allowlist. Fails closed: a non-git cwd, unavailable git, or a repo-identity shift yields 'undetermined' — a distinct, visible outcome, never a silent pass. assert_scope never raises, so the security check can neither crash the worker nor degrade into a pass. Documented blind spots: .gitignore'd paths, create-then-delete. --- src/crossagent/scope.py | 326 ++++++++++++++++++++++++++++++++++++++++ tests/test_scope.py | 239 +++++++++++++++++++++++++++++ 2 files changed, 565 insertions(+) create mode 100644 src/crossagent/scope.py create mode 100644 tests/test_scope.py diff --git a/src/crossagent/scope.py b/src/crossagent/scope.py new file mode 100644 index 0000000..35e53fd --- /dev/null +++ b/src/crossagent/scope.py @@ -0,0 +1,326 @@ +"""Diff-scope assertion for delegated work (slice S4). + +A delegate has write authority over the user's repo that a second-opinion run +never had (research finding [10]: the two-tier delegation pattern crossagent +generalizes has a documented path from untrusted repo text to a committed +backdoor). The caller declares an allowlist of paths the delegate may modify via +``--allow-path``; after the delegate finishes, crossagent asserts that every +file it changed lies inside that allowlist and records a *failed delegation* +otherwise. + +How "modified paths" is determined +----------------------------------- +git, run as an argument list with ``shell=False`` — never through a shell, and +no delegate output is ever interpolated into a command line (matching the +standard set by ``check.py``). A baseline of the working tree's dirty set is +captured *before* the delegate runs; after it finishes the dirty set is +recomputed and diffed against the baseline, so a tree that was *already* dirty is +not misattributed to the delegate. For a path that was dirty both before and +after, the file's content hash is compared so a further modification (or a +revert to HEAD) is still attributed. Paths are compared by their RESOLVED real +location — symlinks and ``..`` eliminated — against the resolved declared roots, +so neither traversal nor a symlink can smuggle a write outside the allowlist. + +Blind spots (documented, never hidden): +- ``.gitignore``-d paths are invisible to ``git status`` and therefore to this + check. A delegate writing to an ignored path (build output, a ``.env`` file) + is not detected. Catching these would require hashing every ignored file + (e.g. all of ``node_modules``) and is left to a follow-up. +- A file the delegate creates and then deletes leaves no trace in the dirty set + and is not detected. +- Content changed inside an already-ignored directory is not detected. + +Fail closed +----------- +If crossagent cannot establish what changed — the cwd is not a git repository, +git is unavailable, or the repo identity shifts mid-run — the outcome is +``undetermined``, NOT a pass. A scope check that silently passes when it cannot +see the changes is worse than no check: it grants false assurance. +""" + +from __future__ import annotations + +import hashlib +import os +import subprocess +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +from .jobs import ScopeResultDict, ScopeStatus + +_GIT_TIMEOUT_SECONDS = 30.0 +# porcelain -z entries are "XY ": two status chars, a space, then the path. +_STATUS_ENTRY_PREFIX = 3 +_HASH_CHUNK_BYTES = 65536 + + +@dataclass(frozen=True) +class ScopeBaseline: + """The working-tree state captured *before* the delegate runs. + + ``repo_root`` is ``None`` when no baseline could be established (not a git + repo, git unavailable), in which case *error* explains why and the scope + outcome is ``undetermined`` (fail closed). ``dirty_hashes`` maps each + already-dirty repo-relative path to its content hash (or ``None`` when the + file could not be read), so pre-existing dirt is not later misattributed. + """ + + repo_root: Optional[str] + dirty_hashes: dict[str, Optional[str]] = field(default_factory=dict) + error: Optional[str] = None + + +@dataclass(frozen=True) +class ScopeOutcome: + """The result of asserting the delegate's writes against the allowlist.""" + + declared: tuple[str, ...] + status: ScopeStatus + violating_paths: tuple[str, ...] + detail: str + + def to_dict(self) -> ScopeResultDict: + """Return the persisted-record shape for ``Job.scope_result``.""" + return { + "declared": list(self.declared), + "status": self.status, + "violating_paths": list(self.violating_paths), + "detail": self.detail, + } + + +# --------------------------------------------------------------------------- +# git helpers (argument lists, shell=False — matches check.py) +# --------------------------------------------------------------------------- + + +def _run_git(args: list[str], cwd: str) -> Optional[subprocess.CompletedProcess]: + """Run ``git `` in *cwd* as an argument list, never through a shell. + + Returns the completed process, or ``None`` when git could not be executed at + all (not installed, timed out, OS error). An un-runnable git means scope + cannot be determined — callers surface that as ``undetermined``, never a + pass. + """ + try: + return subprocess.run( + ["git", *args], + cwd=cwd, + capture_output=True, + text=True, + timeout=_GIT_TIMEOUT_SECONDS, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + + +def _repo_root(cwd: str) -> Optional[str]: + """Return the git worktree root for *cwd*, or ``None`` when it is not one.""" + completed = _run_git(["rev-parse", "--show-toplevel"], cwd) + if completed is None or completed.returncode != 0: + return None + root = completed.stdout.strip() + return root or None + + +def _dirty_paths(cwd: str) -> Optional[list[str]]: + """Return repo-relative paths of every dirty file, or ``None`` on git error. + + Covers tracked-modified, staged, and untracked files. ``--no-renames`` keeps + the output a flat path list (a rename surfaces as delete + add, both + "touched") so there is no rename-arrow parsing to get wrong. Ignored files + are intentionally excluded — see the module docstring's blind spots. + """ + completed = _run_git( + ["status", "--porcelain", "-z", "--untracked-files=all", "--no-renames"], + cwd, + ) + if completed is None or completed.returncode != 0: + return None + paths: list[str] = [] + for entry in completed.stdout.split("\0"): + if len(entry) <= _STATUS_ENTRY_PREFIX: + continue + paths.append(entry[_STATUS_ENTRY_PREFIX:]) + return paths + + +def _hash_file(path: Path) -> Optional[str]: + """Return the sha256 of *path*'s bytes, or ``None`` if it cannot be read.""" + try: + with path.open("rb") as handle: + digest = hashlib.sha256() + for chunk in iter(lambda: handle.read(_HASH_CHUNK_BYTES), b""): + digest.update(chunk) + return digest.hexdigest() + except (OSError, ValueError): + return None + + +# --------------------------------------------------------------------------- +# Path matching (resolve real locations so symlink / .. cannot escape) +# --------------------------------------------------------------------------- + + +def _resolve(path: Path) -> Path: + """Resolve *path* to a real absolute location (symlinks and ``..`` gone). + + ``Path.resolve`` is non-strict on Python 3.9+, so a not-yet-existing path + (e.g. a file the delegate deleted) still normalizes lexically. A resolution + error (symlink loop) falls back to a lexical normalization so matching never + crashes. + """ + try: + return path.resolve() + except OSError: + return Path(os.path.normpath(str(path))) + + +def _resolve_declared_roots(declared: tuple[str, ...], cwd: str) -> list[Path]: + """Resolve each declared allowlist entry, relative to *cwd*, to a real path.""" + base = Path(cwd) + roots: list[Path] = [] + for pattern in declared: + candidate = Path(pattern) + if not candidate.is_absolute(): + candidate = base / candidate + roots.append(_resolve(candidate)) + return roots + + +def _is_in_scope(modified: Path, roots: list[Path]) -> bool: + """Return True when *modified* is equal to or under one declared root. + + Both sides are resolved first, so a ``..`` segment or a symlinked directory + cannot make an out-of-scope write appear in scope: the comparison is on real + locations, not on the paths as typed. + """ + resolved = _resolve(modified) + for root in roots: + try: + resolved.relative_to(root) + return True + except ValueError: + continue + return False + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def capture_baseline(cwd: str) -> ScopeBaseline: + """Capture the pre-delegation working-tree state for a later scope check.""" + root = _repo_root(cwd) + if root is None: + return ScopeBaseline( + repo_root=None, + error=(f"cwd is not a git repository (or git is unavailable): {cwd}"), + ) + entries = _dirty_paths(cwd) + if entries is None: + return ScopeBaseline( + repo_root=None, + error="git status failed while capturing the pre-delegation baseline", + ) + hashes = {rel: _hash_file(Path(root) / rel) for rel in entries} + return ScopeBaseline(repo_root=root, dirty_hashes=hashes) + + +def assert_scope( + baseline: ScopeBaseline, declared_paths: list[str], cwd: str +) -> ScopeOutcome: + """Assert the delegate's writes stayed within *declared_paths*. + + Never raises: any unexpected error is captured as ``undetermined`` (fail + closed and surfaced), so the security check can never crash the worker and + can never silently degrade into a pass. + """ + declared = tuple(declared_paths) + try: + return _assert_scope(baseline, declared, cwd) + except Exception as exc: # fail closed; surface, never crash the worker + return ScopeOutcome( + declared=declared, + status="undetermined", + violating_paths=(), + detail=f"scope check errored unexpectedly: {exc!r}", + ) + + +def _assert_scope( + baseline: ScopeBaseline, declared: tuple[str, ...], cwd: str +) -> ScopeOutcome: + if baseline.repo_root is None: + return ScopeOutcome( + declared, + "undetermined", + (), + baseline.error or "could not determine the pre-delegation baseline", + ) + + # The repo identity must be stable across the run; a cwd that stopped being + # the same worktree means we can no longer attribute changes. Fail closed. + after_root = _repo_root(cwd) + if after_root is None or after_root != baseline.repo_root: + return ScopeOutcome( + declared, + "undetermined", + (), + "git repository became unavailable or changed identity during the run", + ) + + after_paths = _dirty_paths(cwd) + if after_paths is None: + return ScopeOutcome( + declared, + "undetermined", + (), + "git status failed while evaluating post-delegation changes", + ) + + touched = _delegate_touched(baseline, after_paths) + repo_root = Path(baseline.repo_root) + roots = _resolve_declared_roots(declared, cwd) + violating = tuple( + sorted(rel for rel in touched if not _is_in_scope(repo_root / rel, roots)) + ) + if violating: + return ScopeOutcome( + declared, + "violated", + violating, + f"delegate modified {len(violating)} path(s) outside the declared scope", + ) + return ScopeOutcome( + declared, + "ok", + (), + f"all {len(touched)} modified path(s) were inside the declared scope", + ) + + +def _delegate_touched(baseline: ScopeBaseline, after_paths: list[str]) -> set[str]: + """Return repo-relative paths the delegate actually changed. + + Pre-existing dirt (dirty before AND after with identical content) is + excluded so it is never misattributed; a path further modified, newly + dirtied, or reverted to HEAD by the delegate is included. + """ + repo_root = Path(baseline.repo_root or "") + before = baseline.dirty_hashes + before_set = set(before) + after_set = set(after_paths) + touched: set[str] = set() + # Clean before, dirty after -> the delegate created or modified it. + touched |= after_set - before_set + # Dirty before but clean after -> the delegate reverted it to HEAD. + touched |= before_set - after_set + # Dirty before and after -> attribute only if the content actually changed. + for rel in after_set & before_set: + if _hash_file(repo_root / rel) != before[rel]: + touched.add(rel) + return touched diff --git a/tests/test_scope.py b/tests/test_scope.py new file mode 100644 index 0000000..fa07652 --- /dev/null +++ b/tests/test_scope.py @@ -0,0 +1,239 @@ +"""Tests for the diff-scope assertion (slice S4). + +The assertion answers: did the delegate modify only paths inside the declared +allowlist? It determines "modified paths" from git, diffed against a baseline +captured before the delegate ran, and fails CLOSED when it cannot tell. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +from crossagent import scope as scope_mod +from crossagent.scope import ScopeBaseline, assert_scope, capture_baseline + + +def _git(args: list[str], cwd: Path) -> None: + subprocess.run( + ["git", *args], + cwd=str(cwd), + check=True, + capture_output=True, + text=True, + ) + + +def _init_repo(path: Path) -> None: + _git(["init"], path) + # Identity so a commit can be made without touching global config. + _git(["config", "user.email", "test@example.com"], path) + _git(["config", "user.name", "Test"], path) + + +def _commit_file(repo: Path, rel: str, content: str) -> None: + target = repo / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + _git(["add", rel], repo) + _git(["commit", "-m", f"add {rel}"], repo) + + +def _write(repo: Path, rel: str, content: str) -> None: + target = repo / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + + +@pytest.fixture +def repo(tmp_path: Path) -> Path: + _init_repo(tmp_path) + return tmp_path + + +# --------------------------------------------------------------------------- +# Happy path + violation +# --------------------------------------------------------------------------- + + +def test_declared_only_edits_pass(repo: Path): + _commit_file(repo, "src/app.py", "clean\n") + baseline = capture_baseline(str(repo)) + _write(repo, "src/app.py", "edited by delegate\n") + outcome = assert_scope(baseline, ["src"], str(repo)) + assert outcome.status == "ok" + assert outcome.violating_paths == () + + +def test_touching_undeclared_path_fails_and_lists_offenders(repo: Path): + _commit_file(repo, "src/app.py", "clean\n") + _commit_file(repo, "secret/keys.txt", "clean\n") + baseline = capture_baseline(str(repo)) + _write(repo, "src/app.py", "ok edit\n") + _write(repo, "secret/keys.txt", "delegate tampered\n") + outcome = assert_scope(baseline, ["src"], str(repo)) + assert outcome.status == "violated" + assert outcome.violating_paths == ("secret/keys.txt",) + + +def test_new_untracked_file_outside_scope_is_a_violation(repo: Path): + baseline = capture_baseline(str(repo)) + _write(repo, "src/new.py", "in scope\n") + _write(repo, "evil.py", "out of scope\n") + outcome = assert_scope(baseline, ["src"], str(repo)) + assert outcome.status == "violated" + assert outcome.violating_paths == ("evil.py",) + + +def test_delegate_that_changed_nothing_is_ok(repo: Path): + _commit_file(repo, "src/app.py", "clean\n") + baseline = capture_baseline(str(repo)) + outcome = assert_scope(baseline, ["src"], str(repo)) + assert outcome.status == "ok" + assert outcome.violating_paths == () + + +# --------------------------------------------------------------------------- +# Pre-existing dirt is not misattributed +# --------------------------------------------------------------------------- + + +def test_pre_existing_dirty_tree_is_not_misattributed(repo: Path): + """A file already dirty BEFORE the delegate ran must not be blamed on it.""" + _commit_file(repo, "src/app.py", "clean\n") + _commit_file(repo, "notes.txt", "clean\n") + # notes.txt is dirty before the delegate runs, outside the declared scope. + _write(repo, "notes.txt", "user's own uncommitted edit\n") + baseline = capture_baseline(str(repo)) + # The delegate only touches an in-scope file. + _write(repo, "src/app.py", "delegate edit\n") + outcome = assert_scope(baseline, ["src"], str(repo)) + assert outcome.status == "ok", outcome.detail + + +def test_further_modifying_an_already_dirty_file_is_attributed(repo: Path): + """A file dirty before AND further modified by the delegate IS attributed.""" + _commit_file(repo, "app.py", "clean\n") + _write(repo, "app.py", "user edit\n") # dirty before + baseline = capture_baseline(str(repo)) + _write(repo, "app.py", "delegate changed it further\n") # content differs + outcome = assert_scope(baseline, ["src"], str(repo)) + assert outcome.status == "violated" + assert outcome.violating_paths == ("app.py",) + + +def test_reverting_a_pre_existing_change_is_attributed(repo: Path): + """If the delegate reverts a user's dirty file back to HEAD, that is a change + it made and must be attributed.""" + _commit_file(repo, "app.py", "committed\n") + _write(repo, "app.py", "user's uncommitted edit\n") # dirty before + baseline = capture_baseline(str(repo)) + _write(repo, "app.py", "committed\n") # reverted to HEAD -> clean after + outcome = assert_scope(baseline, ["src"], str(repo)) + assert outcome.status == "violated" + assert outcome.violating_paths == ("app.py",) + + +# --------------------------------------------------------------------------- +# Traversal / symlink escape must not defeat the matcher +# --------------------------------------------------------------------------- + + +def test_dotdot_in_declared_scope_does_not_widen_it(repo: Path): + """`--allow-path allowed/../allowed` resolves to `allowed`, not the repo + root, so an out-of-scope sibling is still a violation.""" + _commit_file(repo, "allowed/a.py", "clean\n") + _commit_file(repo, "other/b.py", "clean\n") + baseline = capture_baseline(str(repo)) + _write(repo, "other/b.py", "delegate edit\n") + outcome = assert_scope(baseline, ["allowed/../allowed"], str(repo)) + assert outcome.status == "violated" + assert outcome.violating_paths == ("other/b.py",) + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX symlink semantics") +def test_symlinked_write_target_is_resolved_before_matching(repo: Path): + """A write that lands (via a symlink) in a real location outside the declared + scope is flagged: git reports the real path, which we resolve and match.""" + (repo / "allowed").mkdir() + (repo / "secret").mkdir() + # A symlink inside the allowed dir pointing at the out-of-scope secret dir. + (repo / "allowed" / "link").symlink_to(repo / "secret") + baseline = capture_baseline(str(repo)) + # Writing "through" the symlink creates secret/leak.txt — git reports it at + # its real repo-relative path, which resolves outside `allowed`. + _write(repo, "secret/leak.txt", "exfiltrated\n") + outcome = assert_scope(baseline, ["allowed"], str(repo)) + assert outcome.status == "violated" + assert "secret/leak.txt" in outcome.violating_paths + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX symlink semantics") +def test_symlinked_declared_root_still_contains_its_real_subtree(repo: Path): + """A declared root that is itself a symlink resolves to its target, and a + write inside that target is correctly in scope.""" + (repo / "real").mkdir() + (repo / "alias").symlink_to(repo / "real") + baseline = capture_baseline(str(repo)) + _write(repo, "real/x.py", "edit\n") + outcome = assert_scope(baseline, ["alias"], str(repo)) + assert outcome.status == "ok", outcome.detail + + +# --------------------------------------------------------------------------- +# Fail closed: non-git cwd, git unavailable, identity shift +# --------------------------------------------------------------------------- + + +def test_non_git_cwd_is_undetermined_not_a_pass(tmp_path: Path): + baseline = capture_baseline(str(tmp_path)) + assert baseline.repo_root is None + outcome = assert_scope(baseline, ["src"], str(tmp_path)) + assert outcome.status == "undetermined" + assert outcome.violating_paths == () + assert "git" in outcome.detail.lower() + + +def test_repo_identity_shift_mid_run_is_undetermined(repo: Path, tmp_path_factory): + """If the baseline was captured in one repo but the after-check sees a + different worktree root, fail closed.""" + _commit_file(repo, "src/app.py", "clean\n") + baseline = capture_baseline(str(repo)) + other = tmp_path_factory.mktemp("other_repo") + _init_repo(other) + outcome = assert_scope(baseline, ["src"], str(other)) + assert outcome.status == "undetermined" + + +def test_unexpected_error_is_undetermined_not_a_crash(): + """assert_scope must never raise: a malformed baseline yields undetermined, + fail closed, rather than propagating an exception into the worker.""" + bogus = ScopeBaseline(repo_root=12345) # type: ignore[arg-type] + outcome = assert_scope(bogus, ["src"], "/nonexistent") + assert outcome.status == "undetermined" + assert outcome.violating_paths == () + + +def test_to_dict_shape_is_stable(repo: Path): + baseline = capture_baseline(str(repo)) + outcome = assert_scope(baseline, ["src"], str(repo)) + as_dict = outcome.to_dict() + assert set(as_dict) == {"declared", "status", "violating_paths", "detail"} + assert as_dict["declared"] == ["src"] + assert isinstance(as_dict["violating_paths"], list) + + +def test_git_unavailable_is_undetermined(repo: Path, monkeypatch): + """If git cannot be executed at all, scope is undetermined (fail closed).""" + + def _boom(*_args, **_kwargs): + raise OSError("git not found") + + monkeypatch.setattr(scope_mod.subprocess, "run", _boom) + baseline = capture_baseline(str(repo)) + assert baseline.repo_root is None + outcome = assert_scope(baseline, ["src"], str(repo)) + assert outcome.status == "undetermined" From 66ca81167ccb7e9993e65df022bdccd2f47a4dc1 Mon Sep 17 00:00:00 2001 From: Dat Date: Sun, 26 Jul 2026 16:27:51 +0700 Subject: [PATCH 04/15] feat(delegate): enforce scope and credential scrubbing in the worker (S4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the S4 posture into the delegation path: - start gains --allow-path (repeatable; declares the write allowlist, None when omitted) and --pass-env (opt-in credential passthrough), persisted to command.json. - The worker captures the scope baseline before the delegate runs, builds the advisor env through credentials.scrub_env (lineage set after the scrub so it is never stripped), runs the diff-scope assertion after the delegate + check, and forwards scope_result + withheld_env onto the SAME terminal transition — the only writer of terminal state. Audit events 'scope' (status + offending paths) and 'env_scrub' (withheld names only) are appended. End-to-end tests drive real jobs through worker_main and reload from disk: an out-of-scope write is a failed delegation with the path listed; declared-only edits pass; a non-git cwd is undetermined (failed), not a pass; a secret env var never reaches the child nor appears in any persisted artifact; --pass-env opts one back in; a v2 record still loads. --- src/crossagent/cli.py | 30 +++++ src/crossagent/worker.py | 101 +++++++++++++-- tests/test_job_cli.py | 50 ++++++++ tests/test_worker.py | 257 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 430 insertions(+), 8 deletions(-) diff --git a/src/crossagent/cli.py b/src/crossagent/cli.py index da4be48..1f0a7ae 100644 --- a/src/crossagent/cli.py +++ b/src/crossagent/cli.py @@ -361,6 +361,32 @@ def _parse_job_args(subcommand: str, argv: list[str]) -> argparse.Namespace: "failed (default 600)." ), ) + parser.add_argument( + "--allow-path", + action="append", + default=None, + dest="allow_path", + help=( + "Declare a path (file or directory subtree) the delegate is " + "allowed to modify. Repeatable. When given, crossagent asserts " + "after the run that the delegate touched nothing outside the " + "declared set (compared by resolved real location, so symlink/.. " + "escapes cannot defeat it) and fails the delegation otherwise. " + "Omit to leave scope enforcement off." + ), + ) + parser.add_argument( + "--pass-env", + action="append", + default=[], + dest="pass_env", + help=( + "Name of an environment variable to pass through to the delegate " + "even though it matches a credential pattern (e.g. the advisor's " + "own API key). Repeatable. By default all credential-bearing env " + "vars are withheld from the delegate." + ), + ) parser.add_argument("--json", action="store_true") parser.set_defaults(stream=True) elif subcommand == "wait": @@ -869,6 +895,10 @@ def _write_command_info( "check_timeout": getattr( args, "check_timeout", check_mod.CHECK_DEFAULT_TIMEOUT_SECONDS ), + # Delegation security posture (S4). ``scope_paths`` is ``None`` when no + # --allow-path was given (enforcement off), distinct from an empty list. + "scope_paths": getattr(args, "allow_path", None), + "pass_env": getattr(args, "pass_env", []), } jobs_mod.atomic_json_write(info, job_dir / "command.json") diff --git a/src/crossagent/worker.py b/src/crossagent/worker.py index 26bbcff..079a3df 100644 --- a/src/crossagent/worker.py +++ b/src/crossagent/worker.py @@ -18,10 +18,12 @@ from typing import Any, Optional from . import check as check_mod +from . import credentials as credentials_mod from . import jobs as jobs_mod from . import parsers as parsers_mod from . import registry as reg_mod from . import runner as runner_mod +from . import scope as scope_mod # --------------------------------------------------------------------------- @@ -42,6 +44,12 @@ class _JobCommand: advisor: str check: Optional[str] check_timeout: float + # Delegation security posture (slice S4). ``scope_paths`` is the declared + # allowlist — ``None`` means no scope was declared (enforcement off), which + # stays distinct from a declared-and-empty list. ``pass_env`` names the + # credential env vars the caller opted to pass through to the delegate. + scope_paths: Optional[list[str]] + pass_env: list[str] # --------------------------------------------------------------------------- @@ -133,7 +141,28 @@ def _should_cancel() -> bool: last_event="worker.started", ) - advisor_env = build_advisor_env(job, state_dir) + # Capture the pre-delegation working-tree baseline BEFORE the delegate runs, + # so a tree that was already dirty is not later misattributed to it (S4). + # Only when a scope was declared — otherwise enforcement is off entirely. + scope_baseline = ( + scope_mod.capture_baseline(command.cwd) + if command.scope_paths is not None + else None + ) + + # Credential-bearing env vars are withheld from the delegate by default + # (S4). Record only the NAMES withheld (never values) for the audit trail; + # both the launch env and this record use the same predicate, so they agree. + withheld = credentials_mod.withheld_names(os.environ, command.pass_env) + advisor_env = build_advisor_env(job, state_dir, pass_env=command.pass_env) + if withheld: + jobs_mod.append_event( + job_dir, + "env_scrub", + actor="system:security", + withheld=withheld, + count=len(withheld), + ) try: outcome = runner_mod.run( @@ -200,6 +229,11 @@ def _should_cancel() -> bool: # the delegation is *unverified*, never silently a pass (D5). check_result = _run_configured_check(command, job_dir) + # Run the diff-scope assertion (S4) if an allowlist was declared. It fails + # closed: a violation OR an inability to determine what changed is recorded + # distinctly and drives the delegation verdict to failed, never a pass. + scope_result = _run_scope_assertion(command, scope_baseline, job_dir) + now = datetime.now(timezone.utc).isoformat() # Persist the advisor telemetry the parser extracted (S1) and the check-gate # outcome (S3) on the SAME terminal transition — the worker is the only @@ -221,6 +255,8 @@ def _should_cancel() -> bool: cost_source=parsed.cost_source, model_reported=parsed.model_reported, check_result=check_result, + scope_result=scope_result, + withheld_env=withheld, ) return 0 @@ -255,6 +291,33 @@ def _run_configured_check( return outcome.to_dict() +def _run_scope_assertion( + command: _JobCommand, + baseline: Optional[scope_mod.ScopeBaseline], + job_dir: Path, +) -> Optional[jobs_mod.ScopeResultDict]: + """Assert the delegate's writes against the declared allowlist (S4). + + Returns ``None`` when no scope was declared (enforcement off) — kept + distinct from a declared scope that passed (D7). Otherwise the outcome is + audited (status + offending paths, which are file paths, never secrets) and + persisted. ``assert_scope`` never raises, so this can never crash the worker + nor degrade a scope failure into a silent pass. + """ + if command.scope_paths is None or baseline is None: + return None + outcome = scope_mod.assert_scope(baseline, command.scope_paths, command.cwd) + jobs_mod.append_event( + job_dir, + "scope", + actor="system:scope", + status=outcome.status, + declared=list(outcome.declared), + violating_paths=list(outcome.violating_paths), + ) + return outcome.to_dict() + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -277,9 +340,22 @@ def _load_command(job_dir: Path) -> _JobCommand: check_timeout=float( data.get("check_timeout", check_mod.CHECK_DEFAULT_TIMEOUT_SECONDS) ), + scope_paths=_load_scope_paths(data.get("scope_paths")), + pass_env=[str(name) for name in data.get("pass_env", [])], ) +def _load_scope_paths(raw: Any) -> Optional[list[str]]: + """Return the declared allowlist, or ``None`` when no scope was declared. + + Only a JSON list is a declaration; anything else (absent key, ``null``) + means enforcement is off — kept distinct from a declared empty list. + """ + if not isinstance(raw, list): + return None + return [str(pattern) for pattern in raw] + + def _append_prompt(cmd: list[str], delivery: str, prompt: str) -> None: if delivery == "dashdash": cmd.extend(["--", prompt]) @@ -301,19 +377,28 @@ def _chmod_private(path: Path) -> None: # --------------------------------------------------------------------------- -def build_advisor_env(job: jobs_mod.Job, state_root: Path) -> dict[str, str]: +def build_advisor_env( + job: jobs_mod.Job, + state_root: Path, + *, + pass_env: Optional[list[str]] = None, +) -> dict[str, str]: """Build the environment dict for the advisor subprocess with lineage vars. - Starts from ``os.environ.copy()`` and overwrites (does not setdefault) the - ``CROSSAGENT_PARENT_JOB_ID``, ``CROSSAGENT_TRACE_ID``, - ``CROSSAGENT_ORCHESTRATOR_LABEL``, ``CROSSAGENT_NESTING_DEPTH``, - and ``CROSSAGENT_STATE_DIR`` variables so a nested ``crossagent start`` - inside the advisor inherits the correct lineage. + Starts from ``os.environ.copy()``, then withholds credential-bearing + variables from the delegate (slice S4) — a delegate is partially untrusted + and must not receive the caller's ambient secrets. Variables named in + *pass_env* are the caller's explicit opt-in exceptions (e.g. the advisor's + own API key). Lineage variables are set AFTER the scrub so they are never + stripped: it overwrites (does not setdefault) ``CROSSAGENT_PARENT_JOB_ID``, + ``CROSSAGENT_TRACE_ID``, ``CROSSAGENT_ORCHESTRATOR_LABEL``, + ``CROSSAGENT_NESTING_DEPTH``, and ``CROSSAGENT_STATE_DIR`` so a nested + ``crossagent start`` inside the advisor inherits the correct lineage. This function is deliberately side-effect-free and testable without spawning a process. """ - env = os.environ.copy() + env = credentials_mod.scrub_env(os.environ, pass_through=pass_env or []) env["CROSSAGENT_PARENT_JOB_ID"] = job.job_id env["CROSSAGENT_TRACE_ID"] = job.trace_id or "" env["CROSSAGENT_ORCHESTRATOR_LABEL"] = job.orchestrator_label or "" diff --git a/tests/test_job_cli.py b/tests/test_job_cli.py index 3f9b064..d40cdf1 100644 --- a/tests/test_job_cli.py +++ b/tests/test_job_cli.py @@ -949,6 +949,56 @@ def test_start_without_check_records_none(state_dir, fake_codex_in_path, capsys) assert jobs_mod.delegation_verdict(job) == "unverified" +def test_start_writes_scope_paths_and_pass_env_into_command_json( + state_dir, fake_codex_in_path, capsys +): + code = main( + [ + "start", + "--agent", + "codex", + "--prompt", + "hi", + "--allow-path", + "src", + "--allow-path", + "tests", + "--pass-env", + "ANTHROPIC_API_KEY", + "--json", + ] + ) + captured = capsys.readouterr() + assert code == 0, captured.err + job_id = json.loads(captured.out)["job_id"] + _wait_for_terminal(job_id) + + command_path = ( + jobs_mod.job_dir_path(jobs_mod.default_state_root(), job_id) / "command.json" + ) + info = json.loads(command_path.read_text(encoding="utf-8")) + assert info["scope_paths"] == ["src", "tests"] + assert info["pass_env"] == ["ANTHROPIC_API_KEY"] + + +def test_start_without_allow_path_records_none_scope( + state_dir, fake_codex_in_path, capsys +): + """No --allow-path -> scope_paths null (enforcement off), distinct from [].""" + code = main(["start", "--agent", "codex", "--prompt", "hi", "--json"]) + captured = capsys.readouterr() + assert code == 0, captured.err + job_id = json.loads(captured.out)["job_id"] + _wait_for_terminal(job_id) + + command_path = ( + jobs_mod.job_dir_path(jobs_mod.default_state_root(), job_id) / "command.json" + ) + info = json.loads(command_path.read_text(encoding="utf-8")) + assert info["scope_paths"] is None + assert info["pass_env"] == [] + + def test_require_complete_fails_on_failed_check(state_dir, fake_codex_in_path, capsys): """A delegate that exits 0 but whose check fails must NOT pass the --require-complete gate (D5).""" diff --git a/tests/test_worker.py b/tests/test_worker.py index 5e1e930..74600a1 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import sys import textwrap from datetime import datetime, timezone @@ -129,6 +130,7 @@ def _run_job_through_worker( job_id: str = "job_e2e", check: str | None = None, check_timeout: float = 30.0, + pass_env: list[str] | None = None, ) -> Job: """Set up a job whose advisor is a fake claude-stream script, run the worker synchronously, and return the Job reloaded from disk. @@ -168,6 +170,8 @@ def _run_job_through_worker( if check is not None: command_info["check"] = check command_info["check_timeout"] = check_timeout + if pass_env is not None: + command_info["pass_env"] = pass_env jobs_mod.atomic_json_write(command_info, job_dir / "command.json") exit_code = worker_main(job_id, state_dir) @@ -378,3 +382,256 @@ def test_worker_check_logs_command_and_code_but_not_output(tmp_path): assert len(check_events) == 1 assert check_events[0]["exit_code"] == 2 assert "secret-in-output" not in json.dumps(check_events[0]) + + +# ========================================================================= +# End-to-end delegation security posture (slice S4): drive a real job through +# worker_main and reload from disk. Unit-green is not working — these prove the +# worker actually forwards scope_result/withheld_env onto the terminal record +# and that credential env never reaches the child or any persisted artifact. +# ========================================================================= + +import subprocess # noqa: E402 + +import pytest # noqa: E402 + +from crossagent import scope as scope_mod # noqa: E402 + + +def _git(args, cwd): + subprocess.run( + ["git", *args], cwd=str(cwd), check=True, capture_output=True, text=True + ) + + +def _run_scope_job( + tmp_path: Path, + advisor_body: str, + scope_paths, + *, + git_init: bool = True, + pre_commit=None, + job_id: str = "job_scope", +) -> Job: + """Run a real job whose cwd is a git repo SEPARATE from the state dir, so + crossagent's own state files are never seen as delegate edits.""" + repo = tmp_path / "repo" + repo.mkdir() + if git_init: + _git(["init"], repo) + _git(["config", "user.email", "t@e.com"], repo) + _git(["config", "user.name", "T"], repo) + for rel, content in pre_commit or []: + target = repo / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + _git(["add", rel], repo) + _git(["commit", "-m", f"add {rel}"], repo) + + state_dir = tmp_path / "state" + job_dir = jobs_mod.create_job_dir(state_dir, job_id) + fake_advisor = tmp_path / "fake_advisor.py" # OUTSIDE the repo + fake_advisor.write_text(textwrap.dedent(advisor_body), encoding="utf-8") + + now = datetime.now(timezone.utc).isoformat() + jobs_mod.save_state( + job_dir, + Job( + job_id=job_id, + status=JobState.PENDING, + advisor="claude", + cwd=str(repo), + started_at=now, + updated_at=now, + ), + ) + (job_dir / "prompt").write_text("hello", encoding="utf-8") + command_info = { + "command": [sys.executable, str(fake_advisor)], + "prompt_delivery": "positional", + "cwd": str(repo), + "result_parser": "claude-stream", + "registry_path": str(tmp_path / "sessions.json"), + "key": "", + "name": None, + "model": "", + "advisor": "claude", + "scope_paths": scope_paths, + } + jobs_mod.atomic_json_write(command_info, job_dir / "command.json") + + assert worker_main(job_id, state_dir) == 0 + return jobs_mod.load_state(job_dir) + + +# A fake advisor that writes *rel* under its cwd, then emits a clean result. +def _writer_advisor(rel: str) -> str: + return f""" + import json, os + os.makedirs(os.path.dirname({rel!r}) or '.', exist_ok=True) + with open({rel!r}, 'w') as handle: + handle.write('delegate wrote this\\n') + print(json.dumps({{"type": "result", "subtype": "success", "result": "ok"}})) + """ + + +def test_worker_scope_violation_is_failed_delegation_end_to_end(tmp_path): + """A delegate that writes outside the declared allowlist yields a violated + scope_result — persisted on disk — and a failed verdict, no green badge.""" + job = _run_scope_job( + tmp_path, + _writer_advisor("evil.py"), + scope_paths=["src"], + ) + assert job.status == JobState.SUCCEEDED # the delegate DID finish + assert job.scope_result is not None + assert job.scope_result["status"] == "violated" + assert "evil.py" in job.scope_result["violating_paths"] + assert jobs_mod.delegation_verdict(job) == "failed" + + +def test_worker_scope_in_bounds_passes_end_to_end(tmp_path): + """A delegate that edits only a declared path yields scope ok, persisted.""" + job = _run_scope_job( + tmp_path, + _writer_advisor("src/app.py"), + scope_paths=["src"], + pre_commit=[("src/app.py", "clean\n")], + ) + assert job.status == JobState.SUCCEEDED + assert job.scope_result is not None + assert job.scope_result["status"] == "ok" + assert job.scope_result["violating_paths"] == [] + # No check declared -> unverified (scope ok does not make it verified). + assert jobs_mod.delegation_verdict(job) == "unverified" + + +def test_worker_non_git_cwd_scope_is_undetermined_not_pass(tmp_path): + """A declared scope in a non-git cwd is undetermined (fail closed) and fails + the delegation, never silently passes.""" + job = _run_scope_job( + tmp_path, + _writer_advisor("evil.py"), + scope_paths=["src"], + git_init=False, + ) + assert job.scope_result is not None + assert job.scope_result["status"] == "undetermined" + assert jobs_mod.delegation_verdict(job) == "failed" + + +def test_worker_no_scope_declared_leaves_scope_result_none(tmp_path): + """With no allowlist declared, scope enforcement is off: scope_result stays + None (distinct from a satisfied scope) and the pre-S4 verdict is unchanged.""" + job = _run_scope_job( + tmp_path, + _writer_advisor("anything.py"), + scope_paths=None, + ) + assert job.scope_result is None + assert jobs_mod.delegation_verdict(job) == "unverified" + + +def test_worker_scope_audit_event_records_status_and_paths(tmp_path): + _run_scope_job( + tmp_path, + _writer_advisor("evil.py"), + scope_paths=["src"], + ) + events_path = tmp_path / "state" / "job_scope" / "events.jsonl" + lines = [ + json.loads(line) + for line in events_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + scope_events = [event for event in lines if event.get("event") == "scope"] + assert len(scope_events) == 1 + assert scope_events[0]["status"] == "violated" + assert "evil.py" in scope_events[0]["violating_paths"] + + +# --- No secret propagation ----------------------------------------------- + +_SECRET_NAME = "AWS_SECRET_ACCESS_KEY" +_SECRET_VALUE = "crossagent-super-secret-value-xyz" + +# A fake advisor that records whether it received the secret env var. +_ENV_PROBE_ADVISOR = f""" + import json, os + with open('env_probe.txt', 'w') as handle: + handle.write(os.environ.get({_SECRET_NAME!r}, 'ABSENT')) + print(json.dumps({{"type": "result", "subtype": "success", "result": "ok"}})) +""" + + +def test_worker_secret_env_does_not_reach_child(tmp_path, monkeypatch): + """A credential-bearing env var is withheld from the delegate child.""" + monkeypatch.setenv(_SECRET_NAME, _SECRET_VALUE) + _run_job_through_worker(tmp_path, _ENV_PROBE_ADVISOR) + probe = (tmp_path / "env_probe.txt").read_text(encoding="utf-8") + assert probe == "ABSENT" + + +def test_worker_pass_env_opt_in_reaches_child(tmp_path, monkeypatch): + """An explicitly passed-through credential var DOES reach the delegate.""" + monkeypatch.setenv(_SECRET_NAME, _SECRET_VALUE) + job = _run_job_through_worker(tmp_path, _ENV_PROBE_ADVISOR, pass_env=[_SECRET_NAME]) + probe = (tmp_path / "env_probe.txt").read_text(encoding="utf-8") + assert probe == _SECRET_VALUE + assert job.withheld_env is not None + assert _SECRET_NAME not in job.withheld_env + + +def test_worker_secret_value_absent_from_all_persisted_artifacts(tmp_path, monkeypatch): + """The secret VALUE must not appear in state.json, command.json, events.jsonl + or the redacted command; only the NAME is recorded (names are not secrets).""" + monkeypatch.setenv(_SECRET_NAME, _SECRET_VALUE) + job = _run_job_through_worker(tmp_path, _ENV_PROBE_ADVISOR) + + assert job.withheld_env is not None + assert _SECRET_NAME in job.withheld_env # name recorded... + job_dir = tmp_path / "state" / "job_e2e" + for artifact in ("state.json", "command.json", "events.jsonl"): + text = (job_dir / artifact).read_text(encoding="utf-8") + assert _SECRET_VALUE not in text, artifact # ...but never the value + assert _SECRET_VALUE not in job.redacted_command + + +def test_worker_env_scrub_event_records_name_not_value(tmp_path, monkeypatch): + monkeypatch.setenv(_SECRET_NAME, _SECRET_VALUE) + _run_job_through_worker(tmp_path, _ENV_PROBE_ADVISOR) + events_path = tmp_path / "state" / "job_e2e" / "events.jsonl" + lines = [ + json.loads(line) + for line in events_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + scrub_events = [event for event in lines if event.get("event") == "env_scrub"] + assert len(scrub_events) == 1 + assert _SECRET_NAME in scrub_events[0]["withheld"] + assert _SECRET_VALUE not in json.dumps(scrub_events[0]) + + +def test_worker_v2_record_without_s4_fields_still_loads(tmp_path): + """A pre-S4 record on disk (no scope_result / withheld_env) loads with those + fields absent, not crashing (D7: absent stays distinguishable).""" + job_dir = jobs_mod.create_job_dir(tmp_path / "state", "job_v2") + jobs_mod.atomic_json_write( + { + "schema_version": 2, + "job_id": "job_v2", + "status": "succeeded", + "advisor": "claude", + }, + job_dir / "state.json", + ) + loaded = jobs_mod.load_state(job_dir) + assert loaded.scope_result is None + assert loaded.withheld_env is None + assert jobs_mod.delegation_verdict(loaded) == "unverified" + + +def test_scope_module_importable_without_error(): + # Guard: the module and its git timeout constant are wired. + assert scope_mod._GIT_TIMEOUT_SECONDS > 0 + assert pytest is not None From 4ca08c58f5f74b106a39051b37f037f5dc80c8c7 Mon Sep 17 00:00:00 2001 From: Dat Date: Sun, 26 Jul 2026 17:07:50 +0700 Subject: [PATCH 05/15] feat(jobs): fold independent-verification gate into the delegation verdict (S5) Add VerifyResultDict + Job.verify_result (schema v3, None = not requested, distinct from a ran-and-failed verify per D7) and refactor delegation_verdict into a gate combiner: any failed gate blocks the green path, a passing gate greens it, an inconclusive/absent gate is neutral. A structured verify 'fail' therefore blocks green even when the shell check passed (D6); a prose-only or errored verifier degrades to inconclusive, never a pass (D4). Surface verify_result in runtime_status. --- src/crossagent/jobs.py | 96 ++++++++++++++++++++++++++++---- tests/test_jobs.py | 121 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 207 insertions(+), 10 deletions(-) diff --git a/src/crossagent/jobs.py b/src/crossagent/jobs.py index a7fdb6e..358efe9 100644 --- a/src/crossagent/jobs.py +++ b/src/crossagent/jobs.py @@ -71,6 +71,42 @@ class ScopeResultDict(TypedDict): detail: str +# Outcome of the independent verification pass (slice S5). A FRESH peer session +# grades the delegate's artifact supplied as user-turn input (D6), which removes +# the implicit-authorship channel that weakens self-grading — it does NOT claim +# to eliminate self-preference bias, which is a separate documented effect. +# ``pass`` — the verifier returned a machine-checkable verdict of correct. +# ``fail`` — the verifier returned a machine-checkable verdict of wrong. +# ``unverified`` — the verifier ran but produced no machine-checkable verdict +# (free prose, or the advisor lacks a structured-output +# contract). Treated as inconclusive, never as a pass (D4). +# ``error`` — the verifier could not run (no artifact, launch failure). +# ``unverified``/``error`` are inconclusive: they never green a delegation and +# never hard-fail it. Only ``fail`` blocks the green path. +VerifyVerdict = Literal["pass", "fail", "unverified", "error"] + + +class VerifyResultDict(TypedDict): + """Persisted outcome of the independent verification pass (slice S5). + + ``advisor``/``model`` identify the FRESH peer session that graded the work. + ``verdict`` is the deterministic pass/fail/inconclusive outcome; ``structured`` + records whether a machine-checkable contract (e.g. Claude ``--json-schema`` → + ``structured_output``) produced it, versus a JSON object parsed out of a prose + answer. ``detail`` is a human-readable summary (never the raw artifact). + + A ``Job.verify_result`` of ``None`` means *no verification was requested* — + structurally distinct from a verification that ran and failed, or ran and + could not decide (D7: absent is never the same as ``fail`` or ``unverified``). + """ + + advisor: str + model: Optional[str] + verdict: VerifyVerdict + structured: bool + detail: str + + # The delegation verdict keeps "the delegate finished" separate from "the work # was verified" (D5). See :func:`delegation_verdict`. DelegationVerdict = Literal["verified", "failed", "unverified", "incomplete"] @@ -228,6 +264,13 @@ class Job: # advisor child. ``None`` means the scrub did not run (a pre-S4 record); an # empty list means it ran and withheld nothing (D7: absent != empty). withheld_env: Optional[list[str]] = None + # --- Independent verification pass (schema v3, slice S5) ------------- + # ``verify_result`` is the outcome of grading the delegate's artifact in a + # FRESH peer session (D6). ``None`` means no verification was requested — + # distinct from a verification that ran and failed or could not decide (D7). + # A ``verdict`` of ``fail`` blocks the green path; ``pass`` can green it; + # ``unverified``/``error`` are inconclusive. See :func:`delegation_verdict`. + verify_result: Optional[VerifyResultDict] = None def delegation_verdict(job: Job) -> DelegationVerdict: @@ -244,25 +287,57 @@ def delegation_verdict(job: Job) -> DelegationVerdict: is NOT a pass: a missing gate is never green. - ``verified`` — the delegate finished cleanly and the check exited 0. - Slice S4 folds the diff-scope assertion into this same verdict rather than - adding a fifth state: a delegate that wrote outside its declared allowlist, - or whose adherence to that allowlist could not be determined, has not - produced trustworthy work — that is semantically a *failed* delegation, so - ``scope_result.status`` other than ``ok`` yields ``failed`` (fail closed). - When no scope was declared (``scope_result is None``) this gate is skipped, - preserving the pre-S4 four-state behaviour and its tests unchanged. + Slices S4 and S5 fold their gates into this same verdict rather than adding + new states. Every gate is combined the same way: **any failed gate blocks + the green path** (fail closed), at least one *passed* gate with no failures + greens the delegation, and a delegation with no decisive gate is + ``unverified`` — a missing gate is never a pass. + + Per-gate mapping (a gate that was not requested contributes nothing): + + - **Scope (S4, security gate):** ``violated``/``undetermined`` → fail (a + write outside the allowlist, or an inability to tell what changed, is never + trustworthy). ``ok`` is neutral — an in-bounds scope does not by itself + verify the *work*, so scope alone never greens a delegation. + - **Check (S3):** exit ``0`` → pass; any non-zero → fail. + - **Verify (S5):** ``pass`` → pass; ``fail`` → fail; ``unverified``/``error`` + → neutral (inconclusive; graceful degradation per D4 — a verifier that + could only produce prose, or could not run, never greens and never + hard-fails). + + A failing independent verification therefore blocks green even when the + shell check passed — which is the entire point of the fresh-session verifier + (D6). When no gate was declared this reduces to the pre-S3 behaviour and its + tests are unchanged. """ if not is_terminal(job.status): return "incomplete" if job.status != JobState.SUCCEEDED: return "failed" + + any_pass = False + scope = job.scope_result if scope is not None and scope.get("status") != "ok": return "failed" + check = job.check_result - if check is None: - return "unverified" - return "verified" if check.get("exit_code") == 0 else "failed" + if check is not None: + if check.get("exit_code") == 0: + any_pass = True + else: + return "failed" + + verify = job.verify_result + if verify is not None: + verdict = verify.get("verdict") + if verdict == "pass": + any_pass = True + elif verdict == "fail": + return "failed" + # "unverified"/"error" are inconclusive: neither green nor a hard fail. + + return "verified" if any_pass else "unverified" # --------------------------------------------------------------------------- @@ -594,6 +669,7 @@ def runtime_status(job: Job) -> dict[str, Any]: "check_result": job.check_result, "scope_result": job.scope_result, "withheld_env": job.withheld_env, + "verify_result": job.verify_result, "delegation_verdict": delegation_verdict(job), } diff --git a/tests/test_jobs.py b/tests/test_jobs.py index d4b1b87..27460d0 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -1839,3 +1839,124 @@ def test_v3_scope_result_round_trips_on_disk(tmp_path): loaded = load_state(job_dir) assert loaded.scope_result == _scope("violated", violating=["evil.py"]) assert loaded.withheld_env == ["GITHUB_TOKEN"] + + +# ========================================================================= +# Delegation verdict interaction with the independent verification pass +# (slice S5). A fresh-session verifier grades the delegate's artifact; a +# structured "fail" blocks green even when the shell check passed (D6), +# while a prose-only "unverified" degrades gracefully and never greens. +# ========================================================================= + + +def _verify(verdict, *, structured=True): + return { + "advisor": "codex", + "model": "gpt-5.6-sol", + "verdict": verdict, + "structured": structured, + "detail": "", + } + + +def _succeeded_verify(verify_verdict=None, *, check_exit=None, scope_status=None): + check = ( + None + if check_exit is None + else { + "command": "pytest", + "exit_code": check_exit, + "stdout_tail": "", + "stderr_tail": "", + } + ) + return Job( + job_id="job_v", + status=JobState.SUCCEEDED, + check_result=check, + scope_result=None if scope_status is None else _scope(scope_status), + verify_result=None if verify_verdict is None else _verify(verify_verdict), + ) + + +def test_delegation_verdict_verified_when_verification_passes(): + """A structured pass from the fresh verifier greens the delegation even with + no shell check declared.""" + assert delegation_verdict(_succeeded_verify("pass")) == "verified" + + +def test_delegation_verdict_failed_when_verification_fails(): + assert delegation_verdict(_succeeded_verify("fail")) == "failed" + + +def test_delegation_verdict_verify_fail_blocks_green_despite_passing_check(): + """The load-bearing D6 case: the shell check passed, but the independent + verifier says the work is wrong -> failed. A failing verification blocks the + green path a same-context self-grade might have waved through.""" + job = _succeeded_verify("fail", check_exit=0) + assert delegation_verdict(job) == "failed" + + +def test_delegation_verdict_prose_verification_is_unverified_not_pass(): + """An advisor that produced no machine-checkable verdict (prose) degrades to + 'unverified' and never greens: inconclusive is not a pass (D4).""" + job = _succeeded_verify("unverified", check_exit=None) + assert delegation_verdict(job) == "unverified" + + +def test_delegation_verdict_errored_verification_is_inconclusive(): + """A verifier that could not run leaves the delegation unverified, never a + hard fail (the verifier broke, not the work).""" + assert delegation_verdict(_succeeded_verify("error")) == "unverified" + + +def test_delegation_verdict_prose_verify_does_not_downgrade_passing_check(): + """A passing shell check greens the delegation; an inconclusive verify does + not drag it back to unverified.""" + job = _succeeded_verify("unverified", check_exit=0) + assert delegation_verdict(job) == "verified" + + +def test_delegation_verdict_scope_violation_dominates_passing_verification(): + """Fail closed still wins: a scope violation fails the delegation even when + the independent verifier passed.""" + job = _succeeded_verify("pass", scope_status="violated") + assert delegation_verdict(job) == "failed" + + +def test_delegation_verdict_absent_verify_preserves_pre_s5_behaviour(): + job = Job(job_id="job_v", status=JobState.SUCCEEDED, verify_result=None) + assert delegation_verdict(job) == "unverified" + + +def test_runtime_status_surfaces_verify_result(): + job = _succeeded_verify("fail", check_exit=0) + result = runtime_status(job) + assert result["verify_result"]["verdict"] == "fail" + assert result["delegation_verdict"] == "failed" + + +def test_v3_verify_result_round_trips_on_disk(tmp_path): + job_dir = create_job_dir(tmp_path, "job_verify_rt") + job = Job( + job_id="job_verify_rt", + status=JobState.SUCCEEDED, + verify_result=_verify("pass", structured=True), + ) + save_state(job_dir, job) + loaded = load_state(job_dir) + assert loaded.verify_result == _verify("pass", structured=True) + assert delegation_verdict(loaded) == "verified" + + +def test_v2_record_without_verify_result_still_loads(tmp_path): + """A pre-S5 record on disk (no verify_result key) loads with it absent, not + crashing — absent (not requested) stays distinct from failed (D7).""" + job_dir = create_job_dir(tmp_path, "job_v2_verify") + atomic_json_write( + {"schema_version": 2, "job_id": "job_v2_verify", "status": "succeeded"}, + job_dir / "state.json", + ) + loaded = load_state(job_dir) + assert loaded.verify_result is None + assert delegation_verdict(loaded) == "unverified" From 7558cca1f955c34d88418af444411e12131fab8b Mon Sep 17 00:00:00 2001 From: Dat Date: Sun, 26 Jul 2026 17:07:58 +0700 Subject: [PATCH 06/15] feat(verify): fresh independent verification pass with graceful degradation (S5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spawn a FRESH peer session that grades the delegate's artifact supplied as user-turn input (D6) — build_verifier_command adds no resume/fork/name flag and never reads the session registry, removing the implicit-authorship channel that weakens self-grading. Prefer a machine-checkable contract where the advisor exposes one (Claude --json-schema -> structured_output; new Advisor.json_schema_flag), else parse a JSON verdict out of the answer; free prose -> unverified, never a pass. The verifier is a delegate too: credentials scrubbed and CROSSAGENT_* lineage stripped. Never raises — a broken verifier degrades to error/unverified. Scope-honest: removes the implicit-authorship channel; does NOT claim to eliminate self-preference bias (a separate documented effect), and the source paper did not study many-turn agentic settings — crossagent's own setting. --- src/crossagent/advisors.py | 8 + src/crossagent/verify.py | 366 +++++++++++++++++++++++++++++++++++++ tests/test_verify.py | 274 +++++++++++++++++++++++++++ 3 files changed, 648 insertions(+) create mode 100644 src/crossagent/verify.py create mode 100644 tests/test_verify.py diff --git a/src/crossagent/advisors.py b/src/crossagent/advisors.py index ad27043..18926b5 100644 --- a/src/crossagent/advisors.py +++ b/src/crossagent/advisors.py @@ -46,6 +46,13 @@ class Advisor: result_parser: str = "text" resume_command: tuple[str, ...] | None = None session_event_field: str | None = None + # Flag that requests a machine-checkable JSON output contract for the + # independent verification pass (slice S5). ``None`` means the advisor has no + # such contract, so a verifier built on it degrades to parsing a JSON verdict + # out of the answer text (D4 graceful degradation — never a hard failure). + # Claude exposes ``--json-schema`` (research finding [6]: the payload lands in + # ``structured_output``); no other built-in advisor has a verified equivalent. + json_schema_flag: str | None = None experimental: bool = False notes: str = "" @@ -84,6 +91,7 @@ def supports_stream(self) -> bool: session_name_flag="--name", fork_flag="--fork-session", result_parser="claude-stream", + json_schema_flag="--json-schema", ), "codex": Advisor( name="codex", diff --git a/src/crossagent/verify.py b/src/crossagent/verify.py new file mode 100644 index 0000000..7f28184 --- /dev/null +++ b/src/crossagent/verify.py @@ -0,0 +1,366 @@ +"""Independent verification pass for delegated work (slice S5). + +Research finding [5]: asking the producing agent to grade its own work is a +measurably weaker gate than an independent check, and the degradation is +triggered by *implicit* authorship — the artifact sitting in the model's own +prior/current turn — not by being told it authored the work. Decision D6 follows +from that mechanism: the verifier must be a **FRESH peer session** with the +artifact supplied as **user-turn input**. A resumed session, or one where the +artifact arrives as prior assistant context, reintroduces exactly the +implicit-authorship channel this pass exists to remove. + +Scope honesty: supplying the artifact as user-turn input to a fresh session +*removes the implicit-authorship channel*. It does **not** eliminate +self-preference bias — residual self-recognition preference is a separate +documented effect — and the paper behind this design explicitly did not study +many-turn agentic settings, which is precisely crossagent's setting. Treat the +verdict as a stronger-but-not-infallible gate, not a proof of correctness. + +How freshness is guaranteed +--------------------------- +:func:`build_verifier_command` builds the advisor argv itself and never adds a +resume, fork, or session-name flag and never consults the session registry, so +there is no channel by which a prior conversation (the delegate's own, or any +other) can be attached. The artifact is appended as the advisor's ordinary +prompt argument — which every supported CLI treats as a user turn — so it can +never arrive as assistant context. Both properties are asserted by the tests. + +Machine-checkable contract, with graceful degradation (D4) +---------------------------------------------------------- +Where the advisor exposes a JSON-schema contract (Claude ``--json-schema`` → +``structured_output``), the verifier requests it and reads the structured +verdict. Where it does not, the verifier still asks for a JSON verdict in the +prompt and parses one out of the answer; an answer with no parseable verdict is +recorded as ``unverified`` (free prose), never as a pass. A verifier that cannot +run at all is ``error``. Neither ``unverified`` nor ``error`` hard-fails the +delegation — only a machine-readable ``fail`` blocks the green path. + +Security: the verifier is a delegate too. It runs with credential-bearing env +vars scrubbed (the caller's ``--pass-env`` opt-ins excepted) and with +crossagent's own ``CROSSAGENT_*`` lineage variables stripped, so a spawned +verifier can neither inherit ambient secrets nor silently attach to a job tree. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +from dataclasses import dataclass +from typing import Any, Optional + +from . import credentials as credentials_mod +from . import parsers as parsers_mod +from . import runner as runner_mod +from .advisors import Advisor +from .jobs import VerifyResultDict, VerifyVerdict + +# A verification that never terminates must not hang the worker forever. +VERIFY_DEFAULT_TIMEOUT_SECONDS = 600.0 +# Keep only a bounded head of the delegate's answer/diff in the artifact — an +# LLM prompt does not need (and should not pay for) an unbounded transcript. +_ARTIFACT_MAX_CHARS = 24000 +_DIFF_MAX_CHARS = 16000 +_GIT_TIMEOUT_SECONDS = 30.0 +_LINEAGE_ENV_PREFIX = "CROSSAGENT_" + +# The machine-checkable contract we ask the verifier to satisfy. Kept tiny on +# purpose: a single verdict token plus a short reason. +_VERDICT_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "verdict": {"type": "string", "enum": ["pass", "fail"]}, + "reason": {"type": "string"}, + }, + "required": ["verdict"], + "additionalProperties": False, +} + +_PROMPT_INSTRUCTIONS = ( + "You are an INDEPENDENT reviewer. You did not write the work below; judge it " + "on its merits. Decide whether the delegated work correctly and completely " + "satisfies the stated task. Do not modify any files. Respond with a single " + 'JSON object and nothing else: {"verdict": "pass" | "fail", "reason": ' + '""}. Use "pass" only if the work is correct and complete; ' + 'otherwise "fail".' +) + + +@dataclass(frozen=True) +class VerifyOutcome: + """The result of one independent verification pass.""" + + advisor: str + model: Optional[str] + verdict: VerifyVerdict + structured: bool + detail: str + + def to_dict(self) -> VerifyResultDict: + """Return the persisted-record shape for ``Job.verify_result``.""" + return { + "advisor": self.advisor, + "model": self.model, + "verdict": self.verdict, + "structured": self.structured, + "detail": self.detail, + } + + +# --------------------------------------------------------------------------- +# Fresh command construction (no session flags — this is the freshness contract) +# --------------------------------------------------------------------------- + + +def build_verifier_command( + advisor: Advisor, model: Optional[str], *, schema_path: Optional[str] = None +) -> list[str]: + """Build the verifier argv for *advisor*, WITHOUT the prompt. + + Deliberately omits every session-attachment flag (resume, fork, name) and + never reads the session registry: a fresh session is the whole point (D6). + The prompt is appended separately by :func:`_append_prompt` as a user turn. + When the advisor supports a JSON-schema contract and *schema_path* is given, + single-shot JSON output plus the schema flag are added so the structured + verdict lands in ``structured_output``. + """ + cmd = [advisor.executable, *advisor.base_args, *advisor.invoke_args] + if model and advisor.model_flag: + cmd.extend([advisor.model_flag, model]) + if advisor.supports_stream: + # Single-shot JSON (not streaming): the terminal event is the whole + # payload, which is the cleanest carrier for a structured verdict. + cmd.extend(advisor.json_args) + if advisor.json_schema_flag and schema_path is not None: + cmd.extend([advisor.json_schema_flag, schema_path]) + return cmd + + +def _append_prompt(cmd: list[str], advisor: Advisor, prompt: str) -> None: + """Append *prompt* as the advisor's user-turn argument. + + Mirrors the delivery rule the runner uses so the artifact is delivered + exactly as a normal user prompt — never as assistant/system context. + """ + delivery = advisor.prompt_delivery + if delivery == "dashdash": + cmd.extend(["--", prompt]) + elif delivery.startswith("flag:"): + cmd.extend([delivery.split(":", 1)[1], prompt]) + else: # "positional" + cmd.append(prompt) + + +# --------------------------------------------------------------------------- +# Artifact assembly (what the verifier grades — supplied as user-turn input) +# --------------------------------------------------------------------------- + + +def build_artifact(task_prompt: str, result_text: Optional[str], cwd: str) -> str: + """Assemble the artifact the verifier grades, as a user-turn string. + + Combines the original task, the delegate's answer, and — when *cwd* is a git + repo — a best-effort working-tree diff of what the delegate changed. Bounded + so the verification prompt stays a reasonable size. + """ + sections = [ + _PROMPT_INSTRUCTIONS, + "\n\n=== TASK GIVEN TO THE DELEGATE ===\n" + task_prompt.strip(), + ] + answer = (result_text or "").strip() + sections.append( + "\n\n=== DELEGATE'S ANSWER ===\n" + + (answer or "(the delegate produced no answer)") + ) + diff = _git_diff(cwd) + if diff: + sections.append("\n\n=== WORKING-TREE DIFF (git diff HEAD) ===\n" + diff) + artifact = "".join(sections) + if len(artifact) > _ARTIFACT_MAX_CHARS: + artifact = artifact[:_ARTIFACT_MAX_CHARS] + "\n...[artifact truncated]" + return artifact + + +def _git_diff(cwd: str) -> Optional[str]: + """Return a bounded ``git diff HEAD`` for *cwd*, or ``None`` on any failure. + + Best-effort only (never raises): the answer is still gradable without a diff, + and a non-repo cwd or a git error must not break verification. git is run as + an argument list with ``shell=False`` (matching ``check.py``/``scope.py``); + no delegate output is interpolated into the command line. + """ + try: + completed = subprocess.run( + ["git", "-c", "core.pager=cat", "diff", "HEAD", "--"], + cwd=cwd, + capture_output=True, + text=True, + timeout=_GIT_TIMEOUT_SECONDS, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + if completed.returncode != 0: + return None + diff = completed.stdout + if not diff.strip(): + return None + if len(diff) > _DIFF_MAX_CHARS: + diff = diff[:_DIFF_MAX_CHARS] + "\n...[diff truncated]" + return diff + + +# --------------------------------------------------------------------------- +# Verdict extraction +# --------------------------------------------------------------------------- + + +def _parse_verdict_object(text: str) -> Optional[dict[str, Any]]: + """Extract the verdict JSON object from *text*, tolerant of surrounding prose. + + Tries the whole string first, then the widest ``{...}`` span. Returns ``None`` + when nothing parses to a mapping — i.e. the answer was free prose. + """ + stripped = text.strip() + for candidate in _json_candidates(stripped): + try: + parsed = json.loads(candidate) + except (json.JSONDecodeError, ValueError): + continue + if isinstance(parsed, dict): + return parsed + return None + + +def _json_candidates(text: str) -> list[str]: + candidates = [text] + start = text.find("{") + end = text.rfind("}") + if 0 <= start < end: + candidates.append(text[start : end + 1]) + return candidates + + +def _verdict_from_object(obj: dict[str, Any]) -> tuple[VerifyVerdict, str]: + """Map a parsed verdict object to a ``(verdict, detail)`` pair.""" + raw = obj.get("verdict") + reason = obj.get("reason") + detail = str(reason) if isinstance(reason, str) and reason else "" + if isinstance(raw, bool): + return ("pass" if raw else "fail"), detail + token = str(raw).strip().lower() if raw is not None else "" + if token in ("pass", "passed", "true", "ok", "correct"): + return "pass", detail + if token in ("fail", "failed", "false", "incorrect", "reject", "rejected"): + return "fail", detail + # A JSON object with an unrecognised verdict token is inconclusive, not a + # pass — fall back to prose semantics. + return "unverified", detail or f"unrecognised verdict token: {raw!r}" + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +def verifier_env(pass_env: Optional[list[str]] = None) -> dict[str, str]: + """Return the environment for the verifier subprocess. + + Credential-bearing vars are scrubbed (the caller's ``--pass-env`` opt-ins + excepted), and crossagent's own ``CROSSAGENT_*`` lineage vars are stripped so + a nested crossagent inside the verifier cannot attach to a stale job tree. + """ + scrubbed = credentials_mod.scrub_env(os.environ, pass_through=pass_env or []) + return { + key: value + for key, value in scrubbed.items() + if not key.startswith(_LINEAGE_ENV_PREFIX) + } + + +def run_verification( + advisor: Advisor, + model: Optional[str], + artifact: str, + *, + cwd: str, + pass_env: Optional[list[str]] = None, + timeout: float = VERIFY_DEFAULT_TIMEOUT_SECONDS, +) -> VerifyOutcome: + """Run one fresh, independent verification pass and return its outcome. + + Never raises: any launch or parse failure degrades to an ``error``/ + ``unverified`` verdict — a broken verifier must never crash the worker nor + silently green a delegation. + """ + structured = advisor.json_schema_flag is not None + schema_fd, schema_path = (-1, None) + if structured: + schema_fd, schema_path = tempfile.mkstemp( + prefix="crossagent-verify-", suffix=".json" + ) + try: + if schema_path is not None: + with os.fdopen(schema_fd, "w", encoding="utf-8") as handle: + json.dump(_VERDICT_SCHEMA, handle) + cmd = build_verifier_command(advisor, model, schema_path=schema_path) + _append_prompt(cmd, advisor, artifact) + parser = parsers_mod.get_parser(advisor.result_parser) + try: + outcome = runner_mod.run( + cmd, + cwd=cwd, + env=verifier_env(pass_env), + consumer=parser, + max_runtime_seconds=timeout, + ) + except (OSError, ValueError) as exc: + return VerifyOutcome( + advisor.name, + model, + "error", + structured, + f"verifier failed to launch: {exc}", + ) + finally: + if schema_path is not None: + try: + os.unlink(schema_path) + except OSError: + pass + + parsed = ( + outcome.result + if isinstance(outcome.result, parsers_mod.ParsedResult) + else parsers_mod.ParsedResult() + ) + if outcome.timed_out: + return VerifyOutcome( + advisor.name, + model, + "error", + structured, + f"verifier timed out after {timeout:g}s", + ) + if parsed.failure or parsed.result is None: + return VerifyOutcome( + advisor.name, + model, + "error", + structured, + parsed.error or "verifier produced no answer", + ) + + obj = _parse_verdict_object(parsed.result) + if obj is None: + # Free prose with no machine-checkable verdict: inconclusive, not a pass. + return VerifyOutcome( + advisor.name, + model, + "unverified", + structured, + "verifier returned no machine-checkable verdict (free prose)", + ) + verdict, detail = _verdict_from_object(obj) + return VerifyOutcome(advisor.name, model, verdict, structured, detail) diff --git a/tests/test_verify.py b/tests/test_verify.py new file mode 100644 index 0000000..d7bdf3b --- /dev/null +++ b/tests/test_verify.py @@ -0,0 +1,274 @@ +"""Tests for the independent verification pass (slice S5).""" + +from __future__ import annotations + +import subprocess +import sys +import textwrap +from pathlib import Path + +from crossagent import verify as verify_mod +from crossagent.advisors import Advisor, resolve +from crossagent.verify import ( + VerifyOutcome, + _parse_verdict_object, + _verdict_from_object, + build_artifact, + build_verifier_command, + run_verification, + verifier_env, +) + +# --------------------------------------------------------------------------- +# Freshness contract: the verifier command must carry no session attachment. +# --------------------------------------------------------------------------- + +_SESSION_FLAGS = ("--resume", "--fork-session", "--name") + + +def test_verifier_command_has_no_session_flags(): + """A fresh session is the whole point (D6): no resume/fork/name flag may + appear, so no prior conversation can be attached to the verifier.""" + claude = resolve("claude") + cmd = build_verifier_command(claude, "sonnet", schema_path="/tmp/schema.json") + for flag in _SESSION_FLAGS: + assert flag not in cmd, flag + + +def test_verifier_command_requests_structured_output_when_supported(): + claude = resolve("claude") + cmd = build_verifier_command(claude, None, schema_path="/tmp/schema.json") + assert "--json-schema" in cmd + assert "/tmp/schema.json" in cmd + + +def test_verifier_command_omits_schema_flag_for_unsupported_advisor(): + """An advisor with no json_schema_flag never gets a schema flag — it will + degrade to prose parsing, not crash.""" + codex = resolve("codex") + cmd = build_verifier_command(codex, None, schema_path="/tmp/schema.json") + assert "--json-schema" not in cmd + + +def test_artifact_is_delivered_as_the_prompt_argument(): + """The artifact must arrive as the advisor's user-turn prompt, never as + assistant/system context. For claude (dashdash delivery) it is the final + argument after ``--``.""" + claude = resolve("claude") + cmd = build_verifier_command(claude, None) + verify_mod._append_prompt(cmd, claude, "THE-ARTIFACT") + assert cmd[-1] == "THE-ARTIFACT" + assert cmd[-2] == "--" + + +# --------------------------------------------------------------------------- +# Verdict extraction +# --------------------------------------------------------------------------- + + +def test_parse_verdict_object_from_raw_json(): + obj = _parse_verdict_object('{"verdict": "pass", "reason": "ok"}') + assert obj == {"verdict": "pass", "reason": "ok"} + + +def test_parse_verdict_object_extracts_from_surrounding_prose(): + text = 'Here is my review.\n\n{"verdict": "fail", "reason": "bug"}\n\nThanks!' + obj = _parse_verdict_object(text) + assert obj["verdict"] == "fail" + + +def test_parse_verdict_object_returns_none_for_prose(): + assert _parse_verdict_object("The code looks fine to me.") is None + + +def test_verdict_from_object_maps_tokens(): + assert _verdict_from_object({"verdict": "pass"})[0] == "pass" + assert _verdict_from_object({"verdict": "FAILED"})[0] == "fail" + assert _verdict_from_object({"verdict": True})[0] == "pass" + assert _verdict_from_object({"verdict": False})[0] == "fail" + # An unrecognised token is inconclusive, never a pass. + assert _verdict_from_object({"verdict": "maybe"})[0] == "unverified" + + +# --------------------------------------------------------------------------- +# run_verification end-to-end via a fake advisor script +# --------------------------------------------------------------------------- + + +def _fake_advisor( + tmp_path: Path, + body: str, + *, + result_parser: str = "claude-stream", + json_schema_flag: str | None = "--json-schema", +) -> Advisor: + script = tmp_path / "fake_verifier.py" + script.write_text(textwrap.dedent(body), encoding="utf-8") + return Advisor( + name="fakeverify", + executable=sys.executable, + base_args=(str(script),), + prompt_delivery="positional", + result_parser=result_parser, + json_args=(), + json_schema_flag=json_schema_flag, + ) + + +# A fake claude-style advisor: emits a structured_output verdict and records the +# prompt (its last argv) so a test can prove the artifact arrived as user input. +def _structured_advisor(tmp_path: Path, verdict: str) -> Advisor: + return _fake_advisor( + tmp_path, + f""" + import json, sys + with open('prompt_seen.txt', 'w') as handle: + handle.write(sys.argv[-1]) + print(json.dumps({{ + "type": "result", "subtype": "success", + "structured_output": {{"verdict": {verdict!r}, "reason": "because"}}, + }})) + """, + ) + + +def test_run_verification_structured_pass(tmp_path): + advisor = _structured_advisor(tmp_path, "pass") + outcome = run_verification(advisor, None, "ARTIFACT-TEXT", cwd=str(tmp_path)) + assert outcome.verdict == "pass" + assert outcome.structured is True + # The artifact reached the advisor as its user-turn prompt argument. + assert (tmp_path / "prompt_seen.txt").read_text() == "ARTIFACT-TEXT" + + +def test_run_verification_structured_fail(tmp_path): + advisor = _structured_advisor(tmp_path, "fail") + outcome = run_verification(advisor, None, "ARTIFACT", cwd=str(tmp_path)) + assert outcome.verdict == "fail" + + +def test_run_verification_prose_degrades_to_unverified(tmp_path): + """An advisor with no structured-output contract that returns prose degrades + to 'unverified' — never a pass, never a crash (D4).""" + advisor = _fake_advisor( + tmp_path, + """ + print("The delegate's work looks reasonable to me.") + """, + result_parser="text", + json_schema_flag=None, + ) + outcome = run_verification(advisor, None, "ARTIFACT", cwd=str(tmp_path)) + assert outcome.verdict == "unverified" + assert outcome.structured is False + + +def test_run_verification_prose_advisor_can_still_parse_json_answer(tmp_path): + """A text advisor that happens to answer in clean JSON yields a usable + verdict (structured=False but decisive).""" + advisor = _fake_advisor( + tmp_path, + """ + print('{"verdict": "pass", "reason": "all good"}') + """, + result_parser="text", + json_schema_flag=None, + ) + outcome = run_verification(advisor, None, "ARTIFACT", cwd=str(tmp_path)) + assert outcome.verdict == "pass" + assert outcome.structured is False + + +def test_run_verification_no_result_is_error(tmp_path): + """A verifier that emits no result event is recorded as 'error', which is + inconclusive — it must not green or hard-fail the delegation.""" + advisor = _fake_advisor(tmp_path, "pass\n") + outcome = run_verification(advisor, None, "ARTIFACT", cwd=str(tmp_path)) + assert outcome.verdict == "error" + + +def test_run_verification_never_raises_on_missing_executable(tmp_path): + advisor = Advisor( + name="ghost", + executable="/nonexistent/verifier-binary", + result_parser="text", + ) + outcome = run_verification(advisor, None, "ARTIFACT", cwd=str(tmp_path)) + assert isinstance(outcome, VerifyOutcome) + assert outcome.verdict == "error" + + +# --------------------------------------------------------------------------- +# Credential + lineage scrubbing for the verifier (it is a delegate too) +# --------------------------------------------------------------------------- + + +def test_verifier_env_scrubs_credentials(monkeypatch): + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "super-secret") + env = verifier_env() + assert "AWS_SECRET_ACCESS_KEY" not in env + + +def test_verifier_env_honours_pass_env_opt_in(monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-xxx") + env = verifier_env(["ANTHROPIC_API_KEY"]) + assert env["ANTHROPIC_API_KEY"] == "sk-xxx" + + +def test_verifier_env_strips_lineage_vars(monkeypatch): + monkeypatch.setenv("CROSSAGENT_PARENT_JOB_ID", "job_parent") + monkeypatch.setenv("CROSSAGENT_TRACE_ID", "trace_x") + env = verifier_env() + assert "CROSSAGENT_PARENT_JOB_ID" not in env + assert "CROSSAGENT_TRACE_ID" not in env + + +def test_run_verification_secret_does_not_reach_verifier(tmp_path, monkeypatch): + """End-to-end: a credential env var is withheld from the verifier child.""" + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "leak-me") + advisor = _fake_advisor( + tmp_path, + """ + import json, os + with open('env_probe.txt', 'w') as handle: + handle.write(os.environ.get('AWS_SECRET_ACCESS_KEY', 'ABSENT')) + print(json.dumps({"type": "result", "subtype": "success", + "structured_output": {"verdict": "pass"}})) + """, + ) + run_verification(advisor, None, "ARTIFACT", cwd=str(tmp_path)) + assert (tmp_path / "env_probe.txt").read_text() == "ABSENT" + + +# --------------------------------------------------------------------------- +# Artifact assembly includes a git diff when the cwd is a repo +# --------------------------------------------------------------------------- + + +def _git(args, cwd): + subprocess.run( + ["git", *args], cwd=str(cwd), check=True, capture_output=True, text=True + ) + + +def test_build_artifact_includes_git_diff(tmp_path): + repo = tmp_path / "repo" + repo.mkdir() + _git(["init"], repo) + _git(["config", "user.email", "t@e.com"], repo) + _git(["config", "user.name", "T"], repo) + (repo / "app.py").write_text("original\n", encoding="utf-8") + _git(["add", "app.py"], repo) + _git(["commit", "-m", "init"], repo) + (repo / "app.py").write_text("delegate changed this\n", encoding="utf-8") + + artifact = build_artifact("do the task", "I edited app.py", str(repo)) + assert "TASK GIVEN TO THE DELEGATE" in artifact + assert "I edited app.py" in artifact + assert "delegate changed this" in artifact # the diff is present + + +def test_build_artifact_tolerates_non_git_cwd(tmp_path): + artifact = build_artifact("do the task", "the answer", str(tmp_path)) + assert "the answer" in artifact + assert "WORKING-TREE DIFF" not in artifact From a843b5acf886e4de7f6ebb042f73bc40748a611a Mon Sep 17 00:00:00 2001 From: Dat Date: Sun, 26 Jul 2026 17:08:06 +0700 Subject: [PATCH 07/15] feat(escalate): re-dispatch failed delegations up a ladder as same-trace children (S5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a failed delegation (failing check, scope violation, or failing verification -> delegation_verdict == failed), re-dispatch the task to the next larger peer. Recorded as option (a): a same-trace child job with parent_job_id set and the trace_id preserved, resolved through resolve_lineage — exactly what the shipped analytics.py escalation-rate definition counts, so no analytics change is needed. Ladder = ordered advisor[:model] rungs; each hop hands the remainder down, and the check/verify/scope/pass_env posture is propagated so the larger peer is held to the same write boundary and credential withholding. Bounded twice over: the rung list shrinks each hop, and MAX_NESTING_DEPTH is enforced (a LineageError is caught and audited as a skipped escalation, never a crash or a runaway loop). --- src/crossagent/escalate.py | 280 +++++++++++++++++++++++++++++++++++++ tests/test_escalate.py | 266 +++++++++++++++++++++++++++++++++++ 2 files changed, 546 insertions(+) create mode 100644 src/crossagent/escalate.py create mode 100644 tests/test_escalate.py diff --git a/src/crossagent/escalate.py b/src/crossagent/escalate.py new file mode 100644 index 0000000..c4e4ddc --- /dev/null +++ b/src/crossagent/escalate.py @@ -0,0 +1,280 @@ +"""Escalate-on-failure ladder for delegated work (slice S5). + +When a delegation fails — a failing check, a scope violation, or a failing +independent verification, all of which resolve to +:func:`~crossagent.jobs.delegation_verdict` == ``"failed"`` — the caller may +declare an *escalation ladder*: an ordered list of larger peers to re-dispatch +the same task to. On failure the first rung is spawned; the remaining rungs are +handed to that child so a further failure climbs the next rung. + +Recording (option (a) — satisfies the shipped analytics definition) +------------------------------------------------------------------- +``analytics.py`` (merged before this slice) computes the escalation rate as +"a failed delegation counts as escalated if it has a **same-trace child**". So a +re-dispatch is recorded as exactly that: a child job with the failed job as its +``parent_job_id`` and the **same ``trace_id``**, resolved through the existing +:func:`~crossagent.jobs.resolve_lineage` machinery. Nothing in ``analytics.py`` +changes; the escalation column starts reflecting reality the moment this ships. + +Runaway protection +------------------ +An escalation ladder is a recursion source. It is bounded twice over: + +1. The ladder is a finite list that shrinks by one rung each hop, so it + self-terminates even if every rung fails. +2. Each child is one level deeper, and lineage resolution enforces + :data:`~crossagent.jobs.MAX_NESTING_DEPTH`; a rung that would exceed the cap + raises :class:`~crossagent.jobs.LineageError`, which is caught and recorded as + a skipped escalation rather than crashing or looping. + +Security posture carried to the child +------------------------------------- +An escalated child is a delegate too. It goes through the ordinary worker path, +so credential scrubbing (``credentials.py``) and the diff-scope assertion apply +to it unchanged: the declared ``scope_paths`` allowlist and ``pass_env`` opt-ins +are propagated so the larger peer is held to the *same* write boundary and the +*same* credential withholding as the original delegate. The check and +verification gates are propagated too, so each rung's output is judged the same +way before the ladder climbs again. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Callable, Optional + +from . import advisors as advisors_mod +from . import jobs as jobs_mod +from .advisors import Advisor +from .jobs import Job, delegation_verdict + +# Launches a detached worker for a child job. Injectable so escalation can be +# unit-tested without spawning a real process. +Launcher = Callable[[str, Path], Any] + + +def parse_rungs(rungs: Optional[list[str]]) -> list[tuple[str, Optional[str]]]: + """Parse ``advisor[:model]`` ladder rungs into ``(advisor, model)`` pairs. + + Splits on the first colon only, so a model containing no colon (the usual + case: ``opus``, ``gpt-5.6-sol``) is preserved intact. Blank entries are + dropped so a stray empty ``--escalate-to`` cannot spawn a nameless job. + """ + parsed: list[tuple[str, Optional[str]]] = [] + for raw in rungs or []: + entry = raw.strip() + if not entry: + continue + advisor_name, sep, model = entry.partition(":") + advisor_name = advisor_name.strip() + if not advisor_name: + continue + parsed.append((advisor_name, model.strip() if sep else None)) + return parsed + + +def _build_child_argv(advisor: Advisor, model: Optional[str]) -> list[str]: + """Build the escalated child's advisor argv — a FRESH delegation. + + Mirrors the advisor-invocation core of ``crossagent start`` (default stream + mode) but adds no session-attachment flag: an escalation re-dispatches the + task to a new, larger peer, so there is no prior session to resume. The + child does not inherit the parent's fine-grained invocation flags + (``--tools``, ``--safe-mode``, ``--permission-mode``); its write boundary is + enforced structurally by the propagated scope allowlist instead. + """ + cmd = [advisor.executable, *advisor.base_args, *advisor.invoke_args] + if model and advisor.model_flag: + cmd.extend([advisor.model_flag, model]) + if advisor.supports_stream: + cmd.extend(advisor.stream_args) + return cmd + + +def maybe_escalate( + failed_job: Job, + *, + prompt: str, + state_root: Path, + job_dir: Path, + cwd: str, + registry_path: str, + escalate_to: Optional[list[str]], + check: Optional[str], + check_timeout: float, + scope_paths: Optional[list[str]], + pass_env: list[str], + verify_with: Optional[str], + verify_model: Optional[str], + launcher: Optional[Launcher] = None, +) -> Optional[str]: + """Re-dispatch a *failed* delegation to the next ladder rung, if any. + + Returns the spawned child's job id, or ``None`` when nothing was escalated + (the delegation did not fail, no rungs remain, the rung advisor is unknown, + or the depth cap was reached). Never raises: an escalation that cannot be + launched is recorded and skipped, never allowed to crash the worker. + """ + if delegation_verdict(failed_job) != "failed": + return None + + rungs = parse_rungs(escalate_to) + if not rungs: + return None + advisor_name, model = rungs[0] + # ``remaining`` keeps the raw ``advisor[:model]`` strings for the child, minus + # the rung being spawned now — the ladder that a further failure will climb. + remaining = _drop_first_nonblank(escalate_to) + + try: + advisor = advisors_mod.resolve(advisor_name) + except KeyError as exc: + _audit_skip(job_dir, reason=f"unknown escalation advisor: {exc}") + return None + + child_id = jobs_mod.generate_job_id() + try: + parent_id, trace_id, label, depth = jobs_mod.resolve_lineage( + parent_flag=failed_job.job_id, + state_root=state_root, + new_job_id=child_id, + ) + except jobs_mod.LineageError as exc: + # Depth cap reached (or a corrupt chain): the ladder stops here. This is + # the runaway guard doing its job, not an error. + _audit_skip(job_dir, reason=f"escalation halted: {exc}") + return None + + child_dir = jobs_mod.create_job_dir(state_root, child_id) + _write_child_prompt(child_dir, prompt) + _write_child_command( + child_dir, + advisor=advisor, + model=model, + cwd=cwd, + registry_path=registry_path, + check=check, + check_timeout=check_timeout, + scope_paths=scope_paths, + pass_env=pass_env, + verify_with=verify_with, + verify_model=verify_model, + escalate_to=remaining, + ) + child = Job( + job_id=child_id, + status=jobs_mod.JobState.PENDING, + advisor=advisor.name, + name="", + cwd=cwd, + redacted_command="", + started_at=_now(), + updated_at=_now(), + last_activity_at=_now(), + last_event="escalation.created", + max_runtime_seconds=failed_job.max_runtime_seconds, + termination_grace_seconds=failed_job.termination_grace_seconds, + parent_job_id=parent_id, + trace_id=trace_id, + orchestrator_label=label, + nesting_depth=depth, + ) + jobs_mod.save_state(child_dir, child) + + jobs_mod.append_event( + job_dir, + "escalation", + actor="system:escalate", + child_job_id=child_id, + advisor=advisor.name, + model=model, + trace_id=trace_id, + depth=depth, + ) + + launch = launcher if launcher is not None else _default_launcher + try: + launch(child_id, state_root) + except OSError as exc: + _audit_skip(job_dir, reason=f"escalation worker failed to launch: {exc}") + return None + return child_id + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _drop_first_nonblank(rungs: Optional[list[str]]) -> list[str]: + """Return the non-blank rungs with the first one (the spawned rung) removed.""" + cleaned = [raw for raw in (rungs or []) if raw.strip()] + return cleaned[1:] + + +def _write_child_prompt(child_dir: Path, prompt: str) -> None: + prompt_path = child_dir / "prompt" + prompt_path.write_text(prompt, encoding="utf-8") + try: + prompt_path.chmod(0o600) + except OSError: + pass + + +def _write_child_command( + child_dir: Path, + *, + advisor: Advisor, + model: Optional[str], + cwd: str, + registry_path: str, + check: Optional[str], + check_timeout: float, + scope_paths: Optional[list[str]], + pass_env: list[str], + verify_with: Optional[str], + verify_model: Optional[str], + escalate_to: list[str], +) -> None: + info = { + "command": _build_child_argv(advisor, model), + "prompt_delivery": advisor.prompt_delivery, + "cwd": cwd, + "result_parser": advisor.result_parser, + "registry_path": registry_path, + "key": "", + "name": None, + "model": model or "", + "advisor": advisor.name, + "check": check, + "check_timeout": check_timeout, + # Security posture propagated to the larger peer (same write boundary, + # same credential withholding) — see the module docstring. + "scope_paths": scope_paths, + "pass_env": pass_env, + # The verification gate is re-run on the escalated output, and the + # remaining ladder lets a further failure climb the next rung. + "verify_with": verify_with, + "verify_model": verify_model, + "escalate_to": escalate_to, + } + jobs_mod.atomic_json_write(info, child_dir / "command.json") + + +def _audit_skip(job_dir: Path, *, reason: str) -> None: + jobs_mod.append_event( + job_dir, "escalation_skipped", actor="system:escalate", reason=reason + ) + + +def _default_launcher(child_id: str, state_root: Path) -> Any: + # Lazy import breaks the worker <-> escalate import cycle. + from .worker import start_worker + + return start_worker(child_id, state_root) + + +def _now() -> str: + from datetime import datetime, timezone + + return datetime.now(timezone.utc).isoformat() diff --git a/tests/test_escalate.py b/tests/test_escalate.py new file mode 100644 index 0000000..1bb57ca --- /dev/null +++ b/tests/test_escalate.py @@ -0,0 +1,266 @@ +"""Tests for the escalate-on-failure ladder (slice S5).""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path + +from crossagent import jobs as jobs_mod +from crossagent.escalate import _drop_first_nonblank, maybe_escalate, parse_rungs +from crossagent.jobs import ( + MAX_NESTING_DEPTH, + Job, + JobState, + load_state, + save_state, +) + + +# --------------------------------------------------------------------------- +# Rung parsing +# --------------------------------------------------------------------------- + + +def test_parse_rungs_advisor_only(): + assert parse_rungs(["claude"]) == [("claude", None)] + + +def test_parse_rungs_advisor_and_model(): + assert parse_rungs(["claude:opus", "codex:gpt-5.6-sol"]) == [ + ("claude", "opus"), + ("codex", "gpt-5.6-sol"), + ] + + +def test_parse_rungs_splits_on_first_colon_only(): + assert parse_rungs(["claude:some:model"]) == [("claude", "some:model")] + + +def test_parse_rungs_drops_blank_entries(): + assert parse_rungs(["", " ", "claude"]) == [("claude", None)] + + +def test_parse_rungs_none_is_empty(): + assert parse_rungs(None) == [] + + +def test_drop_first_nonblank_removes_only_the_spawned_rung(): + assert _drop_first_nonblank(["claude:opus", "codex"]) == ["codex"] + assert _drop_first_nonblank(["", "claude", "codex"]) == ["codex"] + assert _drop_first_nonblank(["only"]) == [] + + +# --------------------------------------------------------------------------- +# maybe_escalate +# --------------------------------------------------------------------------- + + +def _failed_parent( + state_root: Path, + *, + job_id="job_parent", + depth=1, + trace="trace_x", + parent_job_id=None, +) -> Job: + """Persist a failed delegation (failing check) as the escalation parent.""" + job_dir = jobs_mod.create_job_dir(state_root, job_id) + now = datetime.now(timezone.utc).isoformat() + job = Job( + job_id=job_id, + status=JobState.SUCCEEDED, # the delegate FINISHED... + advisor="codex", + cwd=str(state_root), + started_at=now, + updated_at=now, + trace_id=trace, + parent_job_id=parent_job_id, + nesting_depth=depth, + # ...but the check failed -> delegation_verdict == "failed". + check_result={ + "command": "pytest", + "exit_code": 1, + "stdout_tail": "", + "stderr_tail": "", + }, + ) + save_state(job_dir, job) + return job + + +def _spawns() -> tuple[list[tuple[str, Path]], object]: + spawned: list[tuple[str, Path]] = [] + + def launcher(child_id: str, state_root: Path) -> None: + spawned.append((child_id, state_root)) + + return spawned, launcher + + +def _escalate(job, state_root, escalate_to, launcher, **overrides): + kwargs = dict( + prompt="do the task", + state_root=state_root, + job_dir=state_root / job.job_id, + cwd=str(state_root), + registry_path=str(state_root / "sessions.json"), + escalate_to=escalate_to, + check="pytest", + check_timeout=30.0, + scope_paths=["src"], + pass_env=["ANTHROPIC_API_KEY"], + verify_with="claude", + verify_model=None, + launcher=launcher, + ) + kwargs.update(overrides) + return maybe_escalate(job, **kwargs) + + +def test_escalation_creates_same_trace_child_with_parent_link(tmp_path): + """Option (a): the re-dispatch is a same-trace child with parent_job_id set, + which is exactly what analytics.py counts as an escalation.""" + state_root = tmp_path / "state" + parent = _failed_parent(state_root) + spawned, launcher = _spawns() + + child_id = _escalate(parent, state_root, ["claude:opus"], launcher) + + assert child_id is not None + child = load_state(state_root / child_id) + assert child.parent_job_id == "job_parent" + assert child.trace_id == "trace_x" # SAME trace + assert child.nesting_depth == 2 # one deeper than the parent + assert child.advisor == "claude" + # The worker was actually launched for the child. + assert spawned == [(child_id, state_root)] + + +def test_escalation_is_recognised_by_analytics(tmp_path): + """The whole point of option (a): the shipped analytics escalation rate must + now see the re-dispatch, without any change to analytics.py.""" + from crossagent.analytics import build_analytics + + state_root = tmp_path / "state" + parent = _failed_parent(state_root) + _spawned, launcher = _spawns() + child_id = _escalate(parent, state_root, ["claude:opus"], launcher) + + parent = load_state(state_root / "job_parent") + child = load_state(state_root / child_id) + escalation = build_analytics([parent, child])["totals"]["escalation"] + assert escalation["failed"] == 1 + assert escalation["escalated"] == 1 + assert escalation["rate"] == 1.0 + + +def test_escalation_propagates_remaining_ladder_and_gates(tmp_path): + state_root = tmp_path / "state" + parent = _failed_parent(state_root) + _spawned, launcher = _spawns() + child_id = _escalate( + parent, state_root, ["claude:opus", "codex:gpt-5.6-sol"], launcher + ) + + command = json.loads((state_root / child_id / "command.json").read_text()) + # The spawned rung is dropped; the rest is handed to the child. + assert command["escalate_to"] == ["codex:gpt-5.6-sol"] + # Security posture + gates carried to the larger peer. + assert command["scope_paths"] == ["src"] + assert command["pass_env"] == ["ANTHROPIC_API_KEY"] + assert command["check"] == "pytest" + assert command["verify_with"] == "claude" + + +def test_no_escalation_when_delegation_did_not_fail(tmp_path): + """A verified/unverified delegation is never escalated.""" + state_root = tmp_path / "state" + job_dir = jobs_mod.create_job_dir(state_root, "job_ok") + now = datetime.now(timezone.utc).isoformat() + job = Job( + job_id="job_ok", + status=JobState.SUCCEEDED, + cwd=str(state_root), + started_at=now, + updated_at=now, + trace_id="trace_ok", + nesting_depth=1, + check_result={ + "command": "pytest", + "exit_code": 0, + "stdout_tail": "", + "stderr_tail": "", + }, + ) + save_state(job_dir, job) + spawned, launcher = _spawns() + assert _escalate(job, state_root, ["claude"], launcher) is None + assert spawned == [] + + +def test_no_escalation_when_no_rungs(tmp_path): + state_root = tmp_path / "state" + parent = _failed_parent(state_root) + spawned, launcher = _spawns() + assert _escalate(parent, state_root, None, launcher) is None + assert _escalate(parent, state_root, [], launcher) is None + assert spawned == [] + + +def test_escalation_halts_at_depth_cap(tmp_path): + """The runaway guard: a parent already at MAX_NESTING_DEPTH cannot spawn a + deeper child. The ladder stops and the skip is audited, not crashed.""" + state_root = tmp_path / "state" + # A parent already at the cap whose ancestor chain is broken: lineage + # resolution falls back to parent.nesting_depth + 1, which exceeds the cap. + parent = _failed_parent( + state_root, depth=MAX_NESTING_DEPTH, parent_job_id="job_missing_ancestor" + ) + spawned, launcher = _spawns() + + result = _escalate(parent, state_root, ["claude:opus"], launcher) + + assert result is None + assert spawned == [] + events = _events(state_root / "job_parent") + skips = [e for e in events if e.get("event") == "escalation_skipped"] + assert len(skips) == 1 + assert ( + "depth" in skips[0]["reason"].lower() or "nesting" in skips[0]["reason"].lower() + ) + + +def test_escalation_unknown_advisor_is_skipped_not_crashed(tmp_path): + state_root = tmp_path / "state" + parent = _failed_parent(state_root) + spawned, launcher = _spawns() + assert _escalate(parent, state_root, ["nosuchadvisor"], launcher) is None + assert spawned == [] + events = _events(state_root / "job_parent") + assert any(e.get("event") == "escalation_skipped" for e in events) + + +def test_escalation_audit_event_on_parent(tmp_path): + state_root = tmp_path / "state" + parent = _failed_parent(state_root) + _spawned, launcher = _spawns() + child_id = _escalate(parent, state_root, ["claude:opus"], launcher) + + events = _events(state_root / "job_parent") + escalations = [e for e in events if e.get("event") == "escalation"] + assert len(escalations) == 1 + assert escalations[0]["child_job_id"] == child_id + assert escalations[0]["advisor"] == "claude" + assert escalations[0]["trace_id"] == "trace_x" + + +def _events(job_dir: Path) -> list[dict]: + path = job_dir / "events.jsonl" + if not path.exists(): + return [] + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] From 4d040e1654e5a2592ae438f7cb79effcd4ead4a7 Mon Sep 17 00:00:00 2001 From: Dat Date: Sun, 26 Jul 2026 17:08:15 +0700 Subject: [PATCH 08/15] feat(delegate): wire --verify-with and --escalate-to through start and the worker (S5) Add --verify-with/--verify-model/--escalate-to to 'start', persist them in command.json, and load them in the worker. The worker runs the fresh verification pass after the check/scope gates and persists verify_result on the SAME terminal transition (the only writer of terminal state); after that transition it re-dispatches up the escalation ladder when the delegation failed. Refine the result-command verdict line to name the actual failing gate. E2E worker tests drive real jobs through worker_main and reload from disk: a structured verify 'fail' blocks green while the delegate exited 0; a pass yields verified; the verifier argv carries no session flag and the artifact is its final user-turn arg; a prose verifier stays unverified; a failed delegation spawns a same-trace child with parent_job_id set at depth+1. --- src/crossagent/cli.py | 63 ++++++++++++-- src/crossagent/worker.py | 102 +++++++++++++++++++++- tests/test_worker.py | 183 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 340 insertions(+), 8 deletions(-) diff --git a/src/crossagent/cli.py b/src/crossagent/cli.py index 1f0a7ae..379f892 100644 --- a/src/crossagent/cli.py +++ b/src/crossagent/cli.py @@ -387,6 +387,37 @@ def _parse_job_args(subcommand: str, argv: list[str]) -> argparse.Namespace: "vars are withheld from the delegate." ), ) + parser.add_argument( + "--verify-with", + dest="verify_with", + help=( + "Advisor that independently verifies the delegate's work in a " + "FRESH peer session, with the diff/answer supplied as user-turn " + "input (removes the implicit-authorship channel that weakens " + "self-grading). A failing verdict blocks the green path; a " + "prose-only or errored verifier degrades to unverified, never a " + "pass. Omit to leave verification off." + ), + ) + parser.add_argument( + "--verify-model", + dest="verify_model", + help="Model/alias for the --verify-with advisor (advisor default if omitted).", + ) + parser.add_argument( + "--escalate-to", + action="append", + default=None, + dest="escalate_to", + metavar="ADVISOR[:MODEL]", + help=( + "Re-dispatch a FAILED delegation (failing check, scope violation, " + "or failing verification) to this larger peer as a same-trace " + "child job. Repeatable to form an escalation ladder; each rung is " + "tried in turn as the previous fails. Bounded by the nesting-depth " + "cap. Omit to leave escalation off." + ), + ) parser.add_argument("--json", action="store_true") parser.set_defaults(stream=True) elif subcommand == "wait": @@ -657,23 +688,38 @@ def _cmd_result(args: argparse.Namespace) -> int: def _print_verdict(job: jobs_mod.Job, verdict: str, *, file: Any = sys.stderr) -> None: """Print a one-line delegation verdict to *file* (stderr by default).""" - check = job.check_result if verdict == "verified": - print("[crossagent] delegation verified — check passed", file=file) + print("[crossagent] delegation verified — all declared gates passed", file=file) elif verdict == "unverified": print( - "[crossagent] delegation UNVERIFIED — no --check ran; the delegate " - "finished but its work was not checked", + "[crossagent] delegation UNVERIFIED — the delegate finished but no " + "gate confirmed its work (no --check/--verify-with, or an " + "inconclusive verifier)", file=file, ) elif verdict == "failed": - exit_code = check.get("exit_code") if check else None print( - f"[crossagent] delegation FAILED verification — check exited {exit_code}", + f"[crossagent] delegation FAILED verification — {_failed_reason(job)}", file=file, ) +def _failed_reason(job: jobs_mod.Job) -> str: + """Describe why a delegation failed, naming the actual failing gate.""" + if job.status != jobs_mod.JobState.SUCCEEDED: + return f"delegate did not finish cleanly (status {job.status.value})" + scope = job.scope_result + if scope is not None and scope.get("status") != "ok": + return f"scope {scope.get('status')} ({len(scope.get('violating_paths') or [])} path(s))" + check = job.check_result + if check is not None and check.get("exit_code") != 0: + return f"check exited {check.get('exit_code')}" + verify = job.verify_result + if verify is not None and verify.get("verdict") == "fail": + return f"independent verification by {verify.get('advisor')} returned fail" + return "a declared gate did not pass" + + def _metrics_summary(job: jobs_mod.Job) -> str: """Return a one-line advisor-metrics summary, or ``""`` when nothing was measured. Cost/token/duration go to stderr so piped stdout stays the result. @@ -899,6 +945,11 @@ def _write_command_info( # --allow-path was given (enforcement off), distinct from an empty list. "scope_paths": getattr(args, "allow_path", None), "pass_env": getattr(args, "pass_env", []), + # Independent verification + escalation ladder (S5). ``verify_with`` is + # ``None`` when off; ``escalate_to`` is the (possibly empty) rung list. + "verify_with": getattr(args, "verify_with", None), + "verify_model": getattr(args, "verify_model", None), + "escalate_to": getattr(args, "escalate_to", None) or [], } jobs_mod.atomic_json_write(info, job_dir / "command.json") diff --git a/src/crossagent/worker.py b/src/crossagent/worker.py index 079a3df..5d3ce79 100644 --- a/src/crossagent/worker.py +++ b/src/crossagent/worker.py @@ -17,13 +17,16 @@ from pathlib import Path from typing import Any, Optional +from . import advisors as advisors_mod from . import check as check_mod from . import credentials as credentials_mod +from . import escalate as escalate_mod from . import jobs as jobs_mod from . import parsers as parsers_mod from . import registry as reg_mod from . import runner as runner_mod from . import scope as scope_mod +from . import verify as verify_mod # --------------------------------------------------------------------------- @@ -50,6 +53,13 @@ class _JobCommand: # credential env vars the caller opted to pass through to the delegate. scope_paths: Optional[list[str]] pass_env: list[str] + # Independent verification + escalation (slice S5). ``verify_with`` names the + # advisor that grades the delegate's artifact in a fresh session (``None`` = + # off). ``escalate_to`` is the ordered ladder of ``advisor[:model]`` rungs a + # failed delegation is re-dispatched up ([] = off). + verify_with: Optional[str] + verify_model: Optional[str] + escalate_to: list[str] # --------------------------------------------------------------------------- @@ -234,6 +244,12 @@ def _should_cancel() -> bool: # distinctly and drives the delegation verdict to failed, never a pass. scope_result = _run_scope_assertion(command, scope_baseline, job_dir) + # Run the independent verification pass (S5) if a verifier was declared. A + # FRESH peer session grades the delegate's artifact supplied as user-turn + # input (D6). A structured ``fail`` verdict blocks the green path; a + # prose-only or errored verifier degrades to inconclusive, never a pass. + verify_result = _run_verification(command, prompt, parsed.result, job_dir) + now = datetime.now(timezone.utc).isoformat() # Persist the advisor telemetry the parser extracted (S1) and the check-gate # outcome (S3) on the SAME terminal transition — the worker is the only @@ -241,7 +257,7 @@ def _should_cancel() -> bool: # disk. ``parsed`` fields and ``check_result`` already default to unknown / # None when nothing was measured (D4/D7), so this never fails the job and # never turns an unmeasured metric into a zero or an unrun check into a pass. - jobs_mod.transition_to( + job = jobs_mod.transition_to( job, final_state, job_dir=job_dir, @@ -257,6 +273,27 @@ def _should_cancel() -> bool: check_result=check_result, scope_result=scope_result, withheld_env=withheld, + verify_result=verify_result, + ) + + # Escalate-on-failure (S5). Runs AFTER the terminal state is persisted, so + # the failed parent is complete on disk before its same-trace child is + # spawned. maybe_escalate is a no-op unless the delegation FAILED and an + # escalation ladder remains; it never raises and respects MAX_NESTING_DEPTH. + escalate_mod.maybe_escalate( + job, + prompt=prompt, + state_root=state_dir, + job_dir=job_dir, + cwd=command.cwd, + registry_path=command.registry_path, + escalate_to=command.escalate_to, + check=command.check, + check_timeout=command.check_timeout, + scope_paths=command.scope_paths, + pass_env=command.pass_env, + verify_with=command.verify_with, + verify_model=command.verify_model, ) return 0 @@ -318,6 +355,66 @@ def _run_scope_assertion( return outcome.to_dict() +def _run_verification( + command: _JobCommand, + prompt: str, + result_text: Optional[str], + job_dir: Path, +) -> Optional[jobs_mod.VerifyResultDict]: + """Run the independent verification pass (S5), if a verifier was declared. + + Returns ``None`` when no verifier was requested — distinct from a + verification that ran and failed (D7). The verifier is a FRESH peer session + grading the delegate's artifact as user-turn input (D6); it is a delegate + too, so it runs with credentials scrubbed. An unknown verifier advisor or a + delegate that produced no artifact is recorded as an ``error`` outcome + (inconclusive), never a crash and never a silent pass. + """ + if not command.verify_with: + return None + try: + advisor = advisors_mod.resolve(command.verify_with) + except KeyError as exc: + outcome = verify_mod.VerifyOutcome( + command.verify_with, + command.verify_model, + "error", + False, + f"unknown verifier advisor: {exc}", + ) + else: + structured = advisor.json_schema_flag is not None + if result_text is None: + outcome = verify_mod.VerifyOutcome( + advisor.name, + command.verify_model, + "error", + structured, + "delegate produced no artifact to verify", + ) + else: + artifact = verify_mod.build_artifact(prompt, result_text, command.cwd) + outcome = verify_mod.run_verification( + advisor, + command.verify_model, + artifact, + cwd=command.cwd, + pass_env=command.pass_env, + timeout=verify_mod.VERIFY_DEFAULT_TIMEOUT_SECONDS, + ) + # Audit the verdict, never the artifact (which can contain repo content). + jobs_mod.append_event( + job_dir, + "verify", + actor="system:verify", + advisor=outcome.advisor, + model=outcome.model, + verdict=outcome.verdict, + structured=outcome.structured, + ) + return outcome.to_dict() + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -342,6 +439,9 @@ def _load_command(job_dir: Path) -> _JobCommand: ), scope_paths=_load_scope_paths(data.get("scope_paths")), pass_env=[str(name) for name in data.get("pass_env", [])], + verify_with=data.get("verify_with"), + verify_model=data.get("verify_model"), + escalate_to=[str(rung) for rung in data.get("escalate_to", [])], ) diff --git a/tests/test_worker.py b/tests/test_worker.py index 74600a1..c404ed5 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -131,12 +131,16 @@ def _run_job_through_worker( check: str | None = None, check_timeout: float = 30.0, pass_env: list[str] | None = None, + verify_with: str | None = None, + verify_model: str | None = None, + escalate_to: list[str] | None = None, ) -> Job: """Set up a job whose advisor is a fake claude-stream script, run the worker synchronously, and return the Job reloaded from disk. When *check* is given it is written into command.json so the worker runs - the S3 check-gate after the delegate finishes.""" + the S3 check-gate after the delegate finishes. ``verify_with`` / + ``escalate_to`` drive the S5 verification pass and escalation ladder.""" state_dir = tmp_path / "state" job_dir = jobs_mod.create_job_dir(state_dir, job_id) @@ -153,6 +157,8 @@ def _run_job_through_worker( cwd=str(tmp_path), started_at=now, updated_at=now, + trace_id=f"trace_{job_id}", + nesting_depth=1, ), ) (job_dir / "prompt").write_text("hello", encoding="utf-8") @@ -172,6 +178,11 @@ def _run_job_through_worker( command_info["check_timeout"] = check_timeout if pass_env is not None: command_info["pass_env"] = pass_env + if verify_with is not None: + command_info["verify_with"] = verify_with + command_info["verify_model"] = verify_model + if escalate_to is not None: + command_info["escalate_to"] = escalate_to jobs_mod.atomic_json_write(command_info, job_dir / "command.json") exit_code = worker_main(job_id, state_dir) @@ -635,3 +646,173 @@ def test_scope_module_importable_without_error(): # Guard: the module and its git timeout constant are wired. assert scope_mod._GIT_TIMEOUT_SECONDS > 0 assert pytest is not None + + +# ========================================================================= +# End-to-end independent verification + escalation (slice S5): drive a real +# job through worker_main and reload state from disk. A unit test over the +# combiner alone would not catch the worker forgetting to forward the verify +# result or to spawn the escalation child — this drives the whole path. +# ========================================================================= + +from crossagent.advisors import Advisor # noqa: E402 + + +def _fake_verifier( + tmp_path: Path, verdict: str, *, record_argv: bool = False +) -> Advisor: + """A fake claude-style verifier advisor emitting a structured verdict.""" + record = ( + "import sys\n" + "open('verifier_argv.json','w').write(__import__('json').dumps(sys.argv))\n" + if record_argv + else "" + ) + script = tmp_path / "fake_verifier.py" + script.write_text( + record + + "import json\n" + + "print(json.dumps({'type':'result','subtype':'success'," + + f"'structured_output': {{'verdict': {verdict!r}, 'reason': 'because'}}}}))\n", + encoding="utf-8", + ) + return Advisor( + name="myverifier", + executable=sys.executable, + base_args=(str(script),), + prompt_delivery="positional", + result_parser="claude-stream", + json_args=(), + json_schema_flag="--json-schema", + ) + + +def test_worker_verification_fail_blocks_green_end_to_end(tmp_path, monkeypatch): + """A structured 'fail' from the fresh verifier is persisted and drives the + delegation verdict to failed even though the delegate exited 0 (D6).""" + verifier = _fake_verifier(tmp_path, "fail") + monkeypatch.setattr( + "crossagent.advisors.resolve", lambda name, config_path=None: verifier + ) + + job = _run_job_through_worker(tmp_path, _RESULT_OK, verify_with="myverifier") + + assert job.status == JobState.SUCCEEDED # the delegate DID finish + assert job.verify_result is not None + assert job.verify_result["verdict"] == "fail" + assert job.verify_result["advisor"] == "myverifier" + assert jobs_mod.delegation_verdict(job) == "failed" + + +def test_worker_verification_pass_yields_verified_end_to_end(tmp_path, monkeypatch): + verifier = _fake_verifier(tmp_path, "pass") + monkeypatch.setattr( + "crossagent.advisors.resolve", lambda name, config_path=None: verifier + ) + + job = _run_job_through_worker(tmp_path, _RESULT_OK, verify_with="myverifier") + + assert job.verify_result["verdict"] == "pass" + assert job.verify_result["structured"] is True + assert jobs_mod.delegation_verdict(job) == "verified" + + +def test_worker_verifier_session_is_fresh_and_artifact_is_user_turn( + tmp_path, monkeypatch +): + """The verifier subprocess receives NO session-attachment flag (fresh + session) and the artifact as its final positional argument (user turn).""" + verifier = _fake_verifier(tmp_path, "pass", record_argv=True) + monkeypatch.setattr( + "crossagent.advisors.resolve", lambda name, config_path=None: verifier + ) + + _run_job_through_worker(tmp_path, _RESULT_OK, verify_with="myverifier") + + argv = json.loads((tmp_path / "verifier_argv.json").read_text()) + for flag in ("--resume", "--fork-session", "--name"): + assert flag not in argv, flag + # The artifact is the LAST argument (user-turn input), and it embeds the + # delegate's answer rather than arriving as prior assistant context. + assert "DELEGATE'S ANSWER" in argv[-1] + assert "ok" in argv[-1] + + +def test_worker_unverified_verifier_does_not_green_end_to_end(tmp_path, monkeypatch): + """A prose-only verifier (no structured verdict) degrades to unverified: it + neither greens nor hard-fails the delegation.""" + prose = tmp_path / "fake_prose_verifier.py" + prose.write_text("print('looks fine to me')\n", encoding="utf-8") + verifier = Advisor( + name="prosever", + executable=sys.executable, + base_args=(str(prose),), + prompt_delivery="positional", + result_parser="text", + json_schema_flag=None, + ) + monkeypatch.setattr( + "crossagent.advisors.resolve", lambda name, config_path=None: verifier + ) + + job = _run_job_through_worker(tmp_path, _RESULT_OK, verify_with="prosever") + + assert job.verify_result["verdict"] == "unverified" + assert jobs_mod.delegation_verdict(job) == "unverified" + + +def test_worker_verify_audit_event_records_verdict_not_artifact(tmp_path, monkeypatch): + verifier = _fake_verifier(tmp_path, "fail") + monkeypatch.setattr( + "crossagent.advisors.resolve", lambda name, config_path=None: verifier + ) + _run_job_through_worker(tmp_path, _RESULT_OK, verify_with="myverifier") + + events_path = tmp_path / "state" / "job_e2e" / "events.jsonl" + lines = [ + json.loads(line) + for line in events_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + verify_events = [event for event in lines if event.get("event") == "verify"] + assert len(verify_events) == 1 + assert verify_events[0]["verdict"] == "fail" + assert verify_events[0]["advisor"] == "myverifier" + + +def test_worker_escalates_failed_delegation_end_to_end(tmp_path, monkeypatch): + """A failed delegation (failing check) with an escalation ladder spawns a + same-trace child with parent_job_id set — the exact shape analytics counts.""" + spawned: list[tuple[str, Path]] = [] + monkeypatch.setattr( + "crossagent.escalate._default_launcher", + lambda child_id, state_root: spawned.append((child_id, state_root)), + ) + + job = _run_job_through_worker( + tmp_path, _RESULT_OK, check=_check_cmd(1), escalate_to=["codex:gpt-5.6-sol"] + ) + + assert jobs_mod.delegation_verdict(job) == "failed" + assert len(spawned) == 1 + child_id, _ = spawned[0] + child = jobs_mod.load_state(tmp_path / "state" / child_id) + assert child.parent_job_id == job.job_id + assert child.trace_id == job.trace_id # SAME trace (option a) + assert child.nesting_depth == 2 + assert child.advisor == "codex" + + +def test_worker_does_not_escalate_a_passing_delegation(tmp_path, monkeypatch): + spawned: list[tuple[str, Path]] = [] + monkeypatch.setattr( + "crossagent.escalate._default_launcher", + lambda child_id, state_root: spawned.append((child_id, state_root)), + ) + + job = _run_job_through_worker( + tmp_path, _RESULT_OK, check=_check_cmd(0), escalate_to=["codex"] + ) + + assert jobs_mod.delegation_verdict(job) == "verified" + assert spawned == [] From 8a56cf7909ce72d4dc78838cf5c3568c62d3a817 Mon Sep 17 00:00:00 2001 From: Dat Date: Mon, 27 Jul 2026 05:53:37 +0700 Subject: [PATCH 09/15] fix(escalate): scope escalation to finished delegates with a failed gate maybe_escalate gated only on delegation_verdict != "failed", but that verdict is "failed" for ANY non-success terminal status, so a CANCELLED job (explicit user intent to stop) or a TIMED_OUT job was silently re-dispatched to a larger, costlier peer. Gate on JobState.SUCCEEDED first so only a declared-gate failure (check / scope / verify) escalates; a delegate that did not finish cleanly does not. TIMED_OUT does not escalate: it produced no graded artifact and a bigger, slower peer is at least as likely to time out again. Also guard the child-staging writes (create_job_dir, prompt/command writes, save_state) against OSError so the module's "never raises" contract holds on a read-only or full disk. --- src/crossagent/escalate.py | 104 +++++++++++++++++++++++-------------- tests/test_escalate.py | 71 +++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 38 deletions(-) diff --git a/src/crossagent/escalate.py b/src/crossagent/escalate.py index c4e4ddc..92e201c 100644 --- a/src/crossagent/escalate.py +++ b/src/crossagent/escalate.py @@ -111,10 +111,30 @@ def maybe_escalate( """Re-dispatch a *failed* delegation to the next ladder rung, if any. Returns the spawned child's job id, or ``None`` when nothing was escalated - (the delegation did not fail, no rungs remain, the rung advisor is unknown, - or the depth cap was reached). Never raises: an escalation that cannot be - launched is recorded and skipped, never allowed to crash the worker. + (the delegate did not finish cleanly, no declared gate failed, no rungs + remain, the rung advisor is unknown, or the depth cap was reached). Never + raises: an escalation that cannot be launched is recorded and skipped, never + allowed to crash the worker. """ + # Escalation re-dispatches a delegate that FINISHED but whose work FAILED a + # declared gate — a failing check, a scope violation, or a failing + # verification (this module's stated scope). A job that did not finish + # cleanly is deliberately out of scope, so gate on SUCCEEDED first rather + # than on ``delegation_verdict != "failed"`` alone (which also returns + # "failed" for CANCELLED / TIMED_OUT / a crashed delegate): + # * CANCELLED is explicit user intent to stop; re-dispatching to a larger, + # costlier peer is the opposite of cancelling and spends real money. + # * TIMED_OUT (and any other non-success terminal status) produced no + # graded artifact — there is no gate failure to escalate, and a bigger + # model is generally slower, so it is at least as likely to time out + # again under the same budget. A hard task that needs a bigger model is a + # fresh dispatch decision, not an automatic ladder climb that silently + # burns budget. So TIMED_OUT does NOT escalate. + # Gating on SUCCEEDED means ``delegation_verdict == "failed"`` below can only + # be a declared-gate failure — mirroring cli._failed_reason's "did not finish + # cleanly" vs. gate-failure distinction. + if failed_job.status != jobs_mod.JobState.SUCCEEDED: + return None if delegation_verdict(failed_job) != "failed": return None @@ -145,41 +165,49 @@ def maybe_escalate( _audit_skip(job_dir, reason=f"escalation halted: {exc}") return None - child_dir = jobs_mod.create_job_dir(state_root, child_id) - _write_child_prompt(child_dir, prompt) - _write_child_command( - child_dir, - advisor=advisor, - model=model, - cwd=cwd, - registry_path=registry_path, - check=check, - check_timeout=check_timeout, - scope_paths=scope_paths, - pass_env=pass_env, - verify_with=verify_with, - verify_model=verify_model, - escalate_to=remaining, - ) - child = Job( - job_id=child_id, - status=jobs_mod.JobState.PENDING, - advisor=advisor.name, - name="", - cwd=cwd, - redacted_command="", - started_at=_now(), - updated_at=_now(), - last_activity_at=_now(), - last_event="escalation.created", - max_runtime_seconds=failed_job.max_runtime_seconds, - termination_grace_seconds=failed_job.termination_grace_seconds, - parent_job_id=parent_id, - trace_id=trace_id, - orchestrator_label=label, - nesting_depth=depth, - ) - jobs_mod.save_state(child_dir, child) + # Staging the child on disk touches the filesystem (mkdir, two file writes, + # a state save). A read-only or full disk raises OSError; catch it here so + # the "never raises" contract holds — a child that cannot be staged is + # recorded and skipped, exactly like a launch failure below. + try: + child_dir = jobs_mod.create_job_dir(state_root, child_id) + _write_child_prompt(child_dir, prompt) + _write_child_command( + child_dir, + advisor=advisor, + model=model, + cwd=cwd, + registry_path=registry_path, + check=check, + check_timeout=check_timeout, + scope_paths=scope_paths, + pass_env=pass_env, + verify_with=verify_with, + verify_model=verify_model, + escalate_to=remaining, + ) + child = Job( + job_id=child_id, + status=jobs_mod.JobState.PENDING, + advisor=advisor.name, + name="", + cwd=cwd, + redacted_command="", + started_at=_now(), + updated_at=_now(), + last_activity_at=_now(), + last_event="escalation.created", + max_runtime_seconds=failed_job.max_runtime_seconds, + termination_grace_seconds=failed_job.termination_grace_seconds, + parent_job_id=parent_id, + trace_id=trace_id, + orchestrator_label=label, + nesting_depth=depth, + ) + jobs_mod.save_state(child_dir, child) + except OSError as exc: + _audit_skip(job_dir, reason=f"escalation could not be staged: {exc}") + return None jobs_mod.append_event( job_dir, diff --git a/tests/test_escalate.py b/tests/test_escalate.py index 1bb57ca..d5363b8 100644 --- a/tests/test_escalate.py +++ b/tests/test_escalate.py @@ -255,6 +255,77 @@ def test_escalation_audit_event_on_parent(tmp_path): assert escalations[0]["trace_id"] == "trace_x" +# --------------------------------------------------------------------------- +# HIGH 1: escalation is scoped to a FINISHED delegate whose declared gate +# failed — never to a job that did not finish cleanly (CANCELLED / TIMED_OUT). +# --------------------------------------------------------------------------- + + +def _terminal_parent( + state_root: Path, *, status: JobState, job_id: str = "job_term" +) -> Job: + """Persist a parent in a non-success terminal state. + + Carries a *failing* check_result, mirroring a real cancel/timeout that + interrupts the delegate mid-run: under the old ``delegation_verdict != + "failed"`` gate (which returns "failed" for ANY non-success terminal status) + this would have been re-dispatched. + """ + job_dir = jobs_mod.create_job_dir(state_root, job_id) + now = datetime.now(timezone.utc).isoformat() + job = Job( + job_id=job_id, + status=status, + advisor="codex", + cwd=str(state_root), + started_at=now, + updated_at=now, + trace_id="trace_term", + nesting_depth=1, + check_result={ + "command": "pytest", + "exit_code": 1, + "stdout_tail": "", + "stderr_tail": "", + }, + ) + save_state(job_dir, job) + return job + + +def test_cancelled_parent_is_never_escalated(tmp_path): + """A user who cancels a job must not have it silently re-dispatched to a + larger, costlier peer — that is the opposite of cancelling (HIGH 1).""" + state_root = tmp_path / "state" + parent = _terminal_parent(state_root, status=JobState.CANCELLED) + spawned, launcher = _spawns() + assert _escalate(parent, state_root, ["claude:opus"], launcher) is None + assert spawned == [] + + +def test_timed_out_parent_is_never_escalated(tmp_path): + """Decision: a TIMED_OUT delegate produced no graded artifact, and a bigger, + slower peer is at least as likely to time out again under the same budget, so + it is NOT auto-escalated — the ladder is for gate failures on completed work, + not for work that never finished (HIGH 1).""" + state_root = tmp_path / "state" + parent = _terminal_parent(state_root, status=JobState.TIMED_OUT) + spawned, launcher = _spawns() + assert _escalate(parent, state_root, ["claude:opus"], launcher) is None + assert spawned == [] + + +def test_gate_failure_parent_still_escalates(tmp_path): + """The working path is not regressed: a SUCCEEDED delegate whose declared + gate (here, the check) failed is still escalated (HIGH 1).""" + state_root = tmp_path / "state" + parent = _failed_parent(state_root) # SUCCEEDED + failing check + spawned, launcher = _spawns() + child_id = _escalate(parent, state_root, ["claude:opus"], launcher) + assert child_id is not None + assert spawned == [(child_id, state_root)] + + def _events(job_dir: Path) -> list[dict]: path = job_dir / "events.jsonl" if not path.exists(): From e4eaf28b009d007b7afad5fc6b7e8033a2e360d1 Mon Sep 17 00:00:00 2001 From: Dat Date: Mon, 27 Jul 2026 05:53:43 +0700 Subject: [PATCH 10/15] fix(verify): make schema temp-file lifecycle exception-safe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tempfile.mkstemp() ran before run_verification's try block, so a read-only /tmp, a full disk, or a restricted TMPDIR raised OSError out of the verifier — which both docstrings promise never happens — past the worker's unguarded caller, leaving the job non-terminal forever after the check and scope gates already ran and destroying the delegate's real work. Extract the schema file into a _schema_file context manager that degrades to non-structured mode when the temp file cannot be created and always unlinks on exit, so verification never propagates OSError. This also trims run_verification back under the 50-line guideline. --- src/crossagent/verify.py | 62 +++++++++++++++++++++++++++++----------- tests/test_verify.py | 21 ++++++++++++++ tests/test_worker.py | 40 ++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 17 deletions(-) diff --git a/src/crossagent/verify.py b/src/crossagent/verify.py index 7f28184..0131451 100644 --- a/src/crossagent/verify.py +++ b/src/crossagent/verify.py @@ -47,6 +47,8 @@ import os import subprocess import tempfile +from collections.abc import Iterator +from contextlib import contextmanager from dataclasses import dataclass from typing import Any, Optional @@ -279,6 +281,45 @@ def verifier_env(pass_env: Optional[list[str]] = None) -> dict[str, str]: } +@contextmanager +def _schema_file(structured: bool) -> Iterator[Optional[str]]: + """Yield a path to a temp file holding the verdict schema, or ``None``. + + Exception-safe by contract (the verifier "never raises", see + :func:`run_verification`): if the schema file cannot be created or written — + a read-only ``/tmp``, a full disk, a restricted ``TMPDIR`` in a sandbox/CI + container — this yields ``None`` so verification degrades to non-structured + mode instead of raising ``OSError`` out into the worker. The file is always + unlinked on exit. When *structured* is False no file is created. + + The mkstemp/write is in its own try/except (setup), separate from the + try/finally around the yield (cleanup), so an exception the caller raises + while the file is in use propagates untouched and is never mistaken for a + setup failure. + """ + if not structured: + yield None + return + schema_path: Optional[str] = None + try: + schema_fd, created_path = tempfile.mkstemp( + prefix="crossagent-verify-", suffix=".json" + ) + with os.fdopen(schema_fd, "w", encoding="utf-8") as handle: + json.dump(_VERDICT_SCHEMA, handle) + schema_path = created_path + except OSError: + schema_path = None + try: + yield schema_path + finally: + if schema_path is not None: + try: + os.unlink(schema_path) + except OSError: + pass + + def run_verification( advisor: Advisor, model: Optional[str], @@ -291,19 +332,12 @@ def run_verification( """Run one fresh, independent verification pass and return its outcome. Never raises: any launch or parse failure degrades to an ``error``/ - ``unverified`` verdict — a broken verifier must never crash the worker nor - silently green a delegation. + ``unverified`` verdict, and an inability to create the temp schema file + degrades to non-structured mode — a broken verifier must never crash the + worker nor silently green a delegation. """ structured = advisor.json_schema_flag is not None - schema_fd, schema_path = (-1, None) - if structured: - schema_fd, schema_path = tempfile.mkstemp( - prefix="crossagent-verify-", suffix=".json" - ) - try: - if schema_path is not None: - with os.fdopen(schema_fd, "w", encoding="utf-8") as handle: - json.dump(_VERDICT_SCHEMA, handle) + with _schema_file(structured) as schema_path: cmd = build_verifier_command(advisor, model, schema_path=schema_path) _append_prompt(cmd, advisor, artifact) parser = parsers_mod.get_parser(advisor.result_parser) @@ -323,12 +357,6 @@ def run_verification( structured, f"verifier failed to launch: {exc}", ) - finally: - if schema_path is not None: - try: - os.unlink(schema_path) - except OSError: - pass parsed = ( outcome.result diff --git a/tests/test_verify.py b/tests/test_verify.py index d7bdf3b..24f3129 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -187,6 +187,27 @@ def test_run_verification_no_result_is_error(tmp_path): assert outcome.verdict == "error" +def test_run_verification_survives_schema_temp_file_failure(tmp_path, monkeypatch): + """HIGH 2: a read-only /tmp, a full disk, or a restricted TMPDIR makes + ``tempfile.mkstemp`` raise OSError. That must degrade to non-structured mode, + never propagate out of ``run_verification`` (which promises never to raise) + and crash the worker after the delegate already did its work.""" + + def _boom(*args, **kwargs): + raise OSError("read-only file system") + + monkeypatch.setattr(verify_mod.tempfile, "mkstemp", _boom) + advisor = _structured_advisor(tmp_path, "pass") + + # Must not raise despite mkstemp failing: + outcome = run_verification(advisor, None, "ARTIFACT", cwd=str(tmp_path)) + + assert isinstance(outcome, VerifyOutcome) + # Degraded gracefully: without the schema flag the fake advisor still emitted + # a JSON verdict, which is parsed out of the answer. + assert outcome.verdict == "pass" + + def test_run_verification_never_raises_on_missing_executable(tmp_path): advisor = Advisor( name="ghost", diff --git a/tests/test_worker.py b/tests/test_worker.py index c404ed5..3a8ebfc 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -780,6 +780,46 @@ def test_worker_verify_audit_event_records_verdict_not_artifact(tmp_path, monkey assert verify_events[0]["advisor"] == "myverifier" +def test_worker_persists_terminal_record_when_verify_schema_temp_file_fails( + tmp_path, monkeypatch +): + """HIGH 2: if the verifier's schema temp file cannot be created — read-only + /tmp, full disk, restricted TMPDIR — the OSError must NOT propagate out of + run_verification into the worker after the check + scope gates already ran. + Otherwise the terminal transition never persists and the job is wedged + non-terminal forever with the delegate's real work destroyed. The worker must + still reach and persist a terminal record.""" + import tempfile + + from crossagent import verify as verify_mod + + verifier = _fake_verifier(tmp_path, "pass") # structured -> mkstemp attempted + monkeypatch.setattr( + "crossagent.advisors.resolve", lambda name, config_path=None: verifier + ) + + # Fail ONLY the verifier's schema temp file; leave the state-persistence + # temp files (a different prefix) working, so this isolates the verify path. + real_mkstemp = tempfile.mkstemp + + def _selective_mkstemp(*args, **kwargs): + if kwargs.get("prefix", "").startswith("crossagent-verify-"): + raise OSError("read-only file system") + return real_mkstemp(*args, **kwargs) + + monkeypatch.setattr(verify_mod.tempfile, "mkstemp", _selective_mkstemp) + + job = _run_job_through_worker(tmp_path, _RESULT_OK, verify_with="myverifier") + + # The worker reached a terminal state and recorded the verification, rather + # than crashing after the delegate's work with the job left non-terminal. + assert jobs_mod.is_terminal(job.status) + assert job.status == JobState.SUCCEEDED + assert job.verify_result is not None + # Degraded to non-structured mode: the verdict still parsed from the answer. + assert job.verify_result["verdict"] == "pass" + + def test_worker_escalates_failed_delegation_end_to_end(tmp_path, monkeypatch): """A failed delegation (failing check) with an escalation ladder spawns a same-trace child with parent_job_id set — the exact shape analytics counts.""" From a8777e6436cac86a6ab2e6024b15d048fc23e68e Mon Sep 17 00:00:00 2001 From: Dat Date: Mon, 27 Jul 2026 05:53:55 +0700 Subject: [PATCH 11/15] fix(cli): scrub credentials on the foreground dispatch path Credential scrubbing was wired into the durable-job worker but not the default `crossagent --agent ... --prompt ...` invocation: _run_advisor ran the advisor subprocess with no env=, so it inherited the caller's full unscrubbed os.environ, and --pass-env was registered only for `start`. The security claim ("credential withholding for delegates") therefore exceeded the implementation for the original dispatch mode. Scrub via the same credentials.scrub_env helper the worker uses (one shared policy, no drift) and add the --pass-env escape hatch to the foreground parser. --- src/crossagent/cli.py | 32 +++++++++++++++-- tests/test_cli.py | 84 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 3 deletions(-) diff --git a/src/crossagent/cli.py b/src/crossagent/cli.py index 379f892..57446c4 100644 --- a/src/crossagent/cli.py +++ b/src/crossagent/cli.py @@ -21,6 +21,7 @@ from . import __version__ from . import advisors as advisors_mod from . import check as check_mod +from . import credentials as credentials_mod from . import jobs as jobs_mod from . import parsers as parsers_mod from . import registry as reg @@ -109,10 +110,16 @@ def build_command( def _run_advisor( - cmd: list[str], cwd: str | None, parser_name: str + cmd: list[str], + cwd: str | None, + parser_name: str, + *, + env: dict[str, str] | None = None, ) -> tuple[int, parsers_mod.ParsedResult]: parser = parsers_mod.get_parser(parser_name) - outcome = runner_mod.run(cmd, cwd=cwd, consumer=parser, max_runtime_seconds=None) + outcome = runner_mod.run( + cmd, cwd=cwd, env=env, consumer=parser, max_runtime_seconds=None + ) parsed = ( outcome.result if isinstance(outcome.result, parsers_mod.ParsedResult) @@ -199,6 +206,18 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument( "--registry", default=str(reg.DEFAULT_REGISTRY), help="Session registry path." ) + parser.add_argument( + "--pass-env", + action="append", + default=[], + dest="pass_env", + help=( + "Name of an environment variable to pass through to the advisor even " + "though it matches a credential pattern (e.g. the advisor's own API " + "key). Repeatable. By default all credential-bearing env vars are " + "withheld from the advisor, matching the durable-job path." + ), + ) parser.add_argument( "--list-advisors", action="store_true", help="Print known advisors and exit." ) @@ -278,7 +297,14 @@ def _dispatch( registry: dict[str, Any], registry_path: Path, ) -> int: - code, parsed = _run_advisor(cmd, args.cwd, advisor.result_parser) + # A foreground advisor is a delegate too: withhold the caller's ambient + # credentials (S4 policy), sharing the exact scrub the durable-job path uses + # (worker.build_advisor_env delegates to the same helper) so the two dispatch + # modes can never drift. ``--pass-env NAME`` is the caller's opt-out. + advisor_env = credentials_mod.scrub_env( + os.environ, pass_through=getattr(args, "pass_env", []) + ) + code, parsed = _run_advisor(cmd, args.cwd, advisor.result_parser, env=advisor_env) if parsed.failure: error = parsed.error or f"{advisor.name} exited with code {code}" diff --git a/tests/test_cli.py b/tests/test_cli.py index 07f7023..291bc83 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,3 +1,5 @@ +import sys + import pytest from crossagent import __version__, advisors @@ -138,6 +140,88 @@ def test_missing_advisor_cli_exits_cleanly_without_logging_prompt(monkeypatch, c assert captured.out == "" +# --------------------------------------------------------------------------- +# HIGH 3: the foreground (default) dispatch path scrubs credentials from the +# advisor env too — not only the durable-job path — with the same --pass-env +# escape hatch, so both dispatch modes share one policy. +# --------------------------------------------------------------------------- + +_FG_SECRET_NAME = "AWS_SECRET_ACCESS_KEY" +_FG_SECRET_VALUE = "fg-super-secret-value" + + +def _env_probe_advisor(tmp_path, probe_name): + """A fake claude-stream advisor that records one env var it was given.""" + script = tmp_path / "fake_fg_advisor.py" + script.write_text( + "import json, os\n" + f"open('fg_env_probe.txt', 'w').write(" + f"os.environ.get({probe_name!r}, 'ABSENT'))\n" + "print(json.dumps({'type': 'result', 'subtype': 'success', " + "'result': 'ok'}))\n", + encoding="utf-8", + ) + return Advisor( + name="fakefg", + executable=sys.executable, + base_args=(str(script),), + prompt_delivery="positional", + result_parser="claude-stream", + ) + + +def test_foreground_advisor_env_is_scrubbed(tmp_path, monkeypatch): + """The default `crossagent --agent ... --prompt ...` invocation must withhold + the caller's ambient credentials from the advisor, matching the durable-job + path (HIGH 3).""" + monkeypatch.setenv(_FG_SECRET_NAME, _FG_SECRET_VALUE) + advisor = _env_probe_advisor(tmp_path, _FG_SECRET_NAME) + monkeypatch.setattr(advisors, "resolve", lambda _name: advisor) + + code = main(["--agent", "fakefg", "--prompt", "hi", "--cwd", str(tmp_path)]) + + assert code == 0 + assert (tmp_path / "fg_env_probe.txt").read_text() == "ABSENT" + + +def test_foreground_pass_env_opts_a_named_var_back_in(tmp_path, monkeypatch): + """--pass-env NAME is the foreground escape hatch: the named credential var + reaches the advisor despite matching a credential pattern (HIGH 3).""" + monkeypatch.setenv(_FG_SECRET_NAME, _FG_SECRET_VALUE) + advisor = _env_probe_advisor(tmp_path, _FG_SECRET_NAME) + monkeypatch.setattr(advisors, "resolve", lambda _name: advisor) + + code = main( + [ + "--agent", + "fakefg", + "--prompt", + "hi", + "--cwd", + str(tmp_path), + "--pass-env", + _FG_SECRET_NAME, + ] + ) + + assert code == 0 + assert (tmp_path / "fg_env_probe.txt").read_text() == _FG_SECRET_VALUE + + +def test_foreground_secret_value_absent_from_output(tmp_path, monkeypatch, capsys): + """No credential VALUE appears in the foreground path's stdout/stderr — its + only output surface (the session registry stores no env) (HIGH 3).""" + monkeypatch.setenv(_FG_SECRET_NAME, _FG_SECRET_VALUE) + advisor = _env_probe_advisor(tmp_path, _FG_SECRET_NAME) + monkeypatch.setattr(advisors, "resolve", lambda _name: advisor) + + main(["--agent", "fakefg", "--prompt", "hi", "--cwd", str(tmp_path)]) + + captured = capsys.readouterr() + assert _FG_SECRET_VALUE not in captured.out + assert _FG_SECRET_VALUE not in captured.err + + def test_version_flag_prints_version_and_exits(capsys): with pytest.raises(SystemExit) as excinfo: main(["--version"]) From d9c6313c78f959bddba48b6bc1027cdf02fe2483 Mon Sep 17 00:00:00 2001 From: Dat Date: Mon, 27 Jul 2026 06:07:56 +0700 Subject: [PATCH 12/15] style(verify,escalate): replace forbidden placeholder names obj and info --- src/crossagent/escalate.py | 4 ++-- src/crossagent/verify.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/crossagent/escalate.py b/src/crossagent/escalate.py index 92e201c..903b2f9 100644 --- a/src/crossagent/escalate.py +++ b/src/crossagent/escalate.py @@ -264,7 +264,7 @@ def _write_child_command( verify_model: Optional[str], escalate_to: list[str], ) -> None: - info = { + command_payload = { "command": _build_child_argv(advisor, model), "prompt_delivery": advisor.prompt_delivery, "cwd": cwd, @@ -286,7 +286,7 @@ def _write_child_command( "verify_model": verify_model, "escalate_to": escalate_to, } - jobs_mod.atomic_json_write(info, child_dir / "command.json") + jobs_mod.atomic_json_write(command_payload, child_dir / "command.json") def _audit_skip(job_dir: Path, *, reason: str) -> None: diff --git a/src/crossagent/verify.py b/src/crossagent/verify.py index 0131451..83b63d8 100644 --- a/src/crossagent/verify.py +++ b/src/crossagent/verify.py @@ -244,10 +244,10 @@ def _json_candidates(text: str) -> list[str]: return candidates -def _verdict_from_object(obj: dict[str, Any]) -> tuple[VerifyVerdict, str]: +def _verdict_from_object(verdict_object: dict[str, Any]) -> tuple[VerifyVerdict, str]: """Map a parsed verdict object to a ``(verdict, detail)`` pair.""" - raw = obj.get("verdict") - reason = obj.get("reason") + raw = verdict_object.get("verdict") + reason = verdict_object.get("reason") detail = str(reason) if isinstance(reason, str) and reason else "" if isinstance(raw, bool): return ("pass" if raw else "fail"), detail @@ -380,8 +380,8 @@ def run_verification( parsed.error or "verifier produced no answer", ) - obj = _parse_verdict_object(parsed.result) - if obj is None: + verdict_object = _parse_verdict_object(parsed.result) + if verdict_object is None: # Free prose with no machine-checkable verdict: inconclusive, not a pass. return VerifyOutcome( advisor.name, @@ -390,5 +390,5 @@ def run_verification( structured, "verifier returned no machine-checkable verdict (free prose)", ) - verdict, detail = _verdict_from_object(obj) + verdict, detail = _verdict_from_object(verdict_object) return VerifyOutcome(advisor.name, model, verdict, structured, detail) From fea3e17f046806168c05dcb4e6561155334ae6a1 Mon Sep 17 00:00:00 2001 From: Dat Date: Mon, 27 Jul 2026 06:08:37 +0700 Subject: [PATCH 13/15] refactor(verify): extract _interpret_run_outcome from run_verification --- src/crossagent/verify.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/crossagent/verify.py b/src/crossagent/verify.py index 83b63d8..af08790 100644 --- a/src/crossagent/verify.py +++ b/src/crossagent/verify.py @@ -357,7 +357,22 @@ def run_verification( structured, f"verifier failed to launch: {exc}", ) + return _interpret_run_outcome(advisor.name, model, structured, outcome, timeout) + +def _interpret_run_outcome( + advisor_name: str, + model: Optional[str], + structured: bool, + outcome: runner_mod.RunOutcome, + timeout: float, +) -> VerifyOutcome: + """Map a completed runner outcome to a ``VerifyOutcome``. + + A timeout or a launch/parse failure degrades to ``error``; free prose with no + parseable verdict is ``unverified`` (inconclusive, never a pass); only a + machine-checkable object yields the graded ``pass``/``fail`` verdict. + """ parsed = ( outcome.result if isinstance(outcome.result, parsers_mod.ParsedResult) @@ -365,7 +380,7 @@ def run_verification( ) if outcome.timed_out: return VerifyOutcome( - advisor.name, + advisor_name, model, "error", structured, @@ -373,7 +388,7 @@ def run_verification( ) if parsed.failure or parsed.result is None: return VerifyOutcome( - advisor.name, + advisor_name, model, "error", structured, @@ -384,11 +399,11 @@ def run_verification( if verdict_object is None: # Free prose with no machine-checkable verdict: inconclusive, not a pass. return VerifyOutcome( - advisor.name, + advisor_name, model, "unverified", structured, "verifier returned no machine-checkable verdict (free prose)", ) verdict, detail = _verdict_from_object(verdict_object) - return VerifyOutcome(advisor.name, model, verdict, structured, detail) + return VerifyOutcome(advisor_name, model, verdict, structured, detail) From 82910254bbdd41e875a77b10cb8cf5e39edd2777 Mon Sep 17 00:00:00 2001 From: Dat Date: Mon, 27 Jul 2026 06:13:29 +0700 Subject: [PATCH 14/15] refactor(escalate): extract eligibility guard, child Job build, and disk staging from maybe_escalate --- src/crossagent/escalate.py | 199 +++++++++++++++++++++++++------------ 1 file changed, 136 insertions(+), 63 deletions(-) diff --git a/src/crossagent/escalate.py b/src/crossagent/escalate.py index 903b2f9..7cf1f19 100644 --- a/src/crossagent/escalate.py +++ b/src/crossagent/escalate.py @@ -116,26 +116,7 @@ def maybe_escalate( raises: an escalation that cannot be launched is recorded and skipped, never allowed to crash the worker. """ - # Escalation re-dispatches a delegate that FINISHED but whose work FAILED a - # declared gate — a failing check, a scope violation, or a failing - # verification (this module's stated scope). A job that did not finish - # cleanly is deliberately out of scope, so gate on SUCCEEDED first rather - # than on ``delegation_verdict != "failed"`` alone (which also returns - # "failed" for CANCELLED / TIMED_OUT / a crashed delegate): - # * CANCELLED is explicit user intent to stop; re-dispatching to a larger, - # costlier peer is the opposite of cancelling and spends real money. - # * TIMED_OUT (and any other non-success terminal status) produced no - # graded artifact — there is no gate failure to escalate, and a bigger - # model is generally slower, so it is at least as likely to time out - # again under the same budget. A hard task that needs a bigger model is a - # fresh dispatch decision, not an automatic ladder climb that silently - # burns budget. So TIMED_OUT does NOT escalate. - # Gating on SUCCEEDED means ``delegation_verdict == "failed"`` below can only - # be a declared-gate failure — mirroring cli._failed_reason's "did not finish - # cleanly" vs. gate-failure distinction. - if failed_job.status != jobs_mod.JobState.SUCCEEDED: - return None - if delegation_verdict(failed_job) != "failed": + if not _is_escalatable_failure(failed_job): return None rungs = parse_rungs(escalate_to) @@ -154,7 +135,7 @@ def maybe_escalate( child_id = jobs_mod.generate_job_id() try: - parent_id, trace_id, label, depth = jobs_mod.resolve_lineage( + lineage = jobs_mod.resolve_lineage( parent_flag=failed_job.job_id, state_root=state_root, new_job_id=child_id, @@ -165,50 +146,28 @@ def maybe_escalate( _audit_skip(job_dir, reason=f"escalation halted: {exc}") return None - # Staging the child on disk touches the filesystem (mkdir, two file writes, - # a state save). A read-only or full disk raises OSError; catch it here so - # the "never raises" contract holds — a child that cannot be staged is - # recorded and skipped, exactly like a launch failure below. - try: - child_dir = jobs_mod.create_job_dir(state_root, child_id) - _write_child_prompt(child_dir, prompt) - _write_child_command( - child_dir, - advisor=advisor, - model=model, - cwd=cwd, - registry_path=registry_path, - check=check, - check_timeout=check_timeout, - scope_paths=scope_paths, - pass_env=pass_env, - verify_with=verify_with, - verify_model=verify_model, - escalate_to=remaining, - ) - child = Job( - job_id=child_id, - status=jobs_mod.JobState.PENDING, - advisor=advisor.name, - name="", - cwd=cwd, - redacted_command="", - started_at=_now(), - updated_at=_now(), - last_activity_at=_now(), - last_event="escalation.created", - max_runtime_seconds=failed_job.max_runtime_seconds, - termination_grace_seconds=failed_job.termination_grace_seconds, - parent_job_id=parent_id, - trace_id=trace_id, - orchestrator_label=label, - nesting_depth=depth, - ) - jobs_mod.save_state(child_dir, child) - except OSError as exc: - _audit_skip(job_dir, reason=f"escalation could not be staged: {exc}") + if not _create_and_save_child( + state_root=state_root, + job_dir=job_dir, + child_id=child_id, + prompt=prompt, + advisor=advisor, + model=model, + cwd=cwd, + registry_path=registry_path, + check=check, + check_timeout=check_timeout, + scope_paths=scope_paths, + pass_env=pass_env, + verify_with=verify_with, + verify_model=verify_model, + escalate_to=remaining, + lineage=lineage, + failed_job=failed_job, + ): return None + _, trace_id, _, depth = lineage jobs_mod.append_event( job_dir, "escalation", @@ -234,6 +193,33 @@ def maybe_escalate( # --------------------------------------------------------------------------- +def _is_escalatable_failure(failed_job: Job) -> bool: + """Return True only for a delegate that FINISHED but FAILED a declared gate. + + Escalation re-dispatches work that failed a check, a scope violation, or a + failing verification (this module's stated scope). A job that did not finish + cleanly is deliberately out of scope, so gate on SUCCEEDED first rather than + on ``delegation_verdict != "failed"`` alone (which also returns "failed" for + CANCELLED / TIMED_OUT / a crashed delegate): + + * CANCELLED is explicit user intent to stop; re-dispatching to a larger, + costlier peer is the opposite of cancelling and spends real money. + * TIMED_OUT (and any other non-success terminal status) produced no graded + artifact — there is no gate failure to escalate, and a bigger model is + generally slower, so it is at least as likely to time out again under the + same budget. A hard task that needs a bigger model is a fresh dispatch + decision, not an automatic ladder climb that silently burns budget. So + TIMED_OUT does NOT escalate. + + Gating on SUCCEEDED means ``delegation_verdict == "failed"`` can only be a + declared-gate failure — mirroring cli._failed_reason's "did not finish + cleanly" vs. gate-failure distinction. + """ + if failed_job.status != jobs_mod.JobState.SUCCEEDED: + return False + return delegation_verdict(failed_job) == "failed" + + def _drop_first_nonblank(rungs: Optional[list[str]]) -> list[str]: """Return the non-blank rungs with the first one (the spawned rung) removed.""" cleaned = [raw for raw in (rungs or []) if raw.strip()] @@ -289,6 +275,93 @@ def _write_child_command( jobs_mod.atomic_json_write(command_payload, child_dir / "command.json") +def _build_child_job( + child_id: str, + advisor: Advisor, + cwd: str, + lineage: tuple[Optional[str], str, Optional[str], Optional[int]], + failed_job: Job, +) -> Job: + """Assemble the PENDING child ``Job`` record for an escalation re-dispatch. + + *lineage* is the ``(parent_id, trace_id, label, depth)`` tuple returned by + :func:`~crossagent.jobs.resolve_lineage`; runtime bounds are inherited from + *failed_job* so the larger peer runs under the same limits. + """ + parent_id, trace_id, label, depth = lineage + return Job( + job_id=child_id, + status=jobs_mod.JobState.PENDING, + advisor=advisor.name, + name="", + cwd=cwd, + redacted_command="", + started_at=_now(), + updated_at=_now(), + last_activity_at=_now(), + last_event="escalation.created", + max_runtime_seconds=failed_job.max_runtime_seconds, + termination_grace_seconds=failed_job.termination_grace_seconds, + parent_job_id=parent_id, + trace_id=trace_id, + orchestrator_label=label, + nesting_depth=depth, + ) + + +def _create_and_save_child( + *, + state_root: Path, + job_dir: Path, + child_id: str, + prompt: str, + advisor: Advisor, + model: Optional[str], + cwd: str, + registry_path: str, + check: Optional[str], + check_timeout: float, + scope_paths: Optional[list[str]], + pass_env: list[str], + verify_with: Optional[str], + verify_model: Optional[str], + escalate_to: list[str], + lineage: tuple[Optional[str], str, Optional[str], Optional[int]], + failed_job: Job, +) -> bool: + """Stage the escalated child on disk: prompt, command, and state record. + + Staging touches the filesystem (mkdir, two file writes, a state save). A + read-only or full disk raises OSError; it is caught here so the caller's + "never raises" contract holds — a child that cannot be staged is recorded on + *job_dir* and skipped, exactly like a launch failure. Returns True on + success, False when staging was skipped. + """ + try: + child_dir = jobs_mod.create_job_dir(state_root, child_id) + _write_child_prompt(child_dir, prompt) + _write_child_command( + child_dir, + advisor=advisor, + model=model, + cwd=cwd, + registry_path=registry_path, + check=check, + check_timeout=check_timeout, + scope_paths=scope_paths, + pass_env=pass_env, + verify_with=verify_with, + verify_model=verify_model, + escalate_to=escalate_to, + ) + child = _build_child_job(child_id, advisor, cwd, lineage, failed_job) + jobs_mod.save_state(child_dir, child) + except OSError as exc: + _audit_skip(job_dir, reason=f"escalation could not be staged: {exc}") + return False + return True + + def _audit_skip(job_dir: Path, *, reason: str) -> None: jobs_mod.append_event( job_dir, "escalation_skipped", actor="system:escalate", reason=reason From c6a81ea62c5359b02c3d615842a0a970ecb44157 Mon Sep 17 00:00:00 2001 From: Dat Date: Mon, 27 Jul 2026 06:19:24 +0700 Subject: [PATCH 15/15] refactor(types): move gate-result shapes to dependency-free types module Relocates CheckResultDict/ScopeResultDict/ScopeStatus/VerifyResultDict/VerifyVerdict out of jobs.py into a new dependency-free types.py. This removes the inverted back-import where the gate producers (check/scope/verify) imported their own persisted-record shapes from their consumer (jobs). jobs re-exports the three it uses as Job field annotations for jobs_mod.* callers (worker). Behaviour unchanged; 513 tests green. --- src/crossagent/check.py | 2 +- src/crossagent/jobs.py | 92 ++++--------------------------------- src/crossagent/scope.py | 2 +- src/crossagent/types.py | 97 ++++++++++++++++++++++++++++++++++++++++ src/crossagent/verify.py | 2 +- 5 files changed, 108 insertions(+), 87 deletions(-) create mode 100644 src/crossagent/types.py diff --git a/src/crossagent/check.py b/src/crossagent/check.py index 8845b84..511aed9 100644 --- a/src/crossagent/check.py +++ b/src/crossagent/check.py @@ -23,7 +23,7 @@ from dataclasses import dataclass from typing import Optional -from .jobs import CheckResultDict +from .types import CheckResultDict # Keep only a bounded tail of the check's output — a full test run can emit # megabytes, and the whole Job record is loaded into memory (and the dashboard). diff --git a/src/crossagent/jobs.py b/src/crossagent/jobs.py index 358efe9..cc53b6e 100644 --- a/src/crossagent/jobs.py +++ b/src/crossagent/jobs.py @@ -16,7 +16,14 @@ from datetime import datetime, timezone from enum import Enum from pathlib import Path -from typing import Any, Callable, Literal, Optional, TypedDict +from typing import Any, Callable, Literal, Optional + +# The gate-result shapes now live in the dependency-free ``types`` module. +# ``jobs`` uses these three as ``Job`` field annotations, which also re-exports +# them for callers that still reference ``jobs_mod.CheckResultDict`` (e.g. +# ``worker``). ``ScopeStatus``/``VerifyVerdict`` are imported straight from +# ``types`` by ``scope``/``verify``. +from .types import CheckResultDict, ScopeResultDict, VerifyResultDict # Where a job's cost figure came from. ``advisor`` is the vendor-declared # estimate, ``computed`` is derived from a local price table, and ``unknown`` @@ -24,89 +31,6 @@ CostSource = Literal["advisor", "computed", "unknown"] -class CheckResultDict(TypedDict): - """Persisted outcome of the independent check-gate (slice S3). - - ``command`` is the caller-supplied check string; ``exit_code`` is the - deterministic delegation verdict (D5) — ``0`` means the work verified, any - non-zero value means the delegation *failed* regardless of whether the - delegate process itself exited 0. ``stdout_tail``/``stderr_tail`` are - bounded tails of the check's output. - - A ``Job.check_result`` of ``None`` means *no check ran* (unverified) — that - stays structurally distinct from a check that ran and failed (D7): absent is - never the same as false. - """ - - command: str - exit_code: int - stdout_tail: str - stderr_tail: str - - -# Outcome of the diff-scope assertion (slice S4). ``ok`` — every path the -# delegate modified was inside the declared allowlist; ``violated`` — it wrote -# outside its declared scope; ``undetermined`` — crossagent could not establish -# what changed (e.g. the cwd is not a git repo). ``undetermined`` is FAIL-CLOSED, -# never a pass: a scope check that silently passes when it cannot see the changes -# grants false assurance. -ScopeStatus = Literal["ok", "violated", "undetermined"] - - -class ScopeResultDict(TypedDict): - """Persisted outcome of the diff-scope assertion (slice S4). - - ``declared`` is the caller-supplied allowlist; ``violating_paths`` lists the - repo-relative paths the delegate modified outside it (empty unless - ``status == "violated"``); ``detail`` is a human-readable summary. - - A ``Job.scope_result`` of ``None`` means *no scope was declared* — scope - enforcement was off — which stays structurally distinct from a scope that - was declared and satisfied (D7: absent is never the same as ``ok``). - """ - - declared: list[str] - status: ScopeStatus - violating_paths: list[str] - detail: str - - -# Outcome of the independent verification pass (slice S5). A FRESH peer session -# grades the delegate's artifact supplied as user-turn input (D6), which removes -# the implicit-authorship channel that weakens self-grading — it does NOT claim -# to eliminate self-preference bias, which is a separate documented effect. -# ``pass`` — the verifier returned a machine-checkable verdict of correct. -# ``fail`` — the verifier returned a machine-checkable verdict of wrong. -# ``unverified`` — the verifier ran but produced no machine-checkable verdict -# (free prose, or the advisor lacks a structured-output -# contract). Treated as inconclusive, never as a pass (D4). -# ``error`` — the verifier could not run (no artifact, launch failure). -# ``unverified``/``error`` are inconclusive: they never green a delegation and -# never hard-fail it. Only ``fail`` blocks the green path. -VerifyVerdict = Literal["pass", "fail", "unverified", "error"] - - -class VerifyResultDict(TypedDict): - """Persisted outcome of the independent verification pass (slice S5). - - ``advisor``/``model`` identify the FRESH peer session that graded the work. - ``verdict`` is the deterministic pass/fail/inconclusive outcome; ``structured`` - records whether a machine-checkable contract (e.g. Claude ``--json-schema`` → - ``structured_output``) produced it, versus a JSON object parsed out of a prose - answer. ``detail`` is a human-readable summary (never the raw artifact). - - A ``Job.verify_result`` of ``None`` means *no verification was requested* — - structurally distinct from a verification that ran and failed, or ran and - could not decide (D7: absent is never the same as ``fail`` or ``unverified``). - """ - - advisor: str - model: Optional[str] - verdict: VerifyVerdict - structured: bool - detail: str - - # The delegation verdict keeps "the delegate finished" separate from "the work # was verified" (D5). See :func:`delegation_verdict`. DelegationVerdict = Literal["verified", "failed", "unverified", "incomplete"] diff --git a/src/crossagent/scope.py b/src/crossagent/scope.py index 35e53fd..70aa33a 100644 --- a/src/crossagent/scope.py +++ b/src/crossagent/scope.py @@ -47,7 +47,7 @@ from pathlib import Path from typing import Optional -from .jobs import ScopeResultDict, ScopeStatus +from .types import ScopeResultDict, ScopeStatus _GIT_TIMEOUT_SECONDS = 30.0 # porcelain -z entries are "XY ": two status chars, a space, then the path. diff --git a/src/crossagent/types.py b/src/crossagent/types.py new file mode 100644 index 0000000..e1e2789 --- /dev/null +++ b/src/crossagent/types.py @@ -0,0 +1,97 @@ +"""Shared persisted-record shapes for the delegation gates. + +These ``TypedDict``/``Literal`` definitions describe how the check (S3), scope +(S4), and verification (S5) gate outcomes are serialized into a ``Job`` record. +They live in their own dependency-free module so the gate modules (``check.py``, +``scope.py``, ``verify.py``) can name the persisted shape they produce WITHOUT +importing from ``jobs.py`` — their consumer — which would be an inverted +dependency. ``jobs.py`` imports these for its ``Job`` fields; this module imports +nothing from the package, so no import cycle is possible. +""" + +from __future__ import annotations + +from typing import Literal, Optional, TypedDict + + +class CheckResultDict(TypedDict): + """Persisted outcome of the independent check-gate (slice S3). + + ``command`` is the caller-supplied check string; ``exit_code`` is the + deterministic delegation verdict (D5) — ``0`` means the work verified, any + non-zero value means the delegation *failed* regardless of whether the + delegate process itself exited 0. ``stdout_tail``/``stderr_tail`` are + bounded tails of the check's output. + + A ``Job.check_result`` of ``None`` means *no check ran* (unverified) — that + stays structurally distinct from a check that ran and failed (D7): absent is + never the same as false. + """ + + command: str + exit_code: int + stdout_tail: str + stderr_tail: str + + +# Outcome of the diff-scope assertion (slice S4). ``ok`` — every path the +# delegate modified was inside the declared allowlist; ``violated`` — it wrote +# outside its declared scope; ``undetermined`` — crossagent could not establish +# what changed (e.g. the cwd is not a git repo). ``undetermined`` is FAIL-CLOSED, +# never a pass: a scope check that silently passes when it cannot see the changes +# grants false assurance. +ScopeStatus = Literal["ok", "violated", "undetermined"] + + +class ScopeResultDict(TypedDict): + """Persisted outcome of the diff-scope assertion (slice S4). + + ``declared`` is the caller-supplied allowlist; ``violating_paths`` lists the + repo-relative paths the delegate modified outside it (empty unless + ``status == "violated"``); ``detail`` is a human-readable summary. + + A ``Job.scope_result`` of ``None`` means *no scope was declared* — scope + enforcement was off — which stays structurally distinct from a scope that + was declared and satisfied (D7: absent is never the same as ``ok``). + """ + + declared: list[str] + status: ScopeStatus + violating_paths: list[str] + detail: str + + +# Outcome of the independent verification pass (slice S5). A FRESH peer session +# grades the delegate's artifact supplied as user-turn input (D6), which removes +# the implicit-authorship channel that weakens self-grading — it does NOT claim +# to eliminate self-preference bias, which is a separate documented effect. +# ``pass`` — the verifier returned a machine-checkable verdict of correct. +# ``fail`` — the verifier returned a machine-checkable verdict of wrong. +# ``unverified`` — the verifier ran but produced no machine-checkable verdict +# (free prose, or the advisor lacks a structured-output +# contract). Treated as inconclusive, never as a pass (D4). +# ``error`` — the verifier could not run (no artifact, launch failure). +# ``unverified``/``error`` are inconclusive: they never green a delegation and +# never hard-fail it. Only ``fail`` blocks the green path. +VerifyVerdict = Literal["pass", "fail", "unverified", "error"] + + +class VerifyResultDict(TypedDict): + """Persisted outcome of the independent verification pass (slice S5). + + ``advisor``/``model`` identify the FRESH peer session that graded the work. + ``verdict`` is the deterministic pass/fail/inconclusive outcome; ``structured`` + records whether a machine-checkable contract (e.g. Claude ``--json-schema`` → + ``structured_output``) produced it, versus a JSON object parsed out of a prose + answer. ``detail`` is a human-readable summary (never the raw artifact). + + A ``Job.verify_result`` of ``None`` means *no verification was requested* — + structurally distinct from a verification that ran and failed, or ran and + could not decide (D7: absent is never the same as ``fail`` or ``unverified``). + """ + + advisor: str + model: Optional[str] + verdict: VerifyVerdict + structured: bool + detail: str diff --git a/src/crossagent/verify.py b/src/crossagent/verify.py index af08790..4c269bd 100644 --- a/src/crossagent/verify.py +++ b/src/crossagent/verify.py @@ -56,7 +56,7 @@ from . import parsers as parsers_mod from . import runner as runner_mod from .advisors import Advisor -from .jobs import VerifyResultDict, VerifyVerdict +from .types import VerifyResultDict, VerifyVerdict # A verification that never terminates must not hang the worker forever. VERIFY_DEFAULT_TIMEOUT_SECONDS = 600.0