diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 0000000..cc6ed48 --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,69 @@ +name: check +# Dogfood the root action (action.yml): `uses: ./` with the probbit built from this commit, on Linux and macOS. +on: + push: + branches: [main] + pull_request: +permissions: + contents: read +jobs: + dogfood: + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + - name: build probbit + run: cargo build --release --locked -p probbit-cli + - name: a strand of the dispatch policy (the receipts the check replays) + shell: bash + run: | + mkdir -p receipts + printf '%s\n' '{"blast":0.2}' '{"error":true}' '{"error":true,"criticism":true,"src":"human"}' '{"praise":true,"src":"env"}' '{"blast":0.9}' \ + | target/release/probbit live examples/persona/dispatch-policy.yaml --seed 2 --clock fixed --strand receipts/day1.strand > /dev/null + - name: the rule lock on the dispatch example + id: dispatch + uses: ./ + with: + persona: examples/persona/dispatch-policy.yaml + props: examples/persona/dispatch-props.json + strands: | + receipts/*.strand + binary: target/release/probbit + - name: the shipped tutor (habit contradictions) + uses: ./ + with: + persona: examples/persona/tutor.yaml + binary: target/release/probbit + - name: the outputs + shell: bash + run: | + test "${{ steps.dispatch.outputs.exit-code }}" = 0 + jq -e '.verdict == {"broken":0,"conflicts":0,"exit":0,"held":3,"strands_ok":1,"strands_total":1,"unproven":0}' "${{ steps.dispatch.outputs.json }}" + install-gate: + # the install path: install.sh fetches a release; probbit 0.8.2 has no check command, so the action must stop with one error + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7 + - name: the action with probbit 0.8.2 + id: old + continue-on-error: true + uses: ./ + with: + persona: examples/persona/tutor.yaml + version: v0.8.2 + json-artifact: 'false' + - name: it stopped at the version gate + shell: bash + run: | + test "${{ steps.old.outcome }}" = failure + "$RUNNER_TEMP/probbit-bin/probbit" version | grep -qx 'probbit 0.8.2' diff --git a/CHANGELOG.md b/CHANGELOG.md index 97b23e6..e05fa4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,29 @@ All notable changes to this project are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## 0.8.3 - unreleased + +### Added +- `probbit check PERSONA [--props FILE] [--strand FILE]... [--seeds LIST] [--depth N] [--hours LIST] [--json]`: one gate for + CI. It runs `persona lint`'s pipeline on the rules (prove each one, then fuzz the ones the bound leaves unknown, the rules' + and habits' thresholds on the grid) and `live verify` on every strand, then prints one line per rule (`held`, + `held_by_construction`, `broken seed N`, `unproven`), one per strand (`ok N events` / `FAIL line K: ...`) and + `check: held H · broken B · unproven U · strands S/T`. Exit 0 all held and replayed; 1 a rule broken, an unresolved habit + contradiction or a failing strand (1 wins over 3); 3 only unproven rules; 2 bad input. `--json`: one document with the + `persona lint` report under `rules`, the strands and the verdict counts. No engine change. +- `action.yml`: the repository is a composite GitHub Action (`uses: BitmapAsset/probbit@`) that installs probbit with + `install.sh` (input `version`, or `binary` for a prebuilt one), runs `probbit check --json`, annotates broken rules and + failing strands as errors and unproven rules as warnings with their seeds, uploads `probbit-check.json` + (`actions/upload-artifact@v4`) and passes exit 3 with a warning when `allow-unproven` is true. Linux and macOS runners; + Windows runners stop with one error. `.github/workflows/check.yml` runs it on every push and pull request. +- `examples/persona/dispatch-policy.yaml` + `dispatch-props.json`: the rule lock on an agent that learns whether to act or + ask first. Every rule is held by construction for seeds 0-99; without its routine habit, `check` finds the broken seeds + (README, "Check in CI"). + +### Fixed +- `cargo +1.78.0 test --workspace --locked` compiles again: a CLI test used `std::io::pipe()` (Rust 1.87); it now gets a + readerless pipe from a child process's stdin. The build MSRV was unaffected. + ## 0.8.2 - 2026-10-14 ### Added diff --git a/README.md b/README.md index b07a8f4..65db538 100644 --- a/README.md +++ b/README.md @@ -482,6 +482,76 @@ Python `probbit.persona_fuzz` / `persona_prove` give the same documents. Both co words a model writes; "none found" from `fuzz` is evidence over the scripts it searched, not a proof ([docs/persona.md](docs/persona.md) §5.6 has the bound and its limits). +### Check in CI: `probbit check` + +`probbit check` is the rule lock in one command: vet the rules a persona must keep before its first task and after every +policy update, then replay the receipts (the strands it wrote). It runs `persona lint`'s pipeline on `--props` (prove each +rule, fuzz the unknown ones) and `live verify` on every `--strand`: + +```sh +probbit check examples/persona/dispatch-policy.yaml --props examples/persona/dispatch-props.json --strand day1.strand +# rule risky_work_asks held_by_construction +# rule repeated_failures_ask held_by_construction +# rule routine_work_acts held_by_construction +# strand day1.strand ok 5 events +# check: held 3 · broken 0 · unproven 0 · strands 1/1 +``` + +Exit 0 when every rule is held and every strand replays; 1 when a rule is broken, a habit contradiction is unresolved or a +strand fails (1 wins); 3 when the only problem is an unproven rule (the bound cannot decide it and the search found no +counterexample); 2 for a bad persona, rule file or flag. `--json` prints one document with the `persona lint` report under +`rules`, one entry per strand and the verdict counts. + +**The rule lock, on the shipped example.** `examples/persona/dispatch-policy.yaml` is an agent that learns from feedback +whether to act on a request or ask a person first; `dispatch-props.json` holds the three rules it must keep (ask before +risky work, ask after two failures in a row, act on routine work). With its `routine_acts` habit, every rule is +`held_by_construction` for seeds 0-99 (exit 0, 0.02 s on an Apple M4). Delete that habit, as a policy update might, and the +same command catches it before the first task: + +```sh +probbit check dispatch-without-routine-habit.yaml --props examples/persona/dispatch-props.json +# rule risky_work_asks held_by_construction +# rule repeated_failures_ask held_by_construction +# rule routine_work_acts broken seeds 3,6,11,15-17,21,36-37,46,49,57,77,83,85 +# check: held 2 · broken 1 · unproven 0 · strands 0/0 (exit 1, 0.67 s) +``` + +The bound proves `routine_work_acts` for 76 of the 100 individuals and leaves 24 unknown; the search breaks it on 15 of +those 24 (9 of them with a single event, so before any learning). The other 9 it cannot break at depth 8: checked alone +(`--seeds 18,20,28,30-31,79,87,92,96`) they report `unproven` and exit 3, which fails the job unless you allow it. Measured +on this file with probbit 0.8.3 (unreleased), seeds 0-99, default depth. + +**As a GitHub Action.** The repository root is a composite action that installs probbit with `install.sh`, runs `probbit +check --json`, annotates each broken rule and failing strand as an error and each unproven rule as a warning (with the seed +numbers), and uploads the JSON report as an artifact: + +```yaml +- uses: actions/checkout@v7 +- uses: BitmapAsset/probbit@v0.8.3 # the action needs probbit 0.8.3 or later + with: + persona: agent/policy.yaml + props: agent/rules.json + strands: | + receipts/*.strand + seeds: 0-99 +``` + +| input | default | | +|---|---|---| +| `persona` | (required) | the persona file | +| `props` | | the rules file; empty = habit contradictions and strands only | +| `strands` | | strands to replay, one path or glob per line; a glob that matches nothing fails the check | +| `seeds` | `0-99` | the individuals | +| `depth` | probbit's (8) | events per fuzzed script | +| `version` | `latest` | the release `install.sh` installs (`v0.8.3`, ...) | +| `binary` | | a probbit binary to use instead (one built earlier in the job); overrides `version` | +| `allow-unproven` | `false` | `true`: exit 3 is a warning and the job passes | +| `json-artifact` | `true` | upload `probbit-check.json` (`actions/upload-artifact@v4`, the only action it uses) | + +Outputs: `exit-code` and `json` (the report's path). Linux and macOS runners work (this repository runs it on `ubuntu-latest` +and `macos-latest`); jq, which the action uses, is on GitHub's images. Windows runners stop at once with one error line: use `ubuntu-latest` or `macos-latest` for this step +(`install.ps1` installs probbit on Windows for running `probbit check` by hand). + ### Live: an individual on a clock, learning inside caps `probbit live` keeps one individual running: JSONL events in, one stance per event out. A clock stamps the hours between events, diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..6299411 --- /dev/null +++ b/action.yml @@ -0,0 +1,151 @@ +name: probbit check +description: Vet a persona's rules (prove, then fuzz the unknown ones) and replay its strands; fail the job on a broken rule or strand. +author: BitmapAsset +branding: + icon: check-circle + color: green +inputs: + persona: + description: The persona file (YAML subset or JSON), relative to the workspace. + required: true + props: + description: 'The rules file ({"props": [...]}; YAML subset or .json). Empty = only the habit contradictions and the strands are checked.' + default: '' + strands: + description: Strands to replay, one path or glob per line (relative to the workspace). A glob that matches nothing fails the check. + default: '' + seeds: + description: The individuals, e.g. 0-99, 7, 1,4,9 or 0-9,20-29. + default: '0-99' + depth: + description: Events per fuzzed script, 1 to 64 (empty = probbit's default, 8). + default: '' + version: + description: The probbit release to install with install.sh (latest, or a tag such as v0.8.3). probbit check needs 0.8.3 or later. + default: latest + binary: + description: Path to a probbit binary to use instead of installing one (for example one built earlier in the job). Overrides version. + default: '' + allow-unproven: + description: "false: an unproven rule (exit 3) fails the job. true: it is a warning and the job passes." + default: 'false' + json-artifact: + description: "true: upload the --json document as an artifact (probbit-check.json)." + default: 'true' +outputs: + exit-code: + description: probbit check's exit code (0 held, 1 broken or a failing strand, 2 bad input, 3 unproven). + value: ${{ steps.check.outputs.exit-code }} + json: + description: Path of the --json document. + value: ${{ steps.check.outputs.json }} +runs: + using: composite + steps: + - name: install probbit + id: install + shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + INPUT_BINARY: ${{ inputs.binary }} + run: | + set -euo pipefail + if [ "$RUNNER_OS" = "Windows" ]; then + echo "::error title=probbit check::Windows runners are not supported by this action yet; run the job on ubuntu-latest or macos-latest" + exit 1 + fi + if [ -n "$INPUT_BINARY" ]; then + bin="$INPUT_BINARY" + [ -f "$bin" ] && [ -x "$bin" ] || { echo "::error title=probbit check::binary: $bin is not an executable file"; exit 1; } + else + dir="$RUNNER_TEMP/probbit-bin" + version="$INPUT_VERSION"; [ "$version" = "latest" ] && version="" + PROBBIT_VERSION="$version" PROBBIT_INSTALL_DIR="$dir" PROBBIT_MAN_DIR="$RUNNER_TEMP/probbit-man" sh "$GITHUB_ACTION_PATH/install.sh" + bin="$dir/probbit" + fi + bin="$(cd "$(dirname "$bin")" && pwd)/$(basename "$bin")" + if ! "$bin" check --help > /dev/null 2>&1; then + echo "::error title=probbit check::$("$bin" version 2>&1) has no check command: it needs probbit 0.8.3 or later (set the version input)" + exit 1 + fi + command -v jq > /dev/null || { echo "::error title=probbit check::jq is not on this runner (it reads the --json document)"; exit 1; } + echo "$("$bin" version)" + echo "bin=$bin" >> "$GITHUB_OUTPUT" + + - name: probbit check + id: check + shell: bash + env: + PROBBIT: ${{ steps.install.outputs.bin }} + INPUT_PERSONA: ${{ inputs.persona }} + INPUT_PROPS: ${{ inputs.props }} + INPUT_STRANDS: ${{ inputs.strands }} + INPUT_SEEDS: ${{ inputs.seeds }} + INPUT_DEPTH: ${{ inputs.depth }} + JOB: ${{ github.job }} + run: | + set +e -uo pipefail # GitHub runs bash with -e: the exit code is read below + args=(check "$INPUT_PERSONA" --seeds "$INPUT_SEEDS" --json) + [ -n "$INPUT_PROPS" ] && args+=(--props "$INPUT_PROPS") + [ -n "$INPUT_DEPTH" ] && args+=(--depth "$INPUT_DEPTH") + nomatch=0 + shopt -s nullglob + while IFS= read -r pat || [ -n "$pat" ]; do + pat="${pat#"${pat%%[![:space:]]*}"}"; pat="${pat%"${pat##*[![:space:]]}"}" + [ -z "$pat" ] && continue + IFS=$'\n'; matches=($pat); unset IFS + if [ "${#matches[@]}" -eq 0 ]; then echo "::error title=probbit check::strands: no file matches $pat"; nomatch=1; continue; fi + for f in "${matches[@]}"; do args+=(--strand "$f"); done + done <<< "$INPUT_STRANDS" + shopt -u nullglob + out="$(mktemp -d "$RUNNER_TEMP/probbit-check.XXXXXX")/probbit-check.json" # one directory per call: a job may run several + echo "probbit ${args[*]}" + "$PROBBIT" "${args[@]}" > "$out"; code=$? + name="probbit-check-$JOB-$RUNNER_OS-$(printf '%s' "$INPUT_PERSONA" | tr -c 'A-Za-z0-9._-' '-')" + echo "json=$out" >> "$GITHUB_OUTPUT"; echo "artifact=$name" >> "$GITHUB_OUTPUT" + if [ "$code" -eq 2 ] || ! jq -e '.verdict' "$out" > /dev/null 2>&1; then + msg=$(jq -r '.error.message // empty' "$out" 2>/dev/null) + echo "::error title=probbit check::${msg:-probbit check exited $code without a report}" + echo "exit-code=$code" >> "$GITHUB_OUTPUT"; exit 0 + fi + props_file="$INPUT_PROPS"; [ -n "$props_file" ] || props_file="$INPUT_PERSONA" + # one line per rule and strand (as probbit check prints them), and an annotation per broken / unproven rule and failed strand + jq -r --arg props "$props_file" ' + def seeds: [.[] | tostring] | join(","); + (.rules.unproven // []) as $un + | (.rules.props // [])[] + | if .verdict == "held_by_construction" then "rule \(.id) held_by_construction" + elif .verdict == "proved" then "rule \(.id) held" + elif .fuzz.verdict == "counterexample" then + "::error file=\($props),title=probbit check: rule broken::\(.id) is broken on seeds \(.fuzz.failing_seeds | seeds) (shortest counterexample: seed \(.fuzz.shortest.seed), \(.fuzz.shortest.script | length | if . == 1 then "1 event" else "\(.) events" end))\nrule \(.id) broken seeds \(.fuzz.failing_seeds | seeds)" + else (.id as $id | ($un | map(select(.id == $id)) | .[0].seeds_unknown // []) as $s + | "::warning file=\($props),title=probbit check: rule unproven::\(.id) is not proved for seeds \($s | seeds) and the search found no counterexample\nrule \(.id) unproven seeds \($s | seeds)") + end' "$out" + jq -r '.strands[] | if .ok then "strand \(.path) ok \(.events) events" + else "::error file=\(.path),title=probbit check: strand fails::\(.path): \(.error)\nstrand \(.path) FAIL \(.error)" end' "$out" + jq -r 'if .verdict.conflicts > 0 then "::error title=probbit check::\(.verdict.conflicts) unresolved habit contradiction(s): run probbit persona lint \(.persona)" else empty end' "$out" + line=$(jq -r '.verdict | "check: held \(.held) · broken \(.broken) · unproven \(.unproven) · strands \(.strands_ok)/\(.strands_total)"' "$out") + echo "$line"; echo "**probbit check** \`$INPUT_PERSONA\`: $line (exit $code)" >> "$GITHUB_STEP_SUMMARY" + [ "$nomatch" -eq 1 ] && [ "$code" -ne 2 ] && code=1 + echo "exit-code=$code" >> "$GITHUB_OUTPUT" + + - name: upload probbit-check.json + if: inputs.json-artifact == 'true' && steps.check.outputs.json != '' + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.check.outputs.artifact }} + path: ${{ steps.check.outputs.json }} + if-no-files-found: ignore + + - name: verdict + shell: bash + env: + CODE: ${{ steps.check.outputs.exit-code }} + ALLOW_UNPROVEN: ${{ inputs.allow-unproven }} + run: | + case "$CODE" in + 0) echo "probbit check: every rule held, every strand replays" ;; + 3) if [ "$ALLOW_UNPROVEN" = "true" ]; then echo "::warning title=probbit check::unproven rules (allow-unproven: true, so the job passes)" + else echo "::error title=probbit check::unproven rules: not proved and not broken (set allow-unproven: true to pass with a warning)"; exit 3; fi ;; + *) echo "probbit check failed (exit ${CODE:-none})"; exit 1 ;; + esac diff --git a/docs/CHEATSHEET.md b/docs/CHEATSHEET.md index 9e8a984..96e69cb 100644 --- a/docs/CHEATSHEET.md +++ b/docs/CHEATSHEET.md @@ -47,6 +47,8 @@ The exact tiers run before the sampler: - `persona`: 0 done; 1 lint found an unresolved contradiction or (with rules) a counterexample, fuzz found a counterexample or prove left a rule unknown; 2 bad persona / state / inputs / script / rule, or bad flag. - `live`: 0 done; 1 verify: a line differs; 2 a bad persona, state, event or flag. +- `check`: 0 every rule held and every strand replays; 1 a rule broken, an unresolved contradiction or a strand that fails + (1 wins); 2 a bad persona, rule file or flag; 3 the only problem is an unproven rule. - `monitor`: 0 every line replays; 1 a line differs; 2 a bad flag, a port that cannot be had, or a file that is not a strand. ## Errors @@ -85,6 +87,14 @@ One channel, the same for every command (0.8.2): | `monitor STRAND [--follow] [--once] [--plain] [--serve] [--open] [--port N]` | the strand replayed as live bars in the terminal, or the Activity Monitor page on 127.0.0.1 (channel table, event-history traces, inspector) | | `monitor --demo [week\|drives] [--open]` | synthetic data: the tutor's week, paced, in under a minute; `drives`: a short goal-signal scenario | +## `probbit check PERSONA` (0.8.3): one gate for CI +`probbit check PERSONA [--props FILE] [--strand FILE]... [--seeds 0-99] [--depth 8] [--hours 1,12,48] [--json]` runs `persona +lint`'s pipeline on the rules (prove, then fuzz the unknown ones) and `live verify` on every strand. One line per rule +(`held`, `held_by_construction`, `broken seed N`, `unproven`), one per strand (`ok N events` / `FAIL line K: ...`), then +`check: held H · broken B · unproven U · strands S/T`. `--json`: `{persona, props, seeds, depth, hours, rules, strands, +verdict}`, where `rules` is the `persona lint` document. In GitHub Actions: `uses: BitmapAsset/probbit@` (README, +"Check in CI"). + ## Testing a character - `lint`: contradicting habits; with `--props`, every rule proved or fuzzed; - `fuzz`: "none found" is evidence over the scripts searched, not a proof; diff --git a/docs/agents.md b/docs/agents.md index 1697064..c811599 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -191,6 +191,13 @@ const answer = call(2, JSON.stringify(request)); // flags go in a "flags" obje Agent runtimes with a decision-model slot (see the provider package). +## Check in CI + +`probbit check PERSONA --props rules.json [--strand FILE]...` vets a persona's rules (prove, then fuzz the unknown ones) and +replays its strands in one call: exit 0 all held and replayed, 1 a rule broken or a strand failing, 3 only unproven rules, +2 bad input. `--json` gives one document for a script to read. In GitHub Actions the repository is itself an action +(README, "Check in CI"). + ## For agents with a shell tool Coding agents with a shell tool call probbit the same way a script does: they run the command, read the JSON, and branch on diff --git a/docs/probbit.1 b/docs/probbit.1 index e351d2c..27a72b2 100644 --- a/docs/probbit.1 +++ b/docs/probbit.1 @@ -12,6 +12,8 @@ probbit \- a virtual probabilistic (p\-bit) processor: joint decisions under har \fBprobbit\fR live verify STRAND [\-\-from\-checkpoint] \fBprobbit\fR live control STRAND pause|resume|retire \-\-by WHO \-\-reason TEXT [\-\-at TIME] +\fBprobbit\fR check PERSONA [\-\-props FILE] [\-\-strand FILE]... [\-\-seeds LIST] + [\-\-depth N] [\-\-hours LIST] [\-\-json] \fBprobbit\fR monitor STRAND [\-\-follow] [\-\-once] [\-\-plain] [\-\-fps N] [\-\-serve] [\-\-open] [\-\-port N] \fBprobbit\fR monitor \-\-demo [week|drives] [\-\-once] [\-\-plain] [\-\-serve] @@ -333,6 +335,49 @@ probbit live examples/persona/tutor.yaml \-\-clock fixed \-\-strand pip.strand < probbit live verify pip.strand .fi .RE +.SS check +one CI gate: prove + fuzz a persona's rules, replay its strands (exit 0 / 1 / 3) +.PP +.nf +\fBprobbit\fR check PERSONA [\-\-props FILE] [\-\-strand FILE]... [\-\-seeds LIST] + [\-\-depth N] [\-\-hours LIST] [\-\-json] +.fi +.PP +The rule lock in one command, for CI: vet the rules a persona must keep before its first task and after every policy update, then replay the receipts. It runs `probbit persona lint`'s pipeline (contradicting habits; with \-\-props, prove each rule over the individuals, then fuzz the ones the bound leaves unknown, the rules' and habits' thresholds on the grid) and `probbit live verify` on every \-\-strand (each strand replays against the persona in its own header). +.PP +One line per rule (held = proved for every event sequence, held_by_construction = a habit implies it, broken seed N = the search found a counterexample, unproven = not proved and no counterexample found), one line per strand (ok N events, or FAIL and the first line that differs), then `check: held H \[u00B7] broken B \[u00B7] unproven U \[u00B7] strands S/T`. +.PP +Flags: +.RS 4 +.TP +\fB\-\-props\fR \fIFILE\fR +rules from a file ({"props": [...]}, each with an optional id; YAML subset, or .json) +.TP +\fB\-\-strand\fR \fIFILE\fR +a strand to replay (`probbit live \-\-strand` writes one); give it once per strand +.TP +\fB\-\-seeds\fR \fILIST\fR +the individuals: 0\-99 (default), 7, 1,4,9 or 0\-9,20\-29; seeds 0 to 2^53, at most 100000 +.TP +\fB\-\-depth\fR \fIN\fR +events per fuzzed script, 1 to 64 (default 8) +.TP +\fB\-\-hours\fR \fILIST\fR +comma\-separated idle hours the fuzzer tries between events (default 1,12,48) +.TP +\fB\-\-json\fR +one document: {persona, props, seeds, depth, hours, rules (the persona lint document), strands [{path, ok, events, final_state, error?}], verdict {held, broken, unproven, conflicts, strands_ok, strands_total, exit}} +.RE +.PP +\fIexit\fR: 0 every rule held and every strand replays; 1 a rule broken, an unresolved habit contradiction or a strand that fails (1 wins over 3); 2 a bad persona, rule file or flag (one {"error"} object on stdout, code "persona", when stdout is not a terminal or with \-\-json); 3 the only problem is an unproven rule. +.PP +Examples: +.RS 4 +.nf +probbit check examples/persona/dispatch\-policy.yaml \-\-props examples/persona/dispatch\-props.json +probbit check agent.yaml \-\-props rules.yaml \-\-strand day1.strand \-\-strand day2.strand \-\-json > check.json +.fi +.RE .SS monitor an individual's inner state, live: terminal bars or a page on 127.0.0.1 .PP diff --git a/examples/persona/dispatch-policy.yaml b/examples/persona/dispatch-policy.yaml new file mode 100644 index 0000000..8c4508f --- /dev/null +++ b/examples/persona/dispatch-policy.yaml @@ -0,0 +1,67 @@ +# A tool-dispatch policy for an agent that learns from feedback: act on a request, or ask a person first. +# The rules it must keep are in dispatch-props.json; `probbit check` vets them (README "Check in CI"). +probbit_persona: 1 +identity: + name: Dispatch + version: 1.0.0 + seed: 0 + summary: acts on routine work; asks a person before risky work or after repeated failures +traits: + - id: action + levels: [act, ask] + logw: [0.8, 0] + spread: 0.3 + say: [take the action, ask a person first] + - id: caution + levels: [measured, careful] + prior: [0.6, 0.4] + spread: 0.3 + say: ["", verify before acting] +couplings: + - {vars: [caution, action], align: 0.6} +inputs: + - id: blast + kind: number + say: blast radius of the request (0 = none, 1 = everything) + effects: {action: 1.2, caution: 1.0} + - id: error + kind: flag + say: the last action failed + effects: {action: 0.4, caution: 0.6} + - id: praise + kind: flag + say: the outcome was confirmed good + - id: criticism + kind: flag + say: the outcome was judged bad +history: + - id: error_streak + of: error + kind: streak + cap: 3 + effects: {action: 0.3} +habits: + # routine work: no blast radius, no failure, no failure streak. Without this habit, feedback can teach some + # individuals to ask about routine work too (see the README section "Check in CI"). + - id: routine_acts + when: {blast: {at_most: 0}, error: false, error_streak: {at_most: 0}} + then: {action: [act]} + say: routine work, just do it + priority: -1 # never applies together with the two below; if it ever did, it yields to them + - id: risky_asks + when: {blast: 0.7} + then: {action: [ask]} + say: high blast radius, ask first + - id: two_errors_ask + when: {error_streak: 2} + then: {action: [ask]} + say: two failures in a row, ask first +learning: + from: [praise, criticism] + traits: [action] + rate: 0.3 + step_cap: 0.05 + total_cap: 0.4 +reward_from: [human, env] +engine: + on_conflict: yield # lint pairs habits by what they require: these never meet, but a turn where they did would ask diff --git a/examples/persona/dispatch-props.json b/examples/persona/dispatch-props.json new file mode 100644 index 0000000..e976221 --- /dev/null +++ b/examples/persona/dispatch-props.json @@ -0,0 +1,7 @@ +{ + "props": [ + {"id": "risky_work_asks", "when": {"blast": 0.7}, "then": {"action": ["ask"]}}, + {"id": "repeated_failures_ask", "when": {"error_streak": 2}, "then": {"action": ["ask"]}}, + {"id": "routine_work_acts", "when": {"blast": {"at_most": 0}, "error": false, "error_streak": {"at_most": 0}}, "then": {"action": ["act"]}} + ] +} diff --git a/probbit-cli/src/main.rs b/probbit-cli/src/main.rs index 0a07669..ec01ffb 100644 --- a/probbit-cli/src/main.rs +++ b/probbit-cli/src/main.rs @@ -11,6 +11,7 @@ //! --progress [MS] (decide, run, evaluate): one JSONL telemetry line on stderr every MS ms (default 100) while the decision runs //! probbit stats [--sweeps N] # the processor's spec sheet: machine, build, effective controls + source, measured updates/s //! probbit persona init|turn|replay|explain|diff|lint|fuzz|prove|check|compile|describe PERSONA [flags] # the individuality layer (docs/persona.md) +//! probbit check PERSONA [--props FILE] [--strand FILE]... [--json] # one CI gate: lint + prove + fuzz the rules, replay the strands //! probbit mcp # a Model Context Protocol server on stdio (tools probbit_decide, probbit_run, probbit_stats, probbit_demo, probbit_evaluate, probbit_persona_init, probbit_persona_turn, probbit_persona_fuzz) //! probbit version //! probbit --help | -h # usage, every flag with its default, exit codes @@ -327,6 +328,23 @@ pub(crate) static COMMANDS: &[Command] = &[ "probbit live examples/persona/tutor.yaml --clock fixed --strand pip.strand < events.jsonl", "probbit live verify pip.strand"], run: live_cmd }, + Command { name: "check", aliases: &[], group: "persona", summary: "one CI gate: prove + fuzz a persona's rules, replay its strands (exit 0 / 1 / 3)", + usage: &["check PERSONA [--props FILE] [--strand FILE]... [--seeds LIST] [--depth N] [--hours LIST] [--json]"], + about: "The rule lock in one command, for CI: vet the rules a persona must keep before its first task and after every policy update, then replay the receipts. It runs `probbit persona lint`'s pipeline (contradicting habits; with --props, prove each rule over the individuals, then fuzz the ones the bound leaves unknown, the rules' and habits' thresholds on the grid) and `probbit live verify` on every --strand (each strand replays against the persona in its own header).\nOne line per rule (held = proved for every event sequence, held_by_construction = a habit implies it, broken seed N = the search found a counterexample, unproven = not proved and no counterexample found), one line per strand (ok N events, or FAIL and the first line that differs), then `check: held H · broken B · unproven U · strands S/T`.", + before: &[], + flags: &[&[ + F_P_PROPS, + Flag { name: "--strand", arg: "FILE", help: "a strand to replay (`probbit live --strand` writes one); give it once per strand" }, + F_P_SEEDS, + Flag { name: "--depth", arg: "N", help: "events per fuzzed script, 1 to 64 (default 8)" }, + Flag { name: "--hours", arg: "LIST", help: "comma-separated idle hours the fuzzer tries between events (default 1,12,48)" }, + Flag { name: "--json", arg: "", help: "one document: {persona, props, seeds, depth, hours, rules (the persona lint document), strands [{path, ok, events, final_state, error?}], verdict {held, broken, unproven, conflicts, strands_ok, strands_total, exit}}" }, + ]], + subs: &[], list_subs: false, shared: false, + after: &[("exit", "0 every rule held and every strand replays; 1 a rule broken, an unresolved habit contradiction or a strand that fails (1 wins over 3); 2 a bad persona, rule file or flag (one {\"error\"} object on stdout, code \"persona\", when stdout is not a terminal or with --json); 3 the only problem is an unproven rule.")], + examples: &["probbit check examples/persona/dispatch-policy.yaml --props examples/persona/dispatch-props.json", + "probbit check agent.yaml --props rules.yaml --strand day1.strand --strand day2.strand --json > check.json"], + run: check_cmd }, Command { name: "monitor", aliases: &[], group: "monitor", summary: "an individual's inner state, live: terminal bars or a page on 127.0.0.1", usage: &["monitor STRAND [--follow] [--once] [--plain] [--fps N] [--serve] [--open] [--port N]", "monitor --demo [week|drives] [--once] [--plain] [--serve] [--open] [--port N]"], @@ -839,6 +857,100 @@ fn live_cmd(args: &[String]) { if refused > 0 { exit(4) } if bad > 0 { exit(2) } } +/// What `persona lint` reports, and `probbit check` reuses: the lint document, its unresolved contradictions, the rules the search +/// broke and the unproven ones (the document's `unproven` entries) +struct Lint { doc: Json, unresolved: usize, broken: usize, unproven: Vec } +/// `persona lint`'s pipeline: the contradictions and plan warnings; with rules (--never / --props, §5.6) prove each one, then fuzz +/// the ones the bound leaves unknown (`depth` events per script, idle `hours`; the grid gets the rules' and habits' thresholds) +fn lint_report(args: &[String], path: &str, p: &persona::Persona, depth: usize, hours: Vec) -> Lint { + let engine = persona_engine(); let eng: persona::Engine = &engine; + let found = persona::lint(p, eng); let unresolved = found.iter().filter(|f| !f.get("resolution").and_then(Json::as_str).is_some_and(|r| r.starts_with("yield"))).count(); + let seeds = seeds_arg(args); let warnings = persona::plan_warnings(p, &seeds, eng); + let mut doc = vec![("persona", jstr(&p.name)), ("conflicts", Json::Arr(found)), ("unresolved", num(unresolved as f64)), ("warnings", Json::Arr(warnings))]; + let (mut broken, mut unproven) = (0, vec![]); + // drives (§2.9): habits that outrank a goal floor, floors whose lift can reach 50 + if let Some(mut n) = persona::floor_notes(p) { n.extend(persona::floor_reach(p, &seeds)); doc.push(("floors", Json::Arr(n))); } + // with rules (§5.6): prove each one, then fuzz the ones the bound leaves unknown + if args.iter().any(|a| a == "--never" || a == "--props") { + let props = rules_arg(args, p); let threads = arg(args, "--threads", std::thread::available_parallelism().map_or(1, |n| n.get())).clamp(1, 1024); + let verdicts = fuzz::prove(p, &props, &seeds, threads, &engine); + let unknown: Vec = props.iter().zip(&verdicts).filter(|(_, v)| matches!(v, fuzz::Verdict::Unknown { .. })).map(|(q, _)| q.clone()).collect(); + let s = fuzz::Search { seeds: seeds.clone(), fuzz_seed: 0, scripts: 60, depth, beam: 4, grid: fuzz::grid(p, &unknown, &[0.0, 0.5, 1.0]), hours, threads }; + let (res, turns) = if unknown.is_empty() { (vec![], 0) } else { fuzz::fuzz(p, &unknown, &s, &engine) }; + let fd = fuzz::doc(p, path, &unknown, &s, &res, turns); let fp = fd.get("properties").and_then(Json::as_arr).unwrap_or(&[]).to_vec(); + broken = fp.iter().filter(|x| x.get("verdict").and_then(Json::as_str) == Some("counterexample")).count(); + let mut entries = fuzz::prove_doc(p, &props, &seeds, &verdicts).get("properties").and_then(Json::as_arr).unwrap_or(&[]).to_vec(); + for e in entries.iter_mut() { let id = e.get("id").cloned(); + if let (Json::Obj(v), Some(f)) = (e, fp.iter().find(|x| x.get("id").cloned() == id)) { v.push(("fuzz".into(), f.clone())); } } + // unproven: the bound left it unknown and the search found no counterexample. Not broken, and not proved either. + for (q, v) in props.iter().zip(&verdicts) { + let fuzz::Verdict::Unknown { failed, .. } = v else { continue }; + if fp.iter().any(|x| x.get("id").and_then(Json::as_str) == Some(q.id.as_str()) && x.get("verdict").and_then(Json::as_str) == Some("counterexample")) { continue; } + let searched = obj(vec![("individuals", num(s.seeds.len() as f64)), ("scripts", num(s.scripts as f64)), ("depth", num(s.depth as f64)), ("beam", num(s.beam as f64)), + ("grid", Json::Arr(s.grid.iter().map(|x| num(*x)).collect())), ("hours", Json::Arr(s.hours.iter().map(|x| num(*x)).collect()))]); + unproven.push(obj(vec![("id", jstr(&q.id)), ("rule", q.rule.clone()), ("seeds_unknown", Json::Arr(failed.iter().map(|f| num(f.0 as f64)).collect())), ("searched", searched)])); } + doc.push(("props", Json::Arr(entries))); doc.push(("broken", num(broken as f64))); doc.push(("unproven", Json::Arr(unproven.clone()))); } + doc.push(("ok", Json::Bool(unresolved == 0 && broken == 0 && unproven.is_empty()))); + Lint { doc: obj(doc), unresolved, broken, unproven } +} +/// `probbit check PERSONA [--props FILE] [--strand FILE]... [--seeds LIST] [--depth N] [--hours LIST] [--json]`: one gate for CI. +/// `persona lint`'s pipeline on the rules (`lint_report`), then `live verify` on every strand; exit 1 when a rule is broken, a +/// contradiction is unresolved or a strand fails, else 3 when a rule is unproven, else 0 +fn check_cmd(args: &[String]) { + // --strand may be given more than once: take its values out before the flags are checked + let (mut rest, mut strands) = (vec![], vec![]); let mut i = 0; + while i < args.len() { + if args[i] == "--strand" { match args.get(i + 1).filter(|v| !v.starts_with("--")) { Some(v) => strands.push(v.clone()), None => fail("--strand needs a value") } i += 2; } + else { rest.push(args[i].clone()); i += 1; } } + let Some(path) = rest.get(1).filter(|a| !a.starts_with("--")).cloned() else { fail("check: the persona file comes first: probbit check PERSONA [--props FILE] [--strand FILE]...") }; + let mut a = vec!["check".to_string()]; a.extend(rest[2..].iter().cloned()); let (v, w) = flags_of("check"); check_flags(&a, &v, &w); + let depth: usize = arg(&a, "--depth", 8); if !(1..=64).contains(&depth) { fail("--depth must be 1 to 64") } + let hours = list_arg(&a, "--hours", &[1.0, 12.0, 48.0], 0.0, 1e6); + let (p, _) = persona::load(&path).unwrap_or_else(|e| bad_input("check: ", e)); + let seeds = seeds_arg(&a); + let Lint { doc: lint, unresolved, broken, unproven } = lint_report(&a, &path, &p, depth, hours.clone()); + let engine = persona_engine(); let eng: persona::Engine = &engine; + let strand_docs: Vec = strands.iter().map(|f| { + let res = std::fs::read_to_string(f).map_err(|e| format!("cannot read: {e}")).and_then(|t| live::verify(&t, eng).map_err(|(line, why)| format!("line {line}: {why}"))); + let mut e = vec![("path", jstr(f))]; + match res { + Ok(d) => { e.push(("ok", Json::Bool(true))); e.push(("events", d.get("events").cloned().unwrap_or(Json::Null))); e.push(("final_state", d.get("final_state").cloned().unwrap_or(Json::Null))); } + Err(why) => { e.push(("ok", Json::Bool(false))); e.push(("events", Json::Null)); e.push(("final_state", Json::Null)); e.push(("error", jstr(&why))); } } + obj(e) }).collect(); + let strands_ok = strand_docs.iter().filter(|s| s.get("ok") == Some(&Json::Bool(true))).count(); + let rules: &[Json] = lint.get("props").and_then(Json::as_arr).unwrap_or(&[]); + let held = rules.iter().filter(|r| matches!(r.get("verdict").and_then(Json::as_str), Some("held_by_construction") | Some("proved"))).count(); + let code = if broken > 0 || unresolved > 0 || strands_ok < strand_docs.len() { 1 } else if !unproven.is_empty() { 3 } else { 0 }; + if a.iter().any(|x| x == "--json") { + let verdict = obj(vec![("held", num(held as f64)), ("broken", num(broken as f64)), ("unproven", num(unproven.len() as f64)), ("conflicts", num(unresolved as f64)), + ("strands_ok", num(strands_ok as f64)), ("strands_total", num(strand_docs.len() as f64)), ("exit", num(code as f64))]); + let props = a.iter().position(|x| x == "--props").and_then(|k| a.get(k + 1)).map_or(Json::Null, |f| jstr(f)); + emit(&persona::canon(&obj(vec![("persona", jstr(&path)), ("props", props), ("seeds", jstr(&fuzz::ranges(&seeds))), ("depth", num(depth as f64)), + ("hours", Json::Arr(hours.iter().map(|x| num(*x)).collect())), ("rules", lint), ("strands", Json::Arr(strand_docs)), ("verdict", verdict)]))); + exit(code) + } + let seed_list = |j: Option<&Json>| fuzz::ranges(&j.and_then(Json::as_arr).unwrap_or(&[]).iter().filter_map(|x| x.as_f64().map(|x| x as u64)).collect::>()); + let width = rules.iter().filter_map(|r| r.get("id").and_then(Json::as_str)).map(|s| s.chars().count()).chain(strands.iter().map(|s| s.chars().count())).max().unwrap_or(0); + let mut o = vec![format!("probbit check {path} ({} {}, seeds {}, depth {depth})", p.name, p.version, fuzz::ranges(&seeds))]; + if unresolved > 0 { o.push(format!("conflicts {unresolved} unresolved habit contradiction{} (probbit persona lint {path})", if unresolved == 1 { "" } else { "s" })); } + if rules.is_empty() { o.push("rules none given (--props FILE)".into()); } + for r in rules { + let id = r.get("id").and_then(Json::as_str).unwrap_or(""); + let what = match (r.get("verdict").and_then(Json::as_str), r.get("fuzz").and_then(|f| f.get("verdict")).and_then(Json::as_str)) { + (Some("held_by_construction"), _) => "held_by_construction".to_string(), + (Some("proved"), _) => "held".to_string(), + (_, Some("counterexample")) => { let s = r.get("fuzz").and_then(|f| f.get("failing_seeds")).and_then(Json::as_arr).map_or(0, |a| a.len()); + format!("broken seed{} {}", if s == 1 { "" } else { "s" }, seed_list(r.get("fuzz").and_then(|f| f.get("failing_seeds")))) } + _ => format!("unproven seeds {} not proved, no counterexample found", seed_list(unproven.iter().find(|u| u.get("id").and_then(Json::as_str) == Some(id)).and_then(|u| u.get("seeds_unknown")))) }; + o.push(format!("rule {id: PERSONA [flags]` (docs/persona.md) fn persona_cmd(args: &[String]) { let sub = args.get(1).map_or("", String::as_str); @@ -883,34 +995,8 @@ fn persona_cmd(args: &[String]) { "check" => emit(&persona_text(&persona::check(&p, eng).unwrap_or_else(|e| bad_input("persona: ", e)), pretty)), "describe" => emit(&persona_text(&persona::describe(&p), pretty)), "fuzz" | "prove" => fuzz_cmd(args, path, &p, &engine), - _ => { let found = persona::lint(&p, eng); let unresolved = found.iter().filter(|f| !f.get("resolution").and_then(Json::as_str).is_some_and(|r| r.starts_with("yield"))).count(); - let seeds = seeds_arg(args); let warnings = persona::plan_warnings(&p, &seeds, eng); - let mut doc = vec![("persona", jstr(&p.name)), ("conflicts", Json::Arr(found)), ("unresolved", num(unresolved as f64)), ("warnings", Json::Arr(warnings))]; - let (mut broken, mut unproven) = (0, vec![]); - // drives (§2.9): habits that outrank a goal floor, floors whose lift can reach 50 - if let Some(mut n) = persona::floor_notes(&p) { n.extend(persona::floor_reach(&p, &seeds)); doc.push(("floors", Json::Arr(n))); } - // with rules (§5.6): prove each one, then fuzz the ones the bound leaves unknown - if args.iter().any(|a| a == "--never" || a == "--props") { - let props = rules_arg(args, &p); let threads = arg(args, "--threads", std::thread::available_parallelism().map_or(1, |n| n.get())).clamp(1, 1024); - let verdicts = fuzz::prove(&p, &props, &seeds, threads, &engine); - let unknown: Vec = props.iter().zip(&verdicts).filter(|(_, v)| matches!(v, fuzz::Verdict::Unknown { .. })).map(|(q, _)| q.clone()).collect(); - let s = fuzz::Search { seeds: seeds.clone(), fuzz_seed: 0, scripts: 60, depth: 8, beam: 4, grid: fuzz::grid(&p, &unknown, &[0.0, 0.5, 1.0]), hours: vec![1.0, 12.0, 48.0], threads }; - let (res, turns) = if unknown.is_empty() { (vec![], 0) } else { fuzz::fuzz(&p, &unknown, &s, &engine) }; - let fd = fuzz::doc(&p, path, &unknown, &s, &res, turns); let fp = fd.get("properties").and_then(Json::as_arr).unwrap_or(&[]).to_vec(); - broken = fp.iter().filter(|x| x.get("verdict").and_then(Json::as_str) == Some("counterexample")).count(); - let mut entries = fuzz::prove_doc(&p, &props, &seeds, &verdicts).get("properties").and_then(Json::as_arr).unwrap_or(&[]).to_vec(); - for e in entries.iter_mut() { let id = e.get("id").cloned(); - if let (Json::Obj(v), Some(f)) = (e, fp.iter().find(|x| x.get("id").cloned() == id)) { v.push(("fuzz".into(), f.clone())); } } - // unproven: the bound left it unknown and the search found no counterexample. Not broken, and not proved either. - for (q, v) in props.iter().zip(&verdicts) { - let fuzz::Verdict::Unknown { failed, .. } = v else { continue }; - if fp.iter().any(|x| x.get("id").and_then(Json::as_str) == Some(q.id.as_str()) && x.get("verdict").and_then(Json::as_str) == Some("counterexample")) { continue; } - let searched = obj(vec![("individuals", num(s.seeds.len() as f64)), ("scripts", num(s.scripts as f64)), ("depth", num(s.depth as f64)), ("beam", num(s.beam as f64)), - ("grid", Json::Arr(s.grid.iter().map(|x| num(*x)).collect())), ("hours", Json::Arr(s.hours.iter().map(|x| num(*x)).collect()))]); - unproven.push(obj(vec![("id", jstr(&q.id)), ("rule", q.rule.clone()), ("seeds_unknown", Json::Arr(failed.iter().map(|f| num(f.0 as f64)).collect())), ("searched", searched)])); } - doc.push(("props", Json::Arr(entries))); doc.push(("broken", num(broken as f64))); doc.push(("unproven", Json::Arr(unproven.clone()))); } - doc.push(("ok", Json::Bool(unresolved == 0 && broken == 0 && unproven.is_empty()))); - emit(&persona_text(&obj(doc), pretty)); + _ => { let Lint { doc, unresolved, broken, unproven } = lint_report(args, path, &p, 8, vec![1.0, 12.0, 48.0]); + emit(&persona_text(&doc, pretty)); if !unproven.is_empty() { let ids: Vec<&str> = unproven.iter().filter_map(|u| u.get("id").and_then(Json::as_str)).collect(); err_line(&format!("lint: {} rule{} unproven, not broken: {} (the bound left {} unknown and the search found no counterexample; `probbit persona prove` names the cell)", @@ -1044,7 +1130,7 @@ mod table_tests { let body = &src[src.find("\nfn main() {").unwrap()..]; let body = &body[..body.find("\n}\n").unwrap()]; let lits: Vec<&str> = body.match_indices("Some(\"").map(|(i, _)| { let r = &body[i + 6..]; &r[..r.find('"').unwrap()] }).collect(); assert!(lits.iter().all(|l| ["--help", "-h"].contains(l)), "main dispatches {lits:?} outside the table"); - for (old, n) in [("decide", 0), ("demo", 0), ("ir", 0), ("run", 0), ("evaluate", 0), ("stats", 0), ("persona", 0), ("live", 0), ("monitor", 0), ("mcp", 0), ("version", 0), ("--version", 0), ("-V", 0)] { + for (old, n) in [("decide", 0), ("demo", 0), ("ir", 0), ("run", 0), ("evaluate", 0), ("stats", 0), ("persona", 0), ("live", 0), ("check", 0), ("monitor", 0), ("mcp", 0), ("version", 0), ("--version", 0), ("-V", 0)] { assert!(command(old).is_some(), "{old} (accepted before the table) is not in it {n}"); } } diff --git a/probbit-cli/tests/cli.rs b/probbit-cli/tests/cli.rs index dc8c854..42a66ed 100644 --- a/probbit-cli/tests/cli.rs +++ b/probbit-cli/tests/cli.rs @@ -1195,8 +1195,12 @@ fn chains_and_threads_have_upper_bounds() { /// flag lost its exit code 2, bad input its error object, and `--progress` into `head -c 1` the whole decision. Now ignored. #[test] fn closed_stderr_pipe_keeps_stdout_and_exit_code() { - let run = |args: &[&str], input: &str| { let (r, w) = std::io::pipe().unwrap(); drop(r); // no reader: every stderr write fails - let mut c = Command::new(env!("CARGO_BIN_EXE_probbit")).args(args).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(w).spawn().unwrap(); + let run = |args: &[&str], input: &str| { + // a pipe with no reader, so every stderr write fails: the write end of a process's stdin once that process has exited + // (`std::io::pipe` would do it directly, but is stable only since Rust 1.87; the MSRV is 1.78) + let mut gone = Command::new(env!("CARGO_BIN_EXE_probbit")).arg("version").stdin(Stdio::piped()).stdout(Stdio::null()).stderr(Stdio::null()).spawn().unwrap(); + let w = gone.stdin.take().unwrap(); gone.wait().unwrap(); + let mut c = Command::new(env!("CARGO_BIN_EXE_probbit")).args(args).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::from(w)).spawn().unwrap(); let _ = c.stdin.take().unwrap().write_all(input.as_bytes()); let o = c.wait_with_output().unwrap(); (o.status.code(), String::from_utf8_lossy(&o.stdout).into_owned()) }; assert_eq!(run(&["decide", "--bogus"], ""), (Some(2), String::new())); let (c, out) = run(&["decide"], "not json"); assert_eq!(c, Some(2)); one_error(&out, "schema", ""); diff --git a/probbit-cli/tests/cli_check.rs b/probbit-cli/tests/cli_check.rs new file mode 100644 index 0000000..22fc683 --- /dev/null +++ b/probbit-cli/tests/cli_check.rs @@ -0,0 +1,131 @@ +//! `probbit check` (0.8.3): `persona lint`'s pipeline on a persona's rules plus `live verify` on its strands, one verdict line and +//! exit 0 (every rule held, every strand replays), 1 (a rule broken or a strand failing; 1 wins), 3 (only unproven rules). +use std::io::Write; +use std::process::{Command, Stdio}; + +#[path = "../src/json.rs"] +#[allow(dead_code)] +mod json; +use json::Json; + +/// `probbit` run from the repository root (the shipped examples are addressed from there): (exit code, stdout, stderr) +fn probbit(args: &[&str], stdin: &str) -> (i32, String, String) { + let mut c = Command::new(env!("CARGO_BIN_EXE_probbit")).args(args).current_dir(format!("{}/..", env!("CARGO_MANIFEST_DIR"))) + .env_remove("PROBBIT_CONFIG").stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap(); + let _ = c.stdin.take().unwrap().write_all(stdin.as_bytes()); let o = c.wait_with_output().unwrap(); + (o.status.code().unwrap_or(-1), String::from_utf8_lossy(&o.stdout).into_owned(), String::from_utf8_lossy(&o.stderr).into_owned()) +} +fn tmp(name: &str) -> String { format!("{}/check-{}-{name}", env!("CARGO_TARGET_TMPDIR"), std::process::id()) } +fn parse(s: &str) -> Json { json::parse(s).unwrap_or_else(|e| panic!("not JSON ({}): {s:.300}", e.msg)) } +fn jw(j: &Json) -> String { json::write(j, false) } +fn s<'a>(j: &'a Json, path: &[&str]) -> &'a Json { path.iter().fold(j, |x, k| x.get(k).unwrap_or_else(|| panic!("no {k} in {:.200}", jw(x)))) } +const DISPATCH: &str = "examples/persona/dispatch-policy.yaml"; +const DISPATCH_PROPS: &str = "examples/persona/dispatch-props.json"; +const TUTOR: &str = "examples/persona/tutor.yaml"; + +/// The shipped rule-lock example: every rule held by construction on seeds 0-99, exit 0 +#[test] +fn dispatch_example_holds_every_rule() { + let (c, out, e) = probbit(&["check", DISPATCH, "--props", DISPATCH_PROPS], ""); assert_eq!(c, 0, "{out}\n{e}"); + assert!(out.starts_with(&format!("probbit check {DISPATCH} (Dispatch 1.0.0, seeds 0-99, depth 8)\n")), "{out}"); + for id in ["risky_work_asks", "repeated_failures_ask", "routine_work_acts"] { + assert!(out.lines().any(|l| l.starts_with("rule ") && l.contains(id) && l.ends_with(" held_by_construction")), "{id}: {out}"); } + assert!(out.ends_with("check: held 3 · broken 0 · unproven 0 · strands 0/0\n"), "{out}"); +} + +/// Without the habit that states it, `routine_work_acts` is broken on some seeds: exit 1, the seeds named; the habits that stay +/// keep their rules held +#[test] +fn dispatch_without_its_routine_habit_is_broken() { + let src = std::fs::read_to_string(format!("{}/../{DISPATCH}", env!("CARGO_MANIFEST_DIR"))).unwrap().replace("\r\n", "\n"); + let (a, b) = (src.find(" - id: routine_acts\n").unwrap(), src.find(" - id: risky_asks\n").unwrap()); + let f = tmp("dispatch-no-habit.yaml"); std::fs::write(&f, format!("{}{}", &src[..a], &src[b..])).unwrap(); + let props = format!("{}/../{DISPATCH_PROPS}", env!("CARGO_MANIFEST_DIR")); + let (c, out, e) = probbit(&["check", &f, "--props", &props, "--seeds", "0-19"], ""); assert_eq!(c, 1, "{out}\n{e}"); + let line = out.lines().find(|l| l.contains("routine_work_acts")).unwrap(); assert!(line.contains(" broken seed"), "{out}"); + assert!(out.lines().any(|l| l.contains("risky_work_asks") && l.ends_with("held_by_construction")), "{out}"); + assert!(out.ends_with("check: held 2 · broken 1 · unproven 0 · strands 0/0\n"), "{out}"); + let _ = std::fs::remove_file(&f); +} + +/// The shipped tutor: a proved rule exits 0 ("held"), a broken one 1, an unproven one 3, broken next to unproven 1 +#[test] +fn exit_codes_follow_the_rules() { + let props = |name: &str, body: &str| { let f = tmp(name); std::fs::write(&f, body).unwrap(); f }; + let proved = props("proved.json", r#"{"props": [{"id": "error_care", "when": {"error": true}, "then": {"caution": {"at_least": "measured"}}}]}"#); + let broken = props("broken.json", r#"{"props": [{"id": "praise_lifts", "when": {"praise": true}, "then": {"valence": {"at_least": "even"}}}]}"#); + let unproven = props("unproven.json", r#"{"props": [{"id": "loss_warm", "when": {"loss": true}, "then": {"warmth": ["warm"]}}]}"#); + let both = props("both.json", r#"{"props": [{"id": "praise_lifts", "when": {"praise": true}, "then": {"valence": {"at_least": "even"}}}, + {"id": "loss_warm", "when": {"loss": true}, "then": {"warmth": ["warm"]}}]}"#); + let (c, out, e) = probbit(&["check", TUTOR, "--props", &proved, "--seeds", "0-2"], ""); assert_eq!(c, 0, "{out}\n{e}"); + assert!(out.lines().any(|l| l.contains("error_care") && l.ends_with(" held")), "{out}"); + let (c, out, e) = probbit(&["check", TUTOR, "--props", &broken, "--seeds", "0-2"], ""); assert_eq!(c, 1, "{out}\n{e}"); + assert!(out.lines().any(|l| l.contains("praise_lifts broken seed")), "{out}"); assert!(out.contains("check: held 0 · broken 1 · unproven 0"), "{out}"); + let (c, out, e) = probbit(&["check", TUTOR, "--props", &unproven, "--seeds", "0-2"], ""); assert_eq!(c, 3, "{out}\n{e}"); + assert!(out.lines().any(|l| l.contains("loss_warm unproven seeds 0-2 not proved")), "{out}"); assert!(out.contains("check: held 0 · broken 0 · unproven 1"), "{out}"); + let (c, out, e) = probbit(&["check", TUTOR, "--props", &both, "--seeds", "0-2"], ""); assert_eq!(c, 1, "1 wins over 3: {out}\n{e}"); + assert!(out.contains("check: held 0 · broken 1 · unproven 1"), "{out}"); + // the --json verdict says the same, and its rules are the persona lint document + let (c, j, e) = probbit(&["check", TUTOR, "--props", &both, "--seeds", "0-2", "--json"], ""); assert_eq!(c, 1, "{e}"); + let j = parse(&j); assert_eq!(jw(s(&j, &["verdict"])), r#"{"broken":1,"conflicts":0,"exit":1,"held":0,"strands_ok":0,"strands_total":0,"unproven":1}"#); + let (_, lint, _) = probbit(&["persona", "lint", TUTOR, "--props", &both, "--seeds", "0-2"], ""); assert_eq!(jw(s(&j, &["rules"])), jw(&parse(&lint))); + for f in [proved, broken, unproven, both] { let _ = std::fs::remove_file(f); } +} + +/// Strands: one that replays (ok N events) and a tampered copy (FAIL with its line): exit 1 although every rule holds; the --json +/// shape, every field named +#[test] +fn strands_replay_or_fail() { + let good = tmp("good.strand"); let _ = std::fs::remove_file(&good); + let events = "{\"blast\":0.2}\n{\"error\":true}\n{\"error\":true,\"criticism\":true,\"src\":\"human\"}\n{\"praise\":true,\"src\":\"env\"}\n{\"blast\":0.9}\n"; + let (c, _, e) = probbit(&["live", DISPATCH, "--seed", "2", "--clock", "fixed", "--strand", &good], events); assert_eq!(c, 0, "{e}"); + let (c, out, e) = probbit(&["check", DISPATCH, "--props", DISPATCH_PROPS, "--seeds", "0-9", "--strand", &good], ""); assert_eq!(c, 0, "{out}\n{e}"); + assert!(out.lines().any(|l| l.starts_with("strand ") && l.contains(&good) && l.ends_with(" ok 5 events")), "{out}"); + assert!(out.ends_with("strands 1/1\n"), "{out}"); + let bad = tmp("tampered.strand"); + let text = std::fs::read_to_string(&good).unwrap(); assert!(text.contains("\"blast\":0.9"), "{text:.300}"); + std::fs::write(&bad, text.replace("\"blast\":0.9", "\"blast\":0.1")).unwrap(); + let (c, out, e) = probbit(&["check", DISPATCH, "--props", DISPATCH_PROPS, "--seeds", "0-9", "--strand", &good, "--strand", &bad], ""); assert_eq!(c, 1, "{out}\n{e}"); + assert!(out.lines().any(|l| l.contains(&bad) && l.contains(" FAIL line ")), "{out}"); + assert!(out.ends_with("check: held 3 · broken 0 · unproven 0 · strands 1/2\n"), "{out}"); + let (c, j, e) = probbit(&["check", DISPATCH, "--props", DISPATCH_PROPS, "--seeds", "0-9", "--depth", "4", "--hours", "1,24", "--strand", &good, "--strand", &bad, "--json"], ""); assert_eq!(c, 1, "{e}"); + let j = parse(&j); + let keys: Vec<&str> = j.as_obj().unwrap().iter().map(|(k, _)| k.as_str()).collect(); + assert_eq!(keys, ["depth", "hours", "persona", "props", "rules", "seeds", "strands", "verdict"]); + assert_eq!((s(&j, &["persona"]).as_str(), s(&j, &["props"]).as_str(), s(&j, &["seeds"]).as_str(), s(&j, &["depth"]).as_f64(), jw(s(&j, &["hours"]))), + (Some(DISPATCH), Some(DISPATCH_PROPS), Some("0-9"), Some(4.0), "[1,24]".to_string())); + assert_eq!(s(&j, &["rules", "ok"]), &Json::Bool(true)); + let st = s(&j, &["strands"]).as_arr().unwrap(); assert_eq!(st.len(), 2); + assert_eq!((s(&st[0], &["ok"]), s(&st[0], &["events"]).as_f64(), st[0].get("error").is_none()), (&Json::Bool(true), Some(5.0), true)); + assert!(s(&st[0], &["final_state"]).as_str().is_some_and(|d| d.starts_with("sha256:"))); + assert_eq!((s(&st[1], &["path"]).as_str(), s(&st[1], &["ok"]), s(&st[1], &["events"]), s(&st[1], &["final_state"])), (Some(bad.as_str()), &Json::Bool(false), &Json::Null, &Json::Null)); + assert!(s(&st[1], &["error"]).as_str().is_some_and(|e| e.starts_with("line ")), "{}", jw(&st[1])); + assert_eq!(jw(s(&j, &["verdict"])), r#"{"broken":0,"conflicts":0,"exit":1,"held":3,"strands_ok":1,"strands_total":2,"unproven":0}"#); + // a strand that cannot be read fails too + let (c, out, _) = probbit(&["check", DISPATCH, "--seeds", "0", "--strand", "no-such.strand"], ""); assert_eq!(c, 1, "{out}"); + assert!(out.contains("no-such.strand FAIL cannot read") && out.contains("rules none given"), "{out}"); + for f in [good, bad] { let _ = std::fs::remove_file(f); } +} + +/// Bad usage exits 2: no persona, a missing --strand value, a flag check does not take, a bad depth, a missing persona file +#[test] +fn bad_usage_exits_2() { + for a in [vec!["check"], vec!["check", DISPATCH, "--strand"], vec!["check", DISPATCH, "--never", "{when: {blast: 0.7}, then: {action: [ask]}}"], + vec!["check", DISPATCH, "--depth", "0"], vec!["check", DISPATCH, "--seeds", "9-1"]] { + let (c, out, e) = probbit(&a, ""); assert_eq!(c, 2, "{a:?}: {e}"); assert!(out.is_empty(), "{a:?}: {out}"); } + let (c, out, _) = probbit(&["check", "no-such-persona.yaml"], ""); assert_eq!(c, 2); assert_eq!(s(&parse(&out), &["error", "code"]).as_str(), Some("persona")); +} + +/// The command table picks it up: the top-level help lists it, `check --help` and `probbit help check` print its usage and exit +/// codes, the completions offer it with its flags, the man page has it +#[test] +fn the_table_lists_check() { + let (c, top, _) = probbit(&["--help"], ""); assert_eq!(c, 0); assert!(top.contains("\n check PERSONA "), "{top}"); + let (c, h, _) = probbit(&["check", "--help"], ""); assert_eq!(c, 0); + assert!(h.starts_with("usage: probbit check PERSONA [--props FILE] [--strand FILE]... [--seeds LIST] [--depth N]\n [--hours LIST] [--json]\n"), "{h}"); + for f in ["--props FILE", "--strand FILE", "--seeds LIST", "--depth N", "--hours LIST", "--json"] { assert!(h.contains(&format!("\n {f} ")), "{f}: {h}"); } + assert!(h.contains("exit: 0 every rule held"), "{h}"); + let (_, h2, _) = probbit(&["help", "check"], ""); assert_eq!(h, h2); + let (_, bash, _) = probbit(&["completions", "bash"], ""); assert!(bash.contains(" check) if [ \"$COMP_CWORD\" -eq 2 ]; then words='--props --strand --seeds --depth --hours --json --help';"), "{bash}"); + let (_, man, _) = probbit(&["man"], ""); assert!(man.contains(".SS check\n"), "man page"); +} diff --git a/probbit-cli/tests/cli_table.rs b/probbit-cli/tests/cli_table.rs index 4ba2828..e5e7454 100644 --- a/probbit-cli/tests/cli_table.rs +++ b/probbit-cli/tests/cli_table.rs @@ -23,7 +23,7 @@ fn listed() -> Vec { #[test] fn every_listed_command_runs_and_every_old_command_is_listed() { let cmds = listed(); - for old in ["persona", "live", "monitor", "decide", "run", "evaluate", "demo", "ir", "mcp", "stats", "version"] { assert!(cmds.iter().any(|c| c == old), "{old} missing from {cmds:?}"); } + for old in ["persona", "live", "check", "monitor", "decide", "run", "evaluate", "demo", "ir", "mcp", "stats", "version"] { assert!(cmds.iter().any(|c| c == old), "{old} missing from {cmds:?}"); } for c in &cmds { let (code, out, err) = probbit(&[c, "--help"]); assert_eq!(code, 0, "{c}: {err}"); assert!(out.starts_with(&format!("usage: probbit {c}")), "{c}: {out}"); assert!(err.is_empty(), "{c}: {err}"); @@ -47,7 +47,7 @@ fn every_listed_subcommand_has_help_and_every_listed_flag_parses() { } } } - for cmd in ["decide", "run", "evaluate", "demo", "ir", "stats", "monitor"] { + for cmd in ["decide", "run", "evaluate", "demo", "ir", "stats", "monitor", "check"] { let (_, h, _) = probbit(&[cmd, "--help"]); let fl = help_flags(&h); assert!(!fl.is_empty(), "{cmd}: no flags listed"); for f in fl {