From a2fc853abb53ae0a80d8967e01f9166250cc3b3b Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 17:28:05 +0000 Subject: [PATCH 01/12] Read every lane's worktree inventory as one row per tree lanes-edit.sh gains `worktrees [ | --all] [--index] [--fetch]`, the #97 inventory of every lane on this workstation as US-separated rows: the lanes are the listing's own (`lanes_rows --all --closed`, so a closed lane's leftovers are not hidden), the trees are `lane_trees_list`'s sidecars, the owner is `lane_is_managed_owned`'s answer, the lifecycle is the snapshot's word for a legacy lane only, and writer liveness is the listing's LIVE. Nothing is recomputed and no directory is walked for a tree no sidecar names; the only git reads are of the recorded head (its subject and its merge-base with main). This is the one implementation that both `lanes --worktrees` and the indexer's new inventory source read, so the listing and the derived index's `worktrees` table cannot disagree. With `--index` the read goes through `index_open worktrees`, which asks `lanes-index export --worktrees` and sets its own export directory, never `LANES_IDX_DIR`, so no published register read is answered out of the index by it. The listing's directory column is handed to `lane_control_root` as the payload it takes, so sixty lanes' logs are not read a second time: the whole read went from 7.9 s to 2.7 s on Eagle's 61 lanes. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lanes-edit.sh | 248 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 242 insertions(+), 6 deletions(-) diff --git a/lanes-edit.sh b/lanes-edit.sh index 8eb08777..d5ea70bf 100755 --- a/lanes-edit.sh +++ b/lanes-edit.sh @@ -119,8 +119,9 @@ # no-op that says so. # # AMENDMENT 14 — THE DERIVED INDEX (`lanes-index`), WHICH NOTHING HERE ACTS ON -# lanes-edit.sh lanes … --index # the listing, out of the index -# lanes-edit.sh history … --index # the diary, out of the index +# lanes-edit.sh lanes … --index # the listing, out of the index +# lanes-edit.sh history … --index # the diary, out of the index +# lanes-edit.sh worktrees … --index # the inventory, out of its table (#161) # # The register's truth stays where it is. `lanes-index` (placed by # `openRepoTools --install`) keeps a DERIVED copy — SQLite, or QA Postgres @@ -133,6 +134,13 @@ # which it read: `read: index () at register@; commits # behind`, or `read: sources (index )` and the sources answer. # +# THE INVENTORY HAS A TABLE TOO (opensoft/openRepoTools#161): `worktrees`, +# under the source `inventory:`, copied from `worktrees --all` +# by `lanes-index sync --source inventory`. +# No act reads it: `lane-reconcile`, `lane-start` and `lane-end` read the +# sidecars and the disk. With `--index` the stderr line is `read: index +# () at inventory: synced `. +# # AMENDMENT 7 — the per-lane object log (`lanes/log/.md`) # lanes-edit.sh log [→|← ] [""] [--text ""] # lanes-edit.sh claim [--force] [--no-github] [--home owner/repo] @@ -351,6 +359,7 @@ # [--generation ] [--operation ] # lanes-edit.sh lane-tree-now # lanes-edit.sh lane-reconcile +# lanes-edit.sh worktrees [ | --all] [--index] [--fetch] # # A lane is RUNNING, SWAPPING, SWAPPED or CLOSED, and WHICH OF THOSE IT IS # WITH NO LIVE HOLDER is what says where its session stopped: `RUNNING` with @@ -1068,6 +1077,9 @@ LOCK="$LANES_DIR/.lanes-edit.lock" # so an exported variable of either name changes nothing. LANES_IDX_DIR="" LANES_INDEX_NUDGE=0 +# AND THE INVENTORY'S (opensoft/openRepoTools#161), for the same reason: the +# `worktrees` table's export directory, set only by `index_open worktrees`. +LANES_WT_IDX_DIR="" LOCK_HELD=0 LANES_PATH="${LANES_PATH:-$(git -C "${LANES_DIR:-.}" rev-parse --show-prefix 2>/dev/null || :)${LANES_FILE##*/}}" @@ -1188,6 +1200,7 @@ cleanup() { [ -n "${TMPD:-}" ] && [ -d "${TMPD:-}" ] && rm -rf -- "$TMPD" [ -n "${SE_CACHE_FILE:-}" ] && rm -f -- "$SE_CACHE_FILE" "$SE_CACHE_FILE".* 2>/dev/null [ -n "${LANES_IDX_DIR:-}" ] && [ -d "${LANES_IDX_DIR:-}" ] && rm -rf -- "$LANES_IDX_DIR" + [ -n "${LANES_WT_IDX_DIR:-}" ] && [ -d "${LANES_WT_IDX_DIR:-}" ] && rm -rf -- "$LANES_WT_IDX_DIR" # AMENDMENT 14(d) — THE NUDGE IS THE LAST THING THIS PROCESS DOES: after the # lock is released at the top of this function, and only where `commit_push` recorded a push # that landed. Tested here rather than inside the function, because `cleanup` @@ -2074,8 +2087,17 @@ index_nudge() { # THE LAG IS SAID, NEVER REFUSED (clause (b)): `` is how many commits this # checkout's `origin/` carries past the indexed one, and `unknown` where # that commit is not here to count from. -index_open() { +# +# `index_open worktrees` IS THE SAME DOOR FOR THE `worktrees` TABLE +# (opensoft/openRepoTools#161): `export --worktrees` writes every +# `inventory:` source the store holds, `LANES_WT_IDX_DIR` is set +# and `LANES_IDX_DIR` is NOT — so not one published read of the register is +# answered out of the index by it — and the line names each source with the +# time it was synced, because an inventory has no commit to count a lag in. +index_open() { # [worktrees] + io_kind="${1-register}" LANES_IDX_DIR="" + LANES_WT_IDX_DIR="" if [ "${LANES_INDEX:-}" = off ]; then printf 'read: sources (index off: LANES_INDEX=off)\n' >&2 return 1 @@ -2094,10 +2116,12 @@ index_open() { return 1 fi io_rc=0 + io_xa="" + [ "$io_kind" = worktrees ] && io_xa=--worktrees if command -v timeout >/dev/null 2>&1; then - timeout -k 2 30 "$io_bin" export --out "$io_dir" /dev/null 2>"$io_dir/.stderr" || io_rc=$? + timeout -k 2 30 "$io_bin" export --out "$io_dir" ${io_xa:+"$io_xa"} /dev/null 2>"$io_dir/.stderr" || io_rc=$? else - "$io_bin" export --out "$io_dir" /dev/null 2>"$io_dir/.stderr" || io_rc=$? + "$io_bin" export --out "$io_dir" ${io_xa:+"$io_xa"} /dev/null 2>"$io_dir/.stderr" || io_rc=$? fi if [ "$io_rc" != 0 ] || [ ! -f "$io_dir/meta" ]; then io_why="" @@ -2113,6 +2137,12 @@ index_open() { return 1 fi io_store="$(sed -n -e 's/^store=//p' "$io_dir/meta" | head -n 1)" + if [ "$io_kind" = worktrees ]; then + io_at="$(awk -F'\t' '$1 == "inventory" { printf "%s%s synced %s", (n++ ? "; " : ""), $2, $3 }' "$io_dir/meta")" + LANES_WT_IDX_DIR="$io_dir" + printf 'read: index (%s) at %s\n' "${io_store:-unknown}" "${io_at:-no inventory source}" >&2 + return 0 + fi io_sha="$(sed -n -e 's/^sha=//p' "$io_dir/meta" | head -n 1)" io_behind=unknown if [ -n "$io_sha" ] && have_remote_ref; then @@ -12536,6 +12566,161 @@ EOF return 0 } +# ------------------------------------- every inventoried worktree, as rows +# +# opensoft/openRepoTools#161 — THE INVENTORY OF EVERY LANE ON THIS WORKSTATION, +# ONE ROW PER TREE, and the one implementation of that sentence: `lanes +# --worktrees` renders what this prints and `lanes-index sync --source +# inventory` copies it into the derived index's `worktrees` table, so the +# listing a person reads and the table another workstation queries cannot come +# to disagree about what a lane left. US-separated, sixteen fields: +# +# TREE +# +# +# UNREAD +# +# IT READS WHAT #97 WROTE AND NOTHING THAT IT DID NOT. The lanes are the +# listing's own (`lanes_rows --all --closed`, so a closed or dormant lane's +# leftovers are not hidden), and so is `writer_live` — the listing's LIVE, out +# of this workstation's session records, empty where those could not be read +# (Amendment 7(d): never "no"). The trees are `lane_trees_list`'s sidecars and +# NO DIRECTORY IS WALKED for a tree no sidecar names: that is +# `lane-reconcile`'s act, and it is an act. `owner` is `lane_is_managed_owned`'s +# answer — the read `managed-projection` makes — as `legacy`, `managed ` +# or `unknown`, and `lifecycle` is the snapshot's own word for a LEGACY lane +# only: a managed or unknown lane's lifecycle is not this tooling's to +# pronounce on (ruling 2026-10-04), and neither is one whose snapshot is +# missing, unreadable or of another schema, so those are `INDETERMINATE`. +# `generation` and `operation` are the snapshot's; `provenance` is the +# generation the TREE'S OWN record was filed under, which is behind +# `generation` exactly when the observation predates the lane's current +# transition. +# +# EVERY FIELD IS THE LAST OBSERVATION, NEVER RECOMPUTED: `branch`, `dirty`, +# `unpushed` and `last_seen_utc` are what the sidecar says the handoff saw. +# Two git reads are made per tree the inventory names, both of the RECORDED +# head and neither of the working tree: `last_commit` (` `) and +# `base`, its merge-base with `origin/main`, else `main`, else `origin/HEAD` +# — in the tree, or in its recorded checkout once the tree is gone. A detached +# head is `detached `; a record that could not be read is `unreadable +# sidecar` and one of another schema `unknown schema `, with the record's +# id where its path would be (a path always begins with `/`). A count that is +# `unknown` is empty — an INTEGER column has no word for it. +# +# The rows are sorted bytewise, which is the order the index's copy is written +# in too: the two reads of one inventory are the same bytes. +# 0 rows · 8 none · 1 the lane listing could not be read +worktree_rows() { # [] + wr_one="${1-}"; wr_out="" + wr_live_ok=1 + live_session_ids >/dev/null 2>&1 || wr_live_ok=0 + wr_lanes=""; wr_lrc=0 + if [ -n "$wr_one" ]; then + wr_lanes="$(lanes_rows --lane "$wr_one" --closed)" || wr_lrc=$? + else + wr_lanes="$(lanes_rows --all --closed)" || wr_lrc=$? + fi + case "$wr_lrc" in 0) : ;; 8) return 8 ;; *) return 1 ;; esac + wr_tab="$(printf '\t')" + while IFS="$wr_tab" read -r wr_l wr_st wr_w wr_pf wr_win wr_sid wr_dir wr_rest; do + [ -n "${wr_l:-}" ] || continue + # ONE CONTROL-ROOT READ FOR A LANE WITH NO TREES, which is nearly every lane + # in the estate: the sidecars are listed only where a `trees` directory is. + # The listing's column 7 IS `lane_payload_field dir` — the same last + # lane-kind payload carrying `dir`, out of the same published logs, `none` + # where there is none — so it is handed to `lane_control_root` as the + # payload it takes, rather than having each of sixty lanes' logs read a + # second time to find what the listing has just said (measured on Eagle's + # 61 lanes: 7.9 s for the whole read with the second reading, 2.7 s + # without it). `none` is not absolute, so it falls to the same third rung + # an absent `dir` does. + wr_root=""; wr_rrc=0 + wr_root="$(lane_control_root "$wr_l" "dir ${wr_dir:-none}" 2>/dev/null)" || wr_rrc=$? + [ "$wr_rrc" = 0 ] || continue + [ -d "$wr_root/trees" ] || continue + # A DIRECTORY THAT IS THERE AND CANNOT BE LISTED IS NOT AN EMPTY ONE, and + # the trees it holds are not trees this lane no longer has (R22). + if [ ! -r "$wr_root/trees" ] || [ ! -x "$wr_root/trees" ]; then + wr_out="${wr_out}UNREAD$US$WS$US$wr_l${US}its trees directory $wr_root/trees is there and could not be listed +" + continue + fi + wr_trees=""; wr_trc=0 + wr_trees="$(lane_trees_list "$wr_l" 2>/dev/null)" || wr_trc=$? + case "$wr_trc" in 0) : ;; *) continue ;; esac + wr_live=0 + [ "$wr_st" = LIVE ] && wr_live=1 + [ "$wr_live_ok" = 1 ] || wr_live="" + wr_own=""; wr_orc=0 + wr_own="$(lane_is_managed_owned "$wr_l" 2>/dev/null)" || wr_orc=$? + wr_own="$(printf '%s\n' "$wr_own" | head -n 1)" + case "$wr_orc" in + 0) if [ -n "$wr_own" ]; then wr_owner="managed $wr_own"; else wr_owner=unknown; fi ;; + 8) wr_owner=legacy ;; + *) wr_owner=unknown ;; + esac + wr_life=INDETERMINATE; wr_gen=""; wr_op="" + wr_snap=""; wr_src=0 + wr_snap="$(lane_state_read "$wr_l" 2>/dev/null)" || wr_src=$? + if [ "$wr_src" = 0 ]; then + wr_s="$(printf '%s\n' "$wr_snap" | awk -F'\t' '$1 == "state" { print $2; exit }')" + if [ "$wr_s" != UNKNOWN-SCHEMA ]; then + wr_gen="$(printf '%s\n' "$wr_snap" | awk -F'\t' '$1 == "generation" { print $2; exit }')" + wr_op="$(printf '%s\n' "$wr_snap" | awk -F'\t' '$1 == "operation" { print $2; exit }')" + case "$wr_gen" in ''|*[!0-9]*) wr_gen="" ;; esac + case "$wr_s" in RUNNING|SWAPPING|SWAPPED|CLOSED) wr_life="$wr_s" ;; esac + fi + fi + [ "$wr_owner" = legacy ] || wr_life=INDETERMINATE + while IFS="$US" read -r t_id t_p t_b t_h t_u t_d t_n t_w t_ob t_co t_g t_o t_s; do + [ -n "${t_id:-}" ] || continue + t_base=""; t_last="" + if [ "${t_s:-}" = "" ]; then + t_branch="unreadable sidecar"; [ -n "${t_p:-}" ] || t_p="$t_id" + t_d=""; t_n=""; t_g=""; t_ob="" + elif [ "${t_s:-}" != "$LANE_STATE_SCHEMA" ]; then + t_branch="unknown schema ${t_s:-}"; [ -n "${t_p:-}" ] || t_p="$t_id" + t_d=""; t_n=""; t_g=""; t_ob="" + else + if [ "$t_b" = detached ]; then t_branch="detached $t_h"; else t_branch="$t_b"; fi + case "$t_h" in + ''|*[!0-9a-f]*) : ;; + *) + if [ "${#t_h}" -eq 40 ]; then + t_gd="" + if [ -d "$t_p" ]; then t_gd="$t_p"; elif [ -d "${t_co:-}" ]; then t_gd="$t_co"; fi + if [ -n "$t_gd" ]; then + t_last="$(git -C "$t_gd" show -s --format='%H %s' "$t_h^{commit}" 2>/dev/null | head -n 1 | tr -d '\037')" + [ -n "$t_last" ] || t_last="$t_h" + for t_ref in refs/remotes/origin/main refs/heads/main refs/remotes/origin/HEAD; do + if git -C "$t_gd" rev-parse --verify -q "$t_ref^{commit}" >/dev/null 2>&1; then + t_base="$(git -C "$t_gd" merge-base "$t_h" "$t_ref" 2>/dev/null || :)" + break + fi + done + else + t_last="$t_h" + fi + fi ;; + esac + case "$t_d" in ''|*[!0-9]*) t_d="" ;; esac + case "$t_n" in ''|*[!0-9]*) t_n="" ;; esac + case "$t_g" in ''|*[!0-9]*) t_g="" ;; esac + fi + wr_out="${wr_out}TREE$US$WS$US$wr_l$US$wr_owner$US$t_p$US$t_branch$US$t_base$US$wr_life$US$wr_gen$US$wr_op$US$wr_live$US$t_ob$US$t_last$US$t_d$US$t_n$US$t_g +" + done <` source the store holds — this + # workstation's and any other that syncs into the same Postgres, which is the + # point of the table — and stderr says which was read: `read: index () + # at inventory: synced `, or `read: sources (index )` and the + # sidecars answer, exactly as without the flag. NO ACT READS THIS: not + # `lane-reconcile`, not `lane-start`, not `lane-end`, not a sweep — each of + # them reads the sidecars and the disk, and this is a listing. + # + # Local by default, as `lanes` is: `--fetch` refreshes the register first. + # 0 rows · 8 none · 2 a lane name refused (Amendment 15) · 64 usage · 1 a read failed + worktrees) + wt_lane=""; wt_all=0; wt_index=0; wt_fetch=0 + wt_usage="worktrees [ | --all] [--index] [--fetch]" + while [ $# -gt 0 ]; do + case "$1" in + --all) wt_all=1; shift ;; + --index) wt_index=1; shift ;; + --fetch) wt_fetch=1; shift ;; + --) shift ;; + -*) die "unknown option '$1' for worktrees ($wt_usage)" 64 ;; + *) [ -z "$wt_lane" ] || die "worktrees takes ONE lane, or --all: $wt_usage" 64; wt_lane="$1"; shift ;; + esac + done + [ "$wt_all" = 0 ] || [ -z "$wt_lane" ] || die "worktrees takes ONE lane OR --all, not both: $wt_usage" 64 + [ -z "$wt_lane" ] || check_lane_name "$wt_lane" + [ "$wt_fetch" = 1 ] || LANES_NO_FETCH=1 + log_sync + if [ -n "$wt_lane" ]; then + wt_lane="$(canon_lane "$wt_lane")" || exit 2 # Amendment 15 + fi + if [ "$wt_index" = 1 ] && index_open worktrees; then + wt_out="$(awk -F"$US" -v l="$(lc "$wt_lane")" 'l == "" || tolower($3) == l' "$LANES_WT_IDX_DIR/worktrees" 2>/dev/null)" + [ -n "$wt_out" ] || exit 8 + printf '%s\n' "$wt_out" + else + wt_out=""; wt_rc=0 + wt_out="$(worktree_rows "$wt_lane")" || wt_rc=$? + case "$wt_rc" in + 0) : ;; + 8) exit 8 ;; + *) die "the lane listing the inventory is read through could not be read (exit $wt_rc). That is NOT 'no worktree is inventoried' (Amendment 7(d))." 1 ;; + esac + printf '%s\n' "$wt_out" + fi + ;; + *) - die "unknown subcommand '$cmd' (verify-row|set-row-state|append-row-status|replace-in-row|append-session-id|append-line|add-row|retire-rows|archive-rows|migrate-state-cells|commit|rename-lane|log|claim|release|who|history|swapped|session-start|guard|idle-holders|live-holder|window-session|transcript-holders|binding|request-handoff|session-lane|window-lane|lane-dir|lane-profile|lane-agent|lane-transcript|lane-last|workspace-root|last-session|forks|duplicate-holder|workstation|fetch-age|lanes|lane-groups|next-free|sibling-filter|resolve-repo|lane-objects|register-row|register-row-local|retired-identity|canon-lane|resolve-home|managed-projection|lane-state|set-lane-state|lane-trees|set-lane-tree|lane-tree-now|lane-reconcile)" 2 + die "unknown subcommand '$cmd' (verify-row|set-row-state|append-row-status|replace-in-row|append-session-id|append-line|add-row|retire-rows|archive-rows|migrate-state-cells|commit|rename-lane|log|claim|release|who|history|swapped|session-start|guard|idle-holders|live-holder|window-session|transcript-holders|binding|request-handoff|session-lane|window-lane|lane-dir|lane-profile|lane-agent|lane-transcript|lane-last|workspace-root|last-session|forks|duplicate-holder|workstation|fetch-age|lanes|lane-groups|next-free|sibling-filter|resolve-repo|lane-objects|register-row|register-row-local|retired-identity|canon-lane|resolve-home|managed-projection|lane-state|set-lane-state|lane-trees|set-lane-tree|lane-tree-now|lane-reconcile|worktrees)" 2 ;; esac From 94fdb4554c41f7da44952ec25c47a1463861742a Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 17:28:17 +0000 Subject: [PATCH 02/12] Index this workstation's worktree inventory as its own source lanes-index gains the source `inventory:` (opensoft/ openRepoTools#161), the "later word" Amendment 14(e) left open for #97's inventories. `sync`/`reconcile --source inventory` read `lanes-edit.sh worktrees --all` - the inventory through the helper that writes it - and upsert the `worktrees` table schema 1 already created with #161's columns, so no store needs a new version. Rows are keyed by (source, `:`) and versioned by a digest of the row and its provenance (the generation the tree's own record was filed under); a tree the inventory no longer names leaves in the same sync. The provenance moves only forward per lane: a sync that reads a lane's lifecycle generation below the one its rows were written at writes nothing for that lane, and reconcile takes it as it now stands. A lane whose trees directory cannot be listed keeps its rows. The source's own row carries a fingerprint of the table as its compare-and-swap token and a sync count. `pr` is asked of `gh` best effort, never under LANES_NO_GITHUB=1, and a MERGED or CLOSED answer is not asked again while the branch stands. The sync lock's rerun mark now names its source, so a register sync that holds the lock when an inventory nudge arrives syncs the inventory before it lets go. `export --worktrees` writes every inventory source the store holds as the helper's own lines, and `status` names this workstation's inventory. A container with no LANES_WORKSTATION syncs nothing. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lanes-index | 558 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 510 insertions(+), 48 deletions(-) diff --git a/lanes-index b/lanes-index index 22d9bc67..94a14f60 100755 --- a/lanes-index +++ b/lanes-index @@ -6,7 +6,8 @@ Amendment 14, ratified by Brett Heap 2026-10-05, verbatim "ratify 48"). lanes-index sync [--source ] write behind what LANDED lanes-index reconcile [--source ] [--dry-run] read the source whole lanes-index status per source: store, lag, errors - lanes-index export --out [--source ] the read flags' reader + lanes-index export --out [--source | --worktrees] + the read flags' reader THE REGISTER'S TRUTH STAYS WHERE IT IS (clause (a)). The workspace repository's `lanes/LANES.md`, `lanes/log/*.md`, `lanes/aliases.tsv` and `lanes/archive/` @@ -31,6 +32,23 @@ indexed commit is compared and swapped IN THE SAME TRANSACTION as the rows it covers, and a sync from a commit that does not descend from it writes nothing. Nothing in any source is ever written from here. +THE INVENTORY IS A SECOND SOURCE (opensoft/openRepoTools#161, the "later word" +clause (e) left open): `inventory:`, THIS workstation's #97 +worktree inventory, copied into the `worktrees` table. It is read through +`lanes-edit.sh worktrees --all` - the sidecars, the lifecycle snapshot, +`managed-projection`'s owner and the listing's liveness, in the helper that +wrote them - and never through a parser of this file's own or a walk of the +disk. Its rows are keyed by (source, `:`) and versioned by the same +digest; a tree the inventory no longer names leaves in the same sync. Its +provenance moves only forward PER LANE: a sync that reads a lane's lifecycle +generation below the one its rows were written at writes nothing for that lane +(`reconcile` takes the lane as it now stands, and says so), and the source's own +row carries a fingerprint of the whole table as its compare-and-swap token and a +sync generation that only counts up. `lanes-edit.sh` nudges `sync --source +inventory` after every inventory write. Nothing that acts reads this table +either: `lane-reconcile`, `lane-start`, `lane-end` and the sweep read the +sidecars and the disk. + THE STORE (clause (c)): QA Postgres where this workstation's configuration names a `url=`, reached through `psql` and only then; otherwise one SQLite file on local disk. The configuration is @@ -44,7 +62,8 @@ connect to Postgres only as `reader_url=` and are refused without it - never with the writer's credential. Exit codes: 0 done · 1 a read or a write failed (nothing half-written) · 2 a -refusal (the configuration, a `--source` that is not this workstation's) · 3 +refusal (the configuration, a `--source` that is not this workstation's, an +inventory on a workstation with no name) · 3 `export` only: the index cannot answer (missing, unreachable, wiped, unknown schema - the reason is in `/why`) · 64 usage. """ @@ -85,6 +104,7 @@ UTC_RE = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z" SHA_RE = re.compile(r"^[0-9a-f]{40}$") SCHEMA_NAME_RE = re.compile(r"^[a-z_][a-z0-9_]{0,62}$") OPEN_VERBS = ("CLAIMED", "TAKEOVER", "OPENED", "LANDING", "WITHDRAWN") +INVENTORY_PREFIX = "inventory:" class Refusal(Exception): @@ -704,11 +724,12 @@ TABLE_COLUMNS = { # indexer writes nothing into them. "swap_states": ["lane", "parent_uuid", "generation", "state", "observed_utc"], "job_summaries": ["lane", "job_id", "result_summary", "observed_utc"], - # RESERVED FOR opensoft/openRepoTools#161, created EMPTY with that issue's - # columns: each lane's #97 worktree inventory under `inventory:`, - # keyed by (source, lane, path) - here the record id `:`. Amendment - # 14(e) leaves inventories to "a later word's"; #161 is that word's issue, and - # this indexer writes nothing into the table. + # opensoft/openRepoTools#161: each lane's #97 worktree inventory under + # `inventory:`, keyed by (source, lane, path) - the record id + # `:`. Amendment 14(e) left inventories to "a later word's"; #161 + # is that word's issue. Created by schema 1 with these columns before a row + # was written into it, so no store needed a new version for it. Written by + # `run_inventory` and by nothing that reads the register. "worktrees": ["workstation", "lane", "owner", "path", "branch", "base", "lifecycle", "generation", "operation", "writer_live", "last_seen_utc", "last_commit", "dirty_count", "unpushed_count", "pr_number", "pr_state"], @@ -718,8 +739,8 @@ INTEGER_COLUMNS = {"ls_order", "line_count", "unreadable_count", "archived", "ro "ordinal", "unreadable", "position", "size_bytes", "generation", "writer_live", "dirty_count", "unpushed_count", "pr_number"} -#: The tables `sync` and `reconcile` write. The ledger's two and #161's -#: `worktrees` are absent on purpose. +#: The tables `sync` and `reconcile` write for the REGISTER. The ledger's two +#: are T058's, and `worktrees` is the inventory source's alone. REGISTER_TABLES = ("source_files", "lanes", "log_lines", "register_lines", "lane_aliases", "transcript_pointers") @@ -1105,7 +1126,13 @@ class SqliteStore: f"SELECT {cols} FROM {table} WHERE source = ?", (source,))] def apply(self, source: str, old: dict | None, new_sha: str, upserts: dict, - deletes: dict) -> None: + deletes: dict, generation: int | None = None) -> None: + """The rows and the provenance, ONE transaction, the compare-and-swap + first. `new_sha` is the register's commit, or the inventory's + fingerprint of its whole table (#161), and `generation` the inventory's + sync count; a record carrying `_provenance` is stamped with that rather + than with `new_sha` - the generation its tree's own record was filed + under.""" import sqlite3 now = utc_now() try: @@ -1121,12 +1148,13 @@ class SqliteStore: if cur is None: self.conn.execute( "INSERT INTO provenance (source, commit_sha, generation, indexed_utc, " - "schema_version) VALUES (?, ?, NULL, ?, ?)", - (source, new_sha, now, SCHEMA_VERSION)) + "schema_version) VALUES (?, ?, ?, ?, ?)", + (source, new_sha, generation, now, SCHEMA_VERSION)) else: self.conn.execute( - "UPDATE provenance SET commit_sha = ?, indexed_utc = ?, schema_version = ? " - "WHERE source = ?", (new_sha, now, SCHEMA_VERSION, source)) + "UPDATE provenance SET commit_sha = ?, generation = ?, indexed_utc = ?, " + "schema_version = ? WHERE source = ?", + (new_sha, generation, now, SCHEMA_VERSION, source)) for table, ids in deletes.items(): for rec_id in ids: self.conn.execute(f"DELETE FROM {table} WHERE source = ? AND record_id = ?", @@ -1141,7 +1169,7 @@ class SqliteStore: f"DO UPDATE SET {sets}") self.conn.executemany(sql, [ [source, rec_id] + [rec.get(c) for c in cols] + - [rec["digest"], new_sha, now, SCHEMA_VERSION] + [rec["digest"], rec.get("_provenance", new_sha), now, SCHEMA_VERSION] for rec_id, rec in recs.items()]) self.conn.execute("COMMIT") except sqlite3.Error as exc: @@ -1340,7 +1368,7 @@ class PostgresStore: return out def apply(self, source: str, old: dict | None, new_sha: str, upserts: dict, - deletes: dict) -> None: + deletes: dict, generation: int | None = None) -> None: now = utc_now() want = old["commit_sha"] if old else None lines = ["SET standard_conforming_strings = on;", "BEGIN;", @@ -1353,7 +1381,8 @@ class PostgresStore: f" INSERT INTO provenance (source, commit_sha, generation, indexed_utc, " f"schema_version) VALUES ({_pg_quote(source)}, NULL, NULL, {_pg_quote(now)}, " f"{SCHEMA_VERSION}) ON CONFLICT (source) DO NOTHING;", - f" UPDATE provenance SET commit_sha = {_pg_quote(new_sha)}, indexed_utc = " + f" UPDATE provenance SET commit_sha = {_pg_quote(new_sha)}, generation = " + f"{_pg_quote(generation)}, indexed_utc = " f"{_pg_quote(now)}, schema_version = {SCHEMA_VERSION} WHERE source = " f"{_pg_quote(source)} AND commit_sha IS NOT DISTINCT FROM {_pg_quote(want)};", " GET DIAGNOSTICS n = ROW_COUNT;", @@ -1371,7 +1400,7 @@ class PostgresStore: sets = ", ".join(f"{c} = EXCLUDED.{c}" for c in allc[2:]) for rec_id, rec in recs.items(): values = [source, rec_id] + [rec.get(c) for c in cols] + \ - [rec["digest"], new_sha, now, SCHEMA_VERSION] + [rec["digest"], rec.get("_provenance", new_sha), now, SCHEMA_VERSION] lines.append(f"INSERT INTO {table} ({', '.join(allc)}) VALUES " f"({', '.join(_pg_quote(v) for v in values)}) ON CONFLICT " f"(source, record_id) DO UPDATE SET {sets};") @@ -1465,20 +1494,36 @@ class SyncLock: return False time.sleep(0.2) - def mark_rerun(self) -> None: + # ONE MARK PER SOURCE (opensoft/openRepoTools#161). The holder syncs again + # for every mark it finds, and a mark says WHICH source asked: a register + # sync holding the lock when an inventory write's nudge arrives syncs the + # inventory before it lets go, rather than the register a second time. The + # register's mark keeps the name it has always had. + KINDS = ("register", "inventory") + + def rerun_path(self, kind: str = "register") -> str: + return self.rerun if kind == "register" else f"{self.rerun}-{kind}" + + def mark_rerun(self, kind: str = "register") -> None: try: - with open(self.rerun, "w", encoding="utf-8") as handle: + with open(self.rerun_path(kind), "w", encoding="utf-8") as handle: handle.write(utc_now() + "\n") except OSError: pass - def take_rerun(self) -> bool: + def take_rerun(self, kind: str = "register") -> bool: try: - os.unlink(self.rerun) + os.unlink(self.rerun_path(kind)) return True except OSError: return False + def take_reruns(self) -> list: + return [kind for kind in self.KINDS if self.take_rerun(kind)] + + def has_rerun(self) -> bool: + return any(os.path.exists(self.rerun_path(kind)) for kind in self.KINDS) + def release(self) -> None: if self.handle is not None: self.handle.close() @@ -1490,6 +1535,10 @@ class SyncLock: def resolve_source(requested: str | None, root: str) -> str: source = source_of(root) if requested and requested != source: + if is_inventory(requested): + raise Refusal(f"{requested} is this workstation's worktree inventory, which " + f"`sync`/`reconcile --source inventory` write and `export " + f"--worktrees` reads; this is the register's path") if requested.startswith("ledger:"): raise Refusal(f"{requested} is the managed ledger's half, which T058 of the 001 " f"feature writes (Amendment 14(g) act 4); this indexer writes the " @@ -1629,10 +1678,323 @@ def run_index(mode: str, requested: str | None, dry_run: bool) -> int: return 0 +# ======================================== the inventory (opensoft/openRepoTools#161) +# +# THIS WORKSTATION'S #97 WORKTREE INVENTORY, AS A SECOND SOURCE. Amendment 14(e) +# left "#97's lifecycle snapshots and worktree inventories, local to one +# workstation by design" to a later word; #161 is it. The source is +# `inventory:` - one per workstation, because the sidecars are on +# that workstation's disk and nowhere else - and its rows are what +# `lanes-edit.sh worktrees --all` prints, which is the inventory read through +# the helper that writes it. Nothing here parses a sidecar, a snapshot or an +# ownership marker, and nothing here lists a directory. + +def _helper_run(args: list, timeout: float, what: str) -> subprocess.CompletedProcess: + helper = lanes_edit_path() + if not helper: + raise StoreUnusable("unread", f"no lanes-edit.sh beside this file or on PATH, so {what} " + f"cannot be read") + try: + return subprocess.run([helper, *args], capture_output=True, encoding="utf-8", + errors="replace", check=False, env=helper_env(), + stdin=subprocess.DEVNULL, timeout=timeout) + except subprocess.TimeoutExpired: + raise StoreUnusable("unread", f"`lanes-edit.sh {' '.join(args)}` did not answer within " + f"{int(timeout)} s") + except OSError as exc: + raise StoreUnusable("unread", f"`lanes-edit.sh {' '.join(args)}` could not be run: {exc}") + + +def this_workstation() -> tuple[str, str]: + """`lanes-edit.sh workstation`'s first two columns: the name every row of + this workstation is filed under, and where it came from.""" + proc = _helper_run(["workstation"], 60, "this workstation's name") + first = (proc.stdout.splitlines() or [""])[0].split("\t") + if proc.returncode != 0 or not first[0]: + why = (proc.stderr.strip().splitlines() or ["no answer"])[-1] + raise StoreUnusable("unread", f"`lanes-edit.sh workstation` exited {proc.returncode}: {why}") + return first[0], (first[1] if len(first) > 1 else "") + + +def inventory_source(requested: str | None) -> str: + """`inventory:`. `--source inventory` is the short name for + it; another workstation's is refused, as another register is - its sidecars + are on its own disk. AND A CONTAINER WITH NO NAME FILES NOTHING (Amendment + 11, `R-A11-14`): a source named for a container's own id is a source no + restart of any workstation will ever find again.""" + name, how = this_workstation() + if how == "container-unset": + raise Refusal("this container has no LANES_WORKSTATION, so its inventory would be " + f"filed under the container's own id `{name}` - a source no restart of " + "any workstation finds again (Amendment 11, R-A11-14). Export " + "LANES_WORKSTATION, as the launcher does") + source = f"{INVENTORY_PREFIX}{name}" + if requested and requested not in ("inventory", source): + raise Refusal(f"this workstation's inventory is {source}, not {requested}: an " + f"inventory is synced by the workstation whose disk holds it") + return source + + +def _int_or_none(value: str): + return int(value) if value.isdigit() else None + + +def tree_record(fields: list) -> dict: + """One `TREE` line of `lanes-edit.sh worktrees`, as the `worktrees` row it + is. An empty field is NULL: a count the inventory recorded as `unknown`, a + base or a last commit that could not be read, a lane with no snapshot.""" + text = [f if f != "" else None for f in fields] + return {"workstation": text[1], "lane": fields[2], "owner": text[3], "path": fields[4], + "branch": text[5], "base": text[6], "lifecycle": text[7], + "generation": _int_or_none(fields[8]), "operation": text[9], + "writer_live": _int_or_none(fields[10]), "last_seen_utc": text[11], + "last_commit": text[12], "dirty_count": _int_or_none(fields[13]), + "unpushed_count": _int_or_none(fields[14]), "pr_number": None, "pr_state": None, + "_provenance": text[15]} + + +def tree_line(row: dict, provenance) -> str: + """The `TREE` line a stored row came from - the inverse of `tree_record`, so + that `lanes --index --worktrees` and `lanes --worktrees` print the same + bytes wherever the index is current.""" + def text(value) -> str: + return "" if value is None else str(value) + return US.join(["TREE"] + [text(row.get(c)) for c in ( + "workstation", "lane", "owner", "path", "branch", "base", "lifecycle", "generation", + "operation", "writer_live", "last_seen_utc", "last_commit", "dirty_count", + "unpushed_count")] + [text(provenance)]) + + +def inventory_digest(record: dict) -> str: + """The register's digest, and the row's provenance with it: two + observations of one tree filed under two generations are two versions.""" + content = ["worktrees"] + [record.get(col) for col in TABLE_COLUMNS["worktrees"]] + \ + [record.get("_provenance")] + blob = json.dumps(content, ensure_ascii=False, separators=(",", ":"), sort_keys=False) + return hashlib.sha256(blob.encode("utf-8")).hexdigest() + + +def collect_inventory() -> tuple[dict, dict]: + """Every tree the inventory names, keyed `:`, and every lane + whose inventory is there and could not be read - whose rows are KEPT, never + taken for a lane with no trees (Amendment 7(d)).""" + proc = _helper_run(["worktrees", "--all"], 600, "the worktree inventory") + if proc.returncode == 8: + return {}, {} + if proc.returncode != 0: + why = (proc.stderr.strip().splitlines() or ["no answer"])[-1] + raise StoreUnusable("unread", f"`lanes-edit.sh worktrees --all` exited " + f"{proc.returncode}: {why}") + trees, unread = {}, {} + for line in proc.stdout.split("\n"): + if not line: + continue + fields = line.split(US) + if fields[0] == "UNREAD" and len(fields) >= 4: + unread[fields[2]] = fields[3] + continue + if fields[0] != "TREE" or len(fields) != 16: + raise StoreUnusable("unread", f"`lanes-edit.sh worktrees` printed a line this " + f"indexer does not read ({len(fields)} fields): {line[:120]!r} - " + f"is the helper on PATH another version?") + record = tree_record(fields) + trees[f"{record['lane']}:{record['path']}"] = record + return trees, unread + + +def _github_repo(path: str) -> str: + if not os.path.isdir(path): + return "" + proc = subprocess.run(["git", "-C", path, "remote", "get-url", "origin"], capture_output=True, + encoding="utf-8", errors="replace", check=False, + stdin=subprocess.DEVNULL) + url = proc.stdout.strip() + if proc.returncode != 0 or "github.com" not in url: + return "" + repo = _norm_repo(url) + return repo if re.match(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$", repo) else "" + + +def pr_of(path: str, branch_name: str): + """`(number, state)` of the newest pull request whose head is this branch, + asked of `gh` - BEST EFFORT, and `(None, None)` wherever it does not answer. + The one network read of this source, and only of it.""" + repo = _github_repo(path) + if not repo: + return None, None + env = dict(os.environ) + env["GH_PROMPT_DISABLED"] = "1" + try: + proc = subprocess.run(["gh", "pr", "list", "--repo", repo, "--head", branch_name, + "--state", "all", "--json", "number,state", "--limit", "1"], + capture_output=True, encoding="utf-8", errors="replace", + check=False, env=env, stdin=subprocess.DEVNULL, timeout=20) + found = json.loads(proc.stdout) if proc.returncode == 0 else [] + except (OSError, subprocess.TimeoutExpired, ValueError): + return None, None + if not isinstance(found, list) or not found or not isinstance(found[0], dict): + return None, None + number = found[0].get("number") + state = found[0].get("state") + return (number if isinstance(number, int) else None, + state if isinstance(state, str) and state else None) + + +def fill_prs(trees: dict, stored: dict, mode: str, held: dict | None = None) -> int: + """`pr_number`/`pr_state` for every tree on a branch. A `sync` carries a + stored answer over for a tree whose branch and last commit have not moved + where that answer is final (MERGED, CLOSED) or `gh` is not to be asked; + `reconcile` asks again. Returns how many were asked.""" + use_gh = os.environ.get("LANES_NO_GITHUB", "") != "1" and bool(shutil.which("gh")) + ask = [] + for rec_id, rec in trees.items(): + name = rec.get("branch") or "" + if not name or " " in name or name in ("unknown", "HEAD") or rec["lane"] in (held or {}): + continue + old = stored.get(rec_id) + if mode == "sync" and old is not None and old.get("branch") == rec["branch"] \ + and old.get("last_commit") == rec["last_commit"] \ + and (old.get("pr_state") in ("MERGED", "CLOSED") or not use_gh): + rec["pr_number"], rec["pr_state"] = old.get("pr_number"), old.get("pr_state") + continue + if use_gh: + ask.append(rec_id) + if ask: + with ThreadPoolExecutor(max_workers=max(1, min(8, len(ask)))) as pool: + answers = list(pool.map(lambda rid: pr_of(trees[rid]["path"], trees[rid]["branch"]), + ask)) + for rec_id, (number, state) in zip(ask, answers): + trees[rec_id]["pr_number"], trees[rec_id]["pr_state"] = number, state + return len(ask) + + +def run_inventory(mode: str, requested: str | None, dry_run: bool) -> int: + cfg = load_config() + source = inventory_source(requested) + store = open_store(cfg, "writer") + if dry_run: + try: + store.open_read() + except StoreUnusable as exc: + if exc.kind not in ("missing", "wiped"): + raise + store = None + else: + store.open_write() + old = store.provenance(source) if store is not None else None + trees, unread = collect_inventory() + label = f"{source} ({cfg['store']})" + cols = TABLE_COLUMNS["worktrees"] + stored = {} if store is None else { + r["record_id"]: r for r in store.rows("worktrees", source, cols + ["digest", "provenance"])} + + # THE PROVENANCE MOVES ONLY FORWARD, PER LANE (14(d)). A lane's lifecycle + # generation is monotonic by `set-lane-state`'s own fence, so a read BELOW + # the generation a lane's rows were written at is a snapshot that was moved + # aside or replaced - and a `sync` writes nothing for that lane rather than + # roll its rows back. `reconcile` is the act that takes the source as it now + # stands, and it says so. A lane whose inventory could not be read keeps its + # rows in either: a read that failed is never an answer. + held: dict = {} + for lane, why in unread.items(): + held[lane] = f"its inventory is there and could not be read ({why})" + read_gen: dict = {} + for rec in trees.values(): + read_gen[rec["lane"]] = max(read_gen.get(rec["lane"], 0), rec["generation"] or 0) + stored_gen: dict = {} + for row in stored.values(): + stored_gen[row["lane"]] = max(stored_gen.get(row["lane"], 0), row["generation"] or 0) + taken: list = [] + for lane, gen in sorted(read_gen.items()): + if lane in held or gen >= stored_gen.get(lane, 0): + continue + if mode == "sync": + held[lane] = (f"its lifecycle generation reads {gen} and its rows were written at " + f"{stored_gen[lane]}; the provenance only moves forward, so nothing " + f"is written for it - `{PROG} reconcile --source inventory` takes " + f"the lane as it now stands") + else: + taken.append(f" lane {lane}: its lifecycle generation went back from " + f"{stored_gen[lane]} to {gen} (a snapshot moved aside or replaced); " + f"reconcile takes it as it now stands") + + fill_prs(trees, stored, mode, held) + upserts, deletes, kept = {}, [], 0 + for rec_id, rec in trees.items(): + rec["digest"] = inventory_digest(rec) + if rec["lane"] in held: + continue + row = stored.get(rec_id) + if row is not None: + if mode == "reconcile": + as_written = {c: row.get(c) for c in cols} + as_written["_provenance"] = row.get("provenance") + same = inventory_digest(as_written) == rec["digest"] == row.get("digest") + else: + same = row.get("digest") == rec["digest"] + if same: + kept += 1 + continue + upserts[rec_id] = rec + for rec_id, row in stored.items(): + if row["lane"] not in held and rec_id not in trees: + deletes.append(rec_id) + after = {rid: rec["digest"] for rid, rec in trees.items() if rec["lane"] not in held} + after.update({rid: row["digest"] for rid, row in stored.items() if row["lane"] in held}) + fingerprint = hashlib.sha256("\n".join( + f"{rid}\t{digest}" for rid, digest in sorted(after.items())).encode("utf-8")).hexdigest() + + n_up, n_del = len(upserts), len(deletes) + if mode == "reconcile": + say(f"{PROG} reconcile: {label}{' - DRY RUN' if dry_run else ''}") + say(f" {'worktrees':<20} {kept:>6} kept {n_up:>6} upserted {n_del:>6} removed") + for line in taken: + say(line) + for lane, why in sorted(held.items()): + say(f" lane {lane}: its rows are kept as they are - {why}") + if dry_run: + say(f" total: {n_up} to upsert, {n_del} to remove - nothing written (--dry-run)") + return 0 + if old is not None and not upserts and not deletes: + if mode == "sync": + say(f"{PROG} sync: {label} - the index is current; nothing to write") + else: + say(" total: nothing differs; nothing written") + write_status(source, last_attempt_utc=utc_now(), last_sync_utc=utc_now(), + last_result="current", last_error="") + return 0 + try: + generation = int((old or {}).get("generation") or 0) + 1 + except (TypeError, ValueError): + generation = 1 + store.apply(source, old, fingerprint, {"worktrees": upserts} if upserts else {}, + {"worktrees": deletes} if deletes else {}, generation=generation) + write_status(source, last_attempt_utc=utc_now(), last_sync_utc=utc_now(), + last_result=f"{n_up} upserted, {n_del} removed at sync {generation}", + last_error="") + if mode == "sync": + say(f"{PROG} sync: {label} - {n_up} upserted, {n_del} removed") + else: + say(f" total: {n_up} upserted, {n_del} removed") + return 0 + + +def is_inventory(requested: str | None) -> bool: + return bool(requested) and (requested == "inventory" or requested.startswith(INVENTORY_PREFIX)) + + +def _sync_once(kind: str, requested: str | None) -> int: + if kind == "inventory": + return _guarded(lambda: run_inventory("sync", requested, False), requested, kind) + return _guarded(lambda: run_index("sync", requested, False), requested, kind) + + def cmd_sync(requested: str | None) -> int: if os.environ.get("LANES_INDEX", "") == "off": say(f"{PROG} sync: LANES_INDEX=off - nothing indexed") return 0 + kind = "inventory" if is_inventory(requested) else "register" + asked = {kind: requested} lock = SyncLock() # THE HANDSHAKE, BOTH HALVES (Codex on #164). A sync that finds the lock # held leaves its mark and then tries ONCE MORE: if that fails, somebody @@ -1640,9 +2002,10 @@ def cmd_sync(requested: str | None) -> int: # marks after it lets go. The holder, for its part, looks again AFTER it # releases - a mark written between its last look and its release is # found there, and it takes the lock back for it. No mark is left with - # nobody due to read it. + # nobody due to read it. AND THE MARK NAMES ITS SOURCE (#161): the holder + # syncs every source a mark names, its own included, before it lets go. if not lock.acquire(0): - lock.mark_rerun() + lock.mark_rerun(kind) if not lock.acquire(0): say(f"{PROG} sync: another sync holds {lock.path}; it will sync again before " f"it is done") @@ -1650,22 +2013,29 @@ def cmd_sync(requested: str | None) -> int: rc = 0 try: rounds = 0 + due = [kind] while True: + for marked in lock.take_reruns(): + if marked not in due: + due.append(marked) + if not due: + lock.release() + if not lock.has_rerun() or not lock.acquire(0): + return rc + continue + current = due[0] rounds += 1 - lock.take_rerun() - rc = _guarded(lambda: run_index("sync", requested, False), requested) - if rc == MOVED and rounds < 5: + result = _sync_once(current, asked.get(current)) + if result == MOVED and rounds < 5: continue - if rc == MOVED: + due.pop(0) + if result == MOVED: say(f"{PROG} sync: origin/{branch()} kept moving through {rounds} reads; " f"nothing written - the next write's nudge syncs it") - return 0 - if rc != 0 or rounds >= 5: - return rc - if os.path.exists(lock.rerun): - continue - lock.release() - if not os.path.exists(lock.rerun) or not lock.acquire(0): + result = 0 + if result != 0: + return result + if rounds >= 8: return rc finally: lock.release() @@ -1675,11 +2045,15 @@ def cmd_reconcile(requested: str | None, dry_run: bool) -> int: if os.environ.get("LANES_INDEX", "") == "off" and not dry_run: say(f"{PROG} reconcile: LANES_INDEX=off - nothing indexed") return 0 + kind = "inventory" if is_inventory(requested) else "register" lock = SyncLock() if not dry_run and not lock.acquire(60): note(f"another sync has held {lock.path} for 60 s; nothing written") return 1 try: + if kind == "inventory": + return _guarded(lambda: run_inventory("reconcile", requested, dry_run), requested, + kind) for _attempt in range(3): rc = _guarded(lambda: run_index("reconcile", requested, dry_run), requested) if rc != MOVED: @@ -1695,7 +2069,7 @@ def cmd_reconcile(requested: str | None, dry_run: bool) -> int: MOVED = -1 -def _guarded(act, requested) -> int: +def _guarded(act, requested, kind: str = "register") -> int: try: return act() except SourceMoved as exc: @@ -1703,15 +2077,22 @@ def _guarded(act, requested) -> int: return MOVED except CasLost as exc: note(str(exc)) - _record_error(requested, str(exc)) + _record_error(requested, str(exc), kind) return 1 except StoreUnusable as exc: note(f"{exc.kind}: {exc.detail} - nothing written") - _record_error(requested, f"{exc.kind}: {exc.detail}") + _record_error(requested, f"{exc.kind}: {exc.detail}", kind) return 1 -def _record_error(requested, message: str) -> None: +def _record_error(requested, message: str, kind: str = "register") -> None: + if kind == "inventory": + try: + source = inventory_source(requested) + except (StoreUnusable, Refusal): + source = requested if requested and requested != "inventory" else "inventory:?" + write_status(source, last_attempt_utc=utc_now(), last_error=f"{utc_now()} {message}") + return try: source = requested or source_of(workspace_root()) except StoreUnusable: @@ -1747,10 +2128,33 @@ def cmd_status() -> int: sources = [p["source"] for p in provs] if source and source not in sources: sources.append(source) + # THIS WORKSTATION'S INVENTORY TOO (#161), indexed or not - its name is the + # helper's, and a helper that cannot say it costs the line, not the status. + inv_source = "" + try: + inv_source = inventory_source(None) + except (StoreUnusable, Refusal): + inv_source = "" + if inv_source and inv_source not in sources: + sources.append(inv_source) for name in sorted(set(sources) | set(local)): prov = next((p for p in provs if p["source"] == name), None) say(f"source {name}") - if prov is None: + if name.startswith(INVENTORY_PREFIX): + if prov is None: + say(" provenance: none - this inventory is not indexed (a wiped or " + "never-synced index; `lanes-index reconcile --source inventory` builds it)") + else: + fp = prov.get("commit_sha") or "" + say(f" provenance: inventory@{fp[:12]} (sync {prov.get('generation')}, indexed " + f"{prov['indexed_utc']}, schema {prov['schema_version']})") + try: + count = len(store.rows("worktrees", name, ["lane"])) + say(f" rows: {count} worktree(s)") + except StoreUnusable as exc: + say(f" rows: unread - {exc.kind}: {exc.detail}") + rc = 1 + elif prov is None: say(" provenance: none - this source is not indexed (a wiped or never-synced " "index; `lanes-index reconcile` builds it)") elif prov.get("commit_sha"): @@ -1771,9 +2175,11 @@ def cmd_status() -> int: return rc -def cmd_export(out_dir: str, requested: str | None) -> int: +def cmd_export(out_dir: str, requested: str | None, worktrees: bool = False) -> int: """The read flags' reader, and nothing else's: the index's image of this - workstation's register in the shapes `lanes-edit.sh` reads from origin. + workstation's register in the shapes `lanes-edit.sh` reads from origin - + or, with `worktrees` (#161), every inventory source's rows as the `TREE` + lines `lanes-edit.sh worktrees` prints. It writes FILES into `out_dir` and prints nothing a caller parses. On success `meta`; on any reason the index cannot answer, `why` - one line, @@ -1792,6 +2198,8 @@ def cmd_export(out_dir: str, requested: str | None) -> int: except Refusal as exc: unusable("refused", str(exc)) return 2 + if worktrees: + return _export_worktrees(out_dir, cfg, unusable) try: root = workspace_root() source = resolve_source(requested, root) @@ -1855,11 +2263,49 @@ def cmd_export(out_dir: str, requested: str | None) -> int: return 0 +def _export_worktrees(out_dir: str, cfg: dict, unusable) -> int: + """EVERY `inventory:` THE STORE HOLDS - this workstation's and + any other syncing into the same Postgres, which is what the table is for: + the next account, on either workstation, sees every open worktree in one + read. `meta` names each source with the time it was synced.""" + try: + store = open_store(cfg, "reader") + store.open_read() + provs = [p for p in store.all_provenance() + if str(p["source"]).startswith(INVENTORY_PREFIX)] + if not provs: + return unusable("wiped", f"no {INVENTORY_PREFIX} source in the " + f"{cfg['store']} store") + lines = [] + for prov in provs: + for row in store.rows("worktrees", prov["source"], + TABLE_COLUMNS["worktrees"] + ["provenance"]): + lines.append(tree_line(row, row.get("provenance"))) + except Refusal as exc: + unusable("refused", str(exc)) + return 2 + except StoreUnusable as exc: + return unusable(exc.kind, exc.detail) + # THE BYTE ORDER `LC_ALL=C sort` GIVES the source read, so the two are one + # stream wherever the index is current. + lines.sort(key=lambda line: line.encode("utf-8")) + with open(os.path.join(out_dir, "worktrees"), "w", encoding="utf-8", newline="") as handle: + for line in lines: + handle.write(line + "\n") + with open(os.path.join(out_dir, "meta"), "w", encoding="utf-8", newline="") as handle: + handle.write(f"store={cfg['store']}\n") + for prov in sorted(provs, key=lambda p: str(p["source"])): + handle.write(f"inventory\t{prov['source']}\t{prov['indexed_utc']}\t" + f"{prov.get('generation')}\n") + return 0 + + USAGE = f"""\ {PROG} sync [--source ] index what LANDED, behind the source {PROG} reconcile [--source ] [--dry-run] read the source whole; put the index right {PROG} status per source: store, provenance, lag, errors -{PROG} export --out [--source ] the `--index` read flags' reader +{PROG} export --out [--source | --worktrees] + the `--index` read flags' reader The DERIVED INDEX of the lane register (lane-collision-protocol Amendment 14). It is written behind the sources by this command alone, read only by tooling @@ -1878,10 +2324,17 @@ that decides nothing, and never a gate: no lane act reads it or waits on it. status The store, each source's provenance against its tip, the lag in commits, and this workstation's last sync and last error. export Writes the index's image of the register into for - `lanes --index` and `lanes-edit.sh history --index`. Not for people. - - --source `register:`, which must be this workstation's own; - the managed ledger's `ledger:` is T058's to write. + `lanes --index` and `lanes-edit.sh history --index` - or, with + `--worktrees`, every inventory source's rows for `lanes --index + --worktrees`. Not for people. + + --source `register:`, which must be this workstation's own + and is the default; `inventory` (or `inventory:`, + this one's) is THIS workstation's #97 worktree inventory, read + through `lanes-edit.sh worktrees --all` into the `worktrees` + table (opensoft/openRepoTools#161) - every inventory write + nudges `sync --source inventory`. The managed ledger's + `ledger:` is T058's to write. THE STORE: SQLite at ${{XDG_STATE_HOME:-~/.local/state}}/openRepoTools/lanes-index.sqlite, or QA Postgres through `psql` where the configuration names a `url=`. @@ -1915,6 +2368,7 @@ def main(argv: list[str]) -> int: verb, args = argv[0], argv[1:] source = None dry_run = False + worktrees = False out_dir = None i = 0 while i < len(args): @@ -1942,6 +2396,10 @@ def main(argv: list[str]) -> int: dry_run = True i += 1 continue + if arg == "--worktrees" and verb == "export": + worktrees = True + i += 1 + continue note(f"unknown argument '{arg}' for {verb} - see {PROG} --help") return 64 try: @@ -1958,7 +2416,11 @@ def main(argv: list[str]) -> int: if not out_dir: note("export needs --out ") return 64 - return cmd_export(out_dir, source) + if worktrees and source: + note("export --worktrees reads every inventory source the store holds; it " + "takes no --source") + return 64 + return cmd_export(out_dir, source, worktrees) except Refusal as exc: note(f"REFUSED: {exc}") return 2 From 18fa6c0f9aa1a9ff504093d63cc26a3dfa85eee1 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 17:28:17 +0000 Subject: [PATCH 03/12] Nudge the indexer after every inventory write set-lane-state, set-lane-tree, the lifecycle follow after a STARTED, RESUMED, ENDED or RETIRED, and a rename's move of the control root write no commit and push nothing, so the derived index's `worktrees` table would only follow them on a person's reconcile. Each now leaves LANES_INDEX_NUDGE_INV for `cleanup`, which starts the same detached `lanes-index sync` a landed push does, as `sync --source inventory`: stdin closed, never waited, its status never read, skipped where lanes-index is not on PATH or LANES_INDEX=off. Like the register's flag it is assigned at start-up whatever the environment says. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lanes-edit.sh | 45 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/lanes-edit.sh b/lanes-edit.sh index d5ea70bf..6307d262 100755 --- a/lanes-edit.sh +++ b/lanes-edit.sh @@ -136,7 +136,9 @@ # # THE INVENTORY HAS A TABLE TOO (opensoft/openRepoTools#161): `worktrees`, # under the source `inventory:`, copied from `worktrees --all` -# by `lanes-index sync --source inventory`. +# by `lanes-index sync --source inventory` — which every inventory write +# (`set-lane-state`, `set-lane-tree`, the lifecycle follow, a rename's move +# of the control root) nudges exactly as a landed push nudges the register's. # No act reads it: `lane-reconcile`, `lane-start` and `lane-end` read the # sidecars and the disk. With `--index` the stderr line is `read: index # () at inventory: synced `. @@ -1077,9 +1079,12 @@ LOCK="$LANES_DIR/.lanes-edit.lock" # so an exported variable of either name changes nothing. LANES_IDX_DIR="" LANES_INDEX_NUDGE=0 -# AND THE INVENTORY'S (opensoft/openRepoTools#161), for the same reason: the -# `worktrees` table's export directory, set only by `index_open worktrees`. +# AND THE INVENTORY'S OWN TWO (opensoft/openRepoTools#161), for the same reason: +# the `worktrees` table's export directory, set only by `index_open worktrees`, +# and the nudge an inventory write leaves for `cleanup` - set only by a write +# of a lifecycle snapshot or a tree record that this process made. LANES_WT_IDX_DIR="" +LANES_INDEX_NUDGE_INV=0 LOCK_HELD=0 LANES_PATH="${LANES_PATH:-$(git -C "${LANES_DIR:-.}" rev-parse --show-prefix 2>/dev/null || :)${LANES_FILE##*/}}" @@ -1203,9 +1208,11 @@ cleanup() { [ -n "${LANES_WT_IDX_DIR:-}" ] && [ -d "${LANES_WT_IDX_DIR:-}" ] && rm -rf -- "$LANES_WT_IDX_DIR" # AMENDMENT 14(d) — THE NUDGE IS THE LAST THING THIS PROCESS DOES: after the # lock is released at the top of this function, and only where `commit_push` recorded a push - # that landed. Tested here rather than inside the function, because `cleanup` - # also runs for a process that died before `index_nudge` was ever defined. - [ "${LANES_INDEX_NUDGE:-0}" = 1 ] && index_nudge + # that landed - or, opensoft/openRepoTools#161, where an inventory writer + # recorded a snapshot or a tree record it had just replaced. Tested here rather + # than inside the function, because `cleanup` also runs for a process that died + # before `index_nudge` was ever defined. + if [ "${LANES_INDEX_NUDGE:-0}" = 1 ] || [ "${LANES_INDEX_NUDGE_INV:-0}" = 1 ]; then index_nudge; fi return 0 } # AND A SIGNAL ACTUALLY STOPS IT (Amendment 8, ruling (h)). `trap cleanup EXIT @@ -2069,12 +2076,29 @@ EOF # `managed-projection`, `lane-reconcile`) has a path to `LANES_IDX_DIR`: it is # assigned empty at start-up whatever the environment says, and set only by # `index_open`, which only the two read arms call. +# +# THE INVENTORY'S NUDGE IS THE SAME ONE (opensoft/openRepoTools#161). The #97 +# inventory writers — `set-lane-state`, `set-lane-tree`, the lifecycle follow +# `write_event` makes after a `STARTED`/`RESUMED`/`ENDED`/`RETIRED`, and the +# rename that moves a lane's control root — write no commit and push nothing, +# so they leave `LANES_INDEX_NUDGE_INV` instead, and the same detached fork is +# made with `--source inventory`: the indexer reads this workstation's +# sidecars as they now stand, through this file's own `worktrees` read. A write +# that did both (a `STARTED` that landed and moved the snapshot) makes both +# forks; `lanes-index` runs one sync at a time per workstation, and a second +# that finds the lock held leaves the holder a mark naming its source. index_nudge() { - LANES_INDEX_NUDGE=0 + in_reg="${LANES_INDEX_NUDGE:-0}"; in_inv="${LANES_INDEX_NUDGE_INV:-0}" + LANES_INDEX_NUDGE=0; LANES_INDEX_NUDGE_INV=0 [ "${LANES_INDEX:-}" = off ] && return 0 in_bin="$(command -v lanes-index 2>/dev/null || :)" [ -n "$in_bin" ] || return 0 - ( "$in_bin" sync /dev/null 2>&1 3>&- 4>&- 5>&- 6>&- 7>&- 8>&- 9>&- & ) >/dev/null 2>&1 + if [ "$in_reg" = 1 ]; then + ( "$in_bin" sync /dev/null 2>&1 3>&- 4>&- 5>&- 6>&- 7>&- 8>&- 9>&- & ) >/dev/null 2>&1 + fi + if [ "$in_inv" = 1 ]; then + ( "$in_bin" sync --source inventory /dev/null 2>&1 3>&- 4>&- 5>&- 6>&- 7>&- 8>&- 9>&- & ) >/dev/null 2>&1 + fi return 0 } @@ -11955,6 +11979,7 @@ lane_state_follow() { # if [ "$(lc "${lsn_old##*/}")" = "$(lc "$lsn_new")" ]; then lsn_tmp="$lsn_old.rename.$$" if mv -- "$lsn_old" "$lsn_tmp" 2>/dev/null && mv -- "$lsn_tmp" "$lsn_to" 2>/dev/null; then + LANES_INDEX_NUDGE_INV=1 note "the lane lifecycle snapshot and inventory moved with the rename: $lsn_old → $lsn_to" else note "the lane lifecycle snapshot and inventory could NOT be moved from $lsn_old to $lsn_to (the rename itself has landed). Move it by hand; until then \`lanes-edit.sh lane-reconcile $lsn_new\` reads no snapshot for this lane." @@ -11996,6 +12022,7 @@ lane_state_rename() { # return 0 fi if mv -- "$lsn_old" "$lsn_to" 2>/dev/null; then + LANES_INDEX_NUDGE_INV=1 note "the lane lifecycle snapshot and inventory moved with the rename: $lsn_old → $lsn_to" else note "the lane lifecycle snapshot and inventory could NOT be moved from $lsn_old to $lsn_to (the rename itself has landed). Move it by hand; until then \`lanes-edit.sh lane-reconcile $lsn_new\` reads no snapshot for this lane." @@ -15761,6 +15788,7 @@ EOF if lane_state_put "$sls_root" "$lane" "$sls_state" "$sls_gen" "$sls_op" \ "$sls_owner" "$sls_agent" "$sls_prof" "$WS" "$sls_kind"; then release_lock + LANES_INDEX_NUDGE_INV=1 # opensoft/openRepoTools#161: `cleanup` nudges the indexer printf 'state\t%s\ngeneration\t%s\noperation\t%s\n' "$sls_state" "$sls_gen" "$sls_op" else release_lock @@ -15914,6 +15942,7 @@ EOF if lane_tree_put "$slt_root" "$lane" "$slt_path" "$slt_co" "$slt_b" "$slt_h" \ "$slt_u" "$slt_d" "$slt_n" "$slt_w" "$slt_g" "$slt_op"; then release_lock + LANES_INDEX_NUDGE_INV=1 # opensoft/openRepoTools#161: `cleanup` nudges the indexer printf '%s\n' "$(tree_id_for "$slt_path")" else release_lock From ab111fb9b6562f9c255b22fa554b755adf9fa9c4 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 17:28:17 +0000 Subject: [PATCH 04/12] List every open worktree with lanes --worktrees `lanes --worktrees [ | --all]` renders `lanes-edit.sh worktrees`: workstation, lane, owner, lifecycle, writer liveness, branch, dirty and unpushed counts, last seen and path, with a footer saying every field is the inventory's last observation and that the current truth of a lane is `lanes-edit.sh lane-reconcile `, which reads the disk. With --index it reads the derived index's `worktrees` table - every workstation syncing into that store - and stderr says which was read, as for the listing. It takes none of the listing's narrowing flags, and a helper predating #161 is named as one. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lanes | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 1 deletion(-) diff --git a/lanes b/lanes index 0b7ffffc..1e54640c 100755 --- a/lanes +++ b/lanes @@ -10,6 +10,11 @@ # lanes --here only this workstation's lanes # lanes --fetch any of the above, after refreshing from origin # lanes --index any of the above, out of the DERIVED INDEX (Amendment 14) +# lanes --worktrees [|--all] +# every open worktree the #97 inventory records - owner, +# branch, lifecycle, dirty/unpushed, last seen, writer +# (opensoft/openRepoTools#161); with --index, out of the +# index's `worktrees` table, every workstation it holds # # THE DEFAULT IS EVERY LANE, AND THE NARROWING IS BY CHECKOUT (A11 Addendum 4 # ruling 6, ratified "a11 addendum 4 yes" with ruling 6 answered NO, over the @@ -105,6 +110,7 @@ usage() { lanes [--fetch] [--index] [--all] [--here] [--closed] [--repo ] [--dir ] [--prefix ] [--ws ] [--rename ] +lanes --worktrees [ | --all] [--index] [--fetch] Inside a lane checkout: THAT REPOSITORY's lanes, ending with the next free position and the exact `lane-start ` that takes it. Outside one: every @@ -147,6 +153,19 @@ is running is never started a second time. home repository and the checkout it sits in are not the same fact. --ws another workstation's lanes. + --worktrees [ | --all] + every open worktree of one lane, or of every lane: the #97 + inventory each lane's handoff recorded beside it — owner, + branch, lifecycle, dirty and unpushed counts, when it was + last seen and whether the lane's writer is live + (opensoft/openRepoTools#161). Every field is that LAST + OBSERVATION and nothing is recomputed: the current truth of a + lane is `lanes-edit.sh lane-reconcile `, which reads the + disk. Without --index it reads this workstation's sidecars; + WITH it, the derived index's `worktrees` table — every + workstation syncing into that store, so Raven sees Eagle's + leftovers — and stderr says which, as for the listing. It + takes none of the listing's narrowing flags. --rename REFUSED, and it names the word that does it: `lane-rename `. This command writes nothing @@ -191,11 +210,22 @@ all_given=0 # 1 once `--all` has been seen, wherever in the line it was # NEXT FREE POSITION footer. here_repo="" args=() +# `--worktrees` (opensoft/openRepoTools#161) TAKES ONE POSITIONAL, THE LANE, and +# the listing takes none — so whether this line asks for worktrees is known +# before the loop below meets a positional, wherever the flag stands in it. +wt_mode=0 +wt_lane="" +# `${1+"$@"}` AND NOT A BARE `"$@"`: bash 3.2 under `set -u` calls the empty +# `"$@"` of a bare `lanes` an unbound variable and exits (tests/run.sh says why). +for wt_a in ${1+"$@"}; do + case "$wt_a" in --worktrees) wt_mode=1 ;; esac +done while [ $# -gt 0 ]; do case "$1" in -h | --help) usage; exit 0 ;; --fetch) do_fetch=1; shift ;; --index) idx_args=(--index); shift ;; + --worktrees) shift ;; # `--all` CLEARS THE LABEL AS WELL AS THE FILTERS. The helper's `--all` drops # `--repo`, `--dir` and `--prefix`, so the ROWS were right — but `here_repo` # is this wrapper's own, and `lanes --all --prefix repoA11` (either order) @@ -243,10 +273,101 @@ while [ $# -gt 0 ]; do (lane-collision-protocol Amendment 16)." 64 ;; --) shift ;; -*) die "unknown option '$1' — see $prog --help" 64 ;; - *) die "lanes takes no positional arguments; did you mean \`lane $1\`? — see $prog --help" 64 ;; + *) + if [ "$wt_mode" = 1 ] && [ -z "$wt_lane" ]; then + wt_lane="$1"; shift + elif [ "$wt_mode" = 1 ]; then + die "--worktrees takes ONE lane, or --all — see $prog --help" 64 + else + die "lanes takes no positional arguments; did you mean \`lane $1\`? — see $prog --help" 64 + fi ;; esac done +# ------------------------------------------------ `--worktrees` (#161) +# +# EVERY OPEN WORKTREE, rendered from `lanes-edit.sh worktrees`' rows — the #97 +# inventory as each lane's handoff last recorded it, read from this +# workstation's sidecars, or with `--index` from the derived index's +# `worktrees` table, which holds every workstation that syncs into it. The two +# reads print the same bytes on stdout wherever the index is current; stderr +# says which was read. It recomputes nothing and it decides nothing: the act +# that reads the disk is `lanes-edit.sh lane-reconcile `, and the footer +# says so. +US="$(printf '\037')" +if [ "$wt_mode" = 1 ]; then + for wt_a in ${args[@]+"${args[@]}"}; do + [ "$wt_a" = --all ] && continue + die "--worktrees takes ONE lane or --all, and none of the listing's narrowing flags ('$wt_a' was given) — see $prog --help" 64 + done + [ "$closed" = 0 ] || die "--worktrees takes ONE lane or --all, not --closed: a closed lane's worktrees are listed with every other's — see $prog --help" 64 + [ "$all_given" = 0 ] || [ -z "$wt_lane" ] || die "--worktrees takes ONE lane OR --all, not both — see $prog --help" 64 + wt_args=() + if [ -n "$wt_lane" ]; then wt_args+=("$wt_lane"); else wt_args+=(--all); fi + [ "$do_fetch" = 1 ] && wt_args+=(--fetch) + rc=0 + wt_err="" + wt_err_file="$(mktemp "${TMPDIR:-/tmp}/lanes-worktrees.XXXXXX" 2>/dev/null || printf '')" + if [ -n "$wt_err_file" ]; then + out="$("$LANES_EDIT" worktrees ${idx_args[@]+"${idx_args[@]}"} "${wt_args[@]}" 2>"$wt_err_file")" || rc=$? + wt_err="$(cat -- "$wt_err_file" 2>/dev/null || printf '')" + # `-e` AND NO `--` AFTER THE PATTERN: BSD's `grep` stops at its first + # operand, so a `--` there is a FILE it cannot open on macOS. + command grep -v -e 'not fetching; reading origin/' "$wt_err_file" >&2 || : + rm -f -- "$wt_err_file" + else + out="$("$LANES_EDIT" worktrees ${idx_args[@]+"${idx_args[@]}"} "${wt_args[@]}")" || rc=$? + fi + case "$rc" in + 0) : ;; + 8) + say "no worktree is inventoried${wt_lane:+ for lane $wt_lane}." + say " each handoff records its lane's trees (lanes-edit.sh set-lane-tree); the disk itself is read by" + say " lanes-edit.sh lane-reconcile " + exit 8 ;; + 64) die "bad arguments for the worktrees read — see $prog --help" 64 ;; + 2) + case "$wt_err" in + *"unknown subcommand 'worktrees'"*) + die "the \`lanes-edit.sh\` on this workstation has no \`worktrees\` read, so it + predates opensoft/openRepoTools#161. Nothing is wrong with the inventory — + take the install: openRepoTools --install" 2 ;; + esac + exit 2 ;; + *) die "$LANES_EDIT worktrees failed (exit $rc). That is NOT 'no worktree is inventoried', and + this command will not print an empty listing on an answer it did not get + (Amendment 7(d)). Run it by hand to see what it says." 1 ;; + esac + printf '%-11s %-26s %-16s %-13s %-7s %-34s %5s %8s %-20s %s\n' \ + WORKSTATION LANE OWNER LIFECYCLE WRITER BRANCH DIRTY UNPUSHED "LAST SEEN" PATH + n=0; nl=0; nw=0; wt_unread=0; wt_lanes=" "; wt_wss=" " + while IFS="$US" read -r w_kind w_ws w_lane w_owner w_path w_branch w_base w_life w_gen w_op w_live w_seen w_last w_dirty w_unp w_prov; do + case "${w_kind:-}" in + TREE) + case "${w_live:-}" in 1) w_wr=live ;; 0) w_wr=none ;; *) w_wr=unknown ;; esac + printf '%-11s %-26s %-16s %-13s %-7s %-34s %5s %8s %-20s %s\n' \ + "$w_ws" "$w_lane" "${w_owner:-unknown}" "${w_life:-INDETERMINATE}" "$w_wr" \ + "${w_branch:-unknown}" "${w_dirty:-?}" "${w_unp:-?}" "${w_seen:-never}" "$w_path" + n=$((n + 1)) + case "$wt_lanes" in *" $w_ws/$w_lane "*) : ;; *) wt_lanes="$wt_lanes$w_ws/$w_lane "; nl=$((nl + 1)) ;; esac + case "$wt_wss" in *" $w_ws "*) : ;; *) wt_wss="$wt_wss$w_ws "; nw=$((nw + 1)) ;; esac ;; + UNREAD) + # `UNREAD `: the third field is the reason. + printf '%-11s %-26s inventory NOT READ: %s\n' "$w_ws" "$w_lane" "$w_owner" + wt_unread=$((wt_unread + 1)) ;; + esac + done < (it reads the disk; this listing decides nothing)" + exit 0 +fi + # `--all` WINS WHEREVER IT APPEARS IN THE LINE, so the label is cleared after the # loop and not inside the arm: `lanes --all --prefix repoA11` and # `lanes --prefix repoA11 --all` are the same command and must print the same From e385830c0a04908e90090f58e00ed07dc2c14080 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 17:28:38 +0000 Subject: [PATCH 05/12] Document the worktrees table under the derived index README-lanes.md's "The derived index" gains the `worktrees` table: what it is for (the next account's first query, across workstations), each column and where it comes from, how rows follow the inventory and why the provenance moves forward per lane, the nudge, the `lanes --worktrees` read with and without --index, and what never reads it - no act, not `lane-reconcile` (there is no `lane-reconcile --index`), `lane-start`, `lane-end`, `lane-handoff` or the worktree sweep (#162). The inventory section points at the listing, and the reserved-table note becomes the inventory's. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- docs/README-lanes.md | 88 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 4 deletions(-) diff --git a/docs/README-lanes.md b/docs/README-lanes.md index 62852685..a7873340 100644 --- a/docs/README-lanes.md +++ b/docs/README-lanes.md @@ -2959,6 +2959,10 @@ lanes-edit.sh lane-tree-now # — what git says about one tree NOW ``` +Every lane's records on this workstation, one row per tree, are `lanes +--worktrees`; the derived index keeps them as its `worktrees` table, which no +act reads (*The derived index*, below). + The full `head` and the `upstream` are why this is not the prose section one more time: `%h` is an abbreviation that lengthens as a repository grows, and `0 unpushed` cannot be told from *this branch tracks nothing at all* without the @@ -3283,15 +3287,91 @@ One versioned schema (`schema_version`); every row carries its `provenance` **Created empty and written by nothing here**: `swap_states` and `job_summaries` — the managed ledger's half, `ledger:`, which the -001 feature's T058 writes (act 4) — and `worktrees`, **reserved for -opensoft/openRepoTools#161** (each lane's #97 inventory under -`inventory:`). Not indexed at all: transcript and handoff bodies, +001 feature's T058 writes (act 4). `worktrees` is the inventory's, below — the +same schema 1, whose table was created with #161's columns before a row was +written into it. Not indexed at all: transcript and handoff bodies, credentials, the ledger's claims, fences, releases and exit witnesses. +### The `worktrees` table — every lane's #97 inventory (opensoft/openRepoTools#161) + +**What it is for: the next account's first query.** A session that resumes a +lane — or that sits down at Raven after a day on Eagle — wants every open +worktree before it decides anything: whose it is, which branch, whether the +lane stopped `RUNNING` or `SWAPPED`, what was dirty or unpushed, when it was +last seen and whether a writer is still live. Each lane's handoff already +records that beside the lane (*The worktree inventory*, above); this table is +the same records in one place, under the source `inventory:` — one +per workstation, because the sidecars live on that workstation's disk and +nowhere else — so a store two workstations sync into answers for both. + +```sh +lanes --worktrees [ | --all] # this workstation's sidecars +lanes --index --worktrees [ | --all] # the table: every workstation it holds +lanes-index sync --source inventory # what every inventory write nudges +lanes-index reconcile --source inventory [--dry-run] +``` + +| column | from | +|---|---| +| `workstation`, `lane` | the workstation's name (`lanes-edit.sh workstation`) and the lane's canonical one | +| `owner` | `legacy`, `managed ` or `unknown` — the read `managed-projection` makes, never a parser of the indexer's | +| `path`, `branch`, `dirty_count`, `unpushed_count`, `last_seen_utc` | the tree's sidecar: the last observation, `detached ` for a detached head, an `unknown` count as NULL. A sidecar that could not be read is `unreadable sidecar`, one of another schema `unknown schema `, each with the record's id where its path would be | +| `base`, `last_commit` | two git reads of the RECORDED head, in the tree or, once it is gone, its recorded checkout: its merge-base with `origin/main` (else `main`, else `origin/HEAD`), and ` ` | +| `lifecycle`, `generation`, `operation` | the lane's snapshot (`lane-state`): `RUNNING`, `SWAPPING`, `SWAPPED` or `CLOSED` for a **legacy** lane; `INDETERMINATE` for a managed or unknown one, whose lifecycle this tooling does not pronounce on, and for a snapshot that is missing, unreadable or of another schema | +| `writer_live` | the listing's own LIVE — a session record on this workstation naming one of the row's ids — and NULL where those records could not be read | +| `pr_number`, `pr_state` | `gh` at sync, best effort, the newest pull request whose head is the branch; NULL wherever it was not asked (`LANES_NO_GITHUB=1`) or did not answer. A MERGED or CLOSED answer is not asked again while the branch and its last commit stand | +| `provenance`, `indexed_utc` | the generation the TREE'S OWN record was filed under — behind `generation` exactly when the observation predates the lane's current transition — and the sync | + +**Read through the helper, never around it.** The indexer reads `lanes-edit.sh +worktrees --all`, which lists each lane's sidecars through `lane-trees`' own +reader, its snapshot through `lane-state`'s and its owner through +`managed-projection`'s, and walks no directory for a tree no sidecar names — +that is `lane-reconcile`'s act. Nothing is recomputed: `dirty` and `unpushed` +are what the handoff saw, and `LAST SEEN` says when. + +**Rows follow the inventory.** Keyed by (source, `:`) and versioned +by a digest of the row and its provenance, so a replay writes nothing; a tree +the inventory no longer names — removed at landing, or by a sweep's disposition +— leaves in the same sync. The provenance moves only forward **per lane**: a +`sync` that reads a lane's lifecycle generation below the one its rows were +written at (a snapshot moved aside and begun again) writes nothing for that lane +and says so, and `reconcile --source inventory` is the act that takes the lane +as it now stands. A lane whose `trees` directory is there and cannot be listed +keeps its rows — it is not a lane with no worktrees. The source's own row +carries a fingerprint of the whole table as its compare-and-swap token and a +sync count that only goes up. A container with no `LANES_WORKSTATION` syncs +nothing (exit 2): a source named for a container's own id is one no restart +finds again. + +**The nudge.** Every inventory write — `set-lane-state`, `set-lane-tree`, the +lifecycle follow after a `STARTED`, `RESUMED`, `ENDED` or `RETIRED`, and a +rename's move of the control root — starts the same detached `lanes-index sync` +a landed push does, as `sync --source inventory`, skipped where `lanes-index` is +not on `PATH` or `LANES_INDEX=off`. One sync at a time per workstation: a nudge +that finds the lock held leaves a mark NAMING ITS SOURCE, and the holder syncs +every source a mark names before it lets go. + +**The read.** `lanes --worktrees` reads this workstation's sidecars; with +`--index` it reads the table — every `inventory:*` source the store holds, so +Raven sees Eagle's leftovers — and stderr says `read: index () at +inventory: synced `, or `read: sources (index )` and the sidecars +answer. Where the table is current the two print the same bytes. Every row is a +last observation: the current truth of a lane is `lanes-edit.sh lane-reconcile +`, which reads the disk, and the footer says so. + +**What never reads it.** No act: not `lane-reconcile` — there is no +`lane-reconcile --index`, by Amendment 14(b) — not `lane-start`, `lane-end`, +`lane-handoff` or the worktree sweep (opensoft/openRepoTools#162). Each of them +reads the sidecars and the disk, exactly as before this table existed; an +inventory index that is missing, stale, unreachable, wiped or wrong changes no +answer of theirs. + ### The tests' promise `tests/test_lanes_index.py`, and a section of `tests/test_lane_helpers.sh` for -the acts only that suite drives, hold the index to Amendment 14's eight: +the acts only that suite drives, hold the index — the inventory's table +included, whose writers and `lane-reconcile` are among the acts — to Amendment +14's eight: **offline no-read** (every act byte-identical against an unreachable store), **poisoned index** (wrong rows change no act's answer), **wiped-index reconcile** (the rebuild equals a synced index row for row), **replay**, From 7550f54d712e5b332bf3b7d1ab0eeafe06f0fd2d Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:18:33 +0000 Subject: [PATCH 06/12] Hold the inventory source to Amendment 14's tests tests/test_lanes_index.py gains an inventory fixture written by the real writers over real worktrees - a RUNNING lane with one clean tree and one dirty, unpushed one, a SWAPPING lane whose session is live, and the leftovers of a valid and a malformed managed marker - and holds the `worktrees` table to the promise #160 made for the register: the rows mirror the sidecars (owner, lifecycle, liveness, counts, base, last commit, provenance); a replay writes nothing and one write moves one row; a tree the inventory no longer names leaves; a wiped table reconciles row for row and a hand-edited row is put right; the provenance only moves forward per lane; the owner column is the seam's answer; `lanes --worktrees` shows the running and the dirty tree; the index read is the source read byte for byte; each of six failures falls back to the sidecars and says so; the inventory writers and `lane-reconcile` are byte-identical against an unreachable store and over a poisoned table; the writers nudge `sync --source inventory` and the reads never do; `gh` is asked best effort; a rerun mark names its source; and another workstation's inventory, or a nameless container's, is refused. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- tests/test_lanes_index.py | 565 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 565 insertions(+) diff --git a/tests/test_lanes_index.py b/tests/test_lanes_index.py index bc323366..584c517f 100644 --- a/tests/test_lanes_index.py +++ b/tests/test_lanes_index.py @@ -1492,3 +1492,568 @@ def test_export_never_creates_a_store_and_the_store_is_private(estate): assert estate.index("sync", "--source", "ledger:Eagle").returncode == 2 assert estate.index("sync", "--source", "register:someone/else").returncode == 2 assert estate.index("bogus").returncode == 64 + + +# =================================== the inventory source (opensoft/openRepoTools#161) +# +# THIS WORKSTATION'S #97 WORKTREE INVENTORY, AS THE INDEX'S SECOND SOURCE. Each +# lane's sidecars are written here by the REAL writers (`set-lane-state`, +# `set-lane-tree`) over REAL worktrees of a repository whose origin is a bare +# repository in `tmp_path`; the two managed lanes' sidecars are written by hand, +# because every legacy writer refuses a managed lane — which is exactly how a +# managed lane comes to have leftovers from before its enrolment. + +WT_COLS = ["workstation", "lane", "owner", "path", "branch", "base", "lifecycle", "generation", + "operation", "writer_live", "last_seen_utc", "last_commit", "dirty_count", + "unpushed_count", "pr_number", "pr_state"] + + +#: `tree_id_for`'s shape: a `cksum` of the path, then the path folded. +TREE_ID = re.compile(r"\bc[0-9]+-[A-Za-z0-9._-]+") + + +class Inventory: + """An Estate with an inventory: repoA-1 RUNNING with one clean tree and one + dirty, unpushed one; repoA-2 SWAPPING with one tree and a LIVE session; + repoM-1 (a valid marker) and repoM-2 (a malformed one) each with a sidecar + left over from before enrolment.""" + + def __init__(self, root: Path): + self.e = Estate(root) + self.root = root + self.lsr = root / "lane-state" + self.e.env["LANES_LANE_STATE_ROOT"] = str(self.lsr) + self.repo = root / "repos" / "repoA" + self.origin = root / "repos" / "repoA.git" + self.trees = self.repo / ".claude" / "worktrees" + g = self._g + subprocess.run(["git", "init", "-q", "--bare", "-b", "main", str(self.origin)], + check=True, env=self.e.env) + subprocess.run(["git", "clone", "-q", str(self.origin), str(self.repo)], + check=True, env=self.e.env, capture_output=True) + (self.repo / "a.txt").write_text("seed\n") + g(self.repo, "add", "-A") + g(self.repo, "commit", "-q", "-m", "the lane's first commit") + g(self.repo, "push", "-q", "origin", "main") + self.trees.mkdir(parents=True) + self.run_tree = self.trees / "w-run" + g(self.repo, "worktree", "add", "-q", "-b", "feat/run", str(self.run_tree)) + g(self.run_tree, "push", "-q", "-u", "origin", "feat/run") + self.dirty_tree = self.trees / "w-dirty" + g(self.repo, "worktree", "add", "-q", "-b", "feat/dirty", str(self.dirty_tree)) + g(self.dirty_tree, "push", "-q", "-u", "origin", "feat/dirty") + (self.dirty_tree / "b.txt").write_text("committed, not pushed\n") + g(self.dirty_tree, "add", "b.txt") + g(self.dirty_tree, "commit", "-q", "-m", "work not yet pushed") + (self.dirty_tree / "c.txt").write_text("not even committed\n") + self.a2_tree = self.trees / "w-a2" + g(self.repo, "worktree", "add", "-q", "-b", "feat/a2", str(self.a2_tree)) + self.ok("set-lane-state", "repoA-1", "RUNNING", "--operation", "op-test-a1") + self.ok("set-lane-tree", "repoA-1", str(self.run_tree), "--checkout", str(self.repo)) + self.ok("set-lane-tree", "repoA-1", str(self.dirty_tree), "--checkout", str(self.repo)) + self.ok("set-lane-state", "repoA-2", "RUNNING", "--operation", "op-test-a2") + self.ok("set-lane-state", "repoA-2", "SWAPPING", "--operation", "op-test-a2s") + self.ok("set-lane-tree", "repoA-2", str(self.a2_tree), "--checkout", str(self.repo), + "--generation", "2", "--operation", "op-test-a2s") + for lane, head in (("repoM-1", "1" * 40), ("repoM-2", "2" * 40)): + trees = self.lsr / lane / "trees" + trees.mkdir(parents=True) + (trees / f"c0-{lane}.yaml").write_text("\n".join([ + "schema: 1", f"lane: {lane}", f"tree: c0-{lane}", + f"path: /nowhere/{lane}/w", "checkout: unknown", f"branch: feat/{lane}", + f"head: {head}", "upstream: none", "dirty: 0", "unpushed: 0", + "writer: none", "generation: 0", "operation: none", + "observed: 2026-10-04T00:00:00Z", ""])) + # repoA-2's session is LIVE on this workstation: the listing's own read. + self.sleeper = subprocess.Popen(["sleep", "600"]) + sessions = self.e.home / ".claude" / "sessions" + sessions.mkdir(parents=True, exist_ok=True) + (sessions / "live.json").write_text( + '{"pid":%d,"sessionId":"%s","kind":"interactive","status":"busy"}\n' + % (self.sleeper.pid, UUID_A2)) + + def _g(self, cwd: Path, *args: str) -> str: + proc = subprocess.run(["git", *args], cwd=str(cwd), capture_output=True, text=True, + env=self.e.env, check=False) + assert proc.returncode == 0, f"git {' '.join(args)}: {proc.stderr}" + return proc.stdout.strip() + + def ok(self, *args: str) -> subprocess.CompletedProcess: + res = self.e.edit(*args) + assert res.returncode == 0, (args, res.stdout, res.stderr) + return res + + def sync(self, *extra: str, env: dict | None = None) -> subprocess.CompletedProcess: + return self.e.index("sync", "--source", "inventory", *extra, env=env) + + def rows(self) -> dict: + conn = sqlite3.connect(str(self.e.sqlite)) + conn.row_factory = sqlite3.Row + out = {r["record_id"]: dict(r) for r in conn.execute( + "SELECT * FROM worktrees WHERE source = 'inventory:Eagle'")} + conn.close() + return out + + def norm(self, text: str) -> str: + """An answer with what is the SANDBOX's taken out: its paths (as given + and resolved), a tree id - `c-`, whose tail + alone survives a path longer than 180 bytes - and the live session's + pid.""" + text = TREE_ID.sub("", text) + text = text.replace(os.path.realpath(self.root), str(self.root)) + text = re.sub(rf"\b{self.sleeper.pid}\b", "", text) + return normal(text, self.root) + + def close(self) -> None: + self.sleeper.kill() + self.sleeper.wait() + + +@pytest.fixture +def inv(tmp_path): + made = Inventory(tmp_path / "i") + yield made + made.close() + + +def test_the_inventory_source_mirrors_every_lanes_sidecars(inv): + """THE TABLE IS THE INVENTORY (#161): one row per tree each lane's sidecars + name, every column from the helper that wrote it - the owner from + `managed-projection`'s read, the lifecycle from the snapshot, the liveness + from the listing's own session read, the counts and the stamp from the + observation, the base and the last commit read of the RECORDED head - and + nothing of the register's touched.""" + res = inv.sync() + assert res.returncode == 0, res.stderr + assert "inventory:Eagle (sqlite) - 5 upserted, 0 removed" in res.stdout, res.stdout + rows = inv.rows() + run = rows[f"repoA-1:{inv.run_tree}"] + dirty = rows[f"repoA-1:{inv.dirty_tree}"] + a2 = rows[f"repoA-2:{inv.a2_tree}"] + main = inv._g(inv.repo, "rev-parse", "origin/main") + assert (run["workstation"], run["owner"], run["branch"], run["lifecycle"]) == \ + ("Eagle", "legacy", "feat/run", "RUNNING") + assert (run["dirty_count"], run["unpushed_count"], run["writer_live"]) == (0, 0, 0) + assert run["base"] == main + assert run["last_commit"] == f"{main} the lane's first commit" + assert (run["generation"], run["operation"], run["provenance"]) == (1, "op-test-a1", "0") + assert UTC.fullmatch(run["last_seen_utc"]), run["last_seen_utc"] + head = inv._g(inv.dirty_tree, "rev-parse", "HEAD") + assert (dirty["dirty_count"], dirty["unpushed_count"]) == (1, 1) + assert dirty["last_commit"] == f"{head} work not yet pushed" + assert dirty["base"] == main + assert (a2["lifecycle"], a2["generation"], a2["writer_live"], a2["provenance"]) == \ + ("SWAPPING", 2, 1, "2") + assert (a2["pr_number"], a2["pr_state"]) == (None, None) + m1 = rows["repoM-1:/nowhere/repoM-1/w"] + m2 = rows["repoM-2:/nowhere/repoM-2/w"] + assert (m1["owner"], m1["lifecycle"]) == ("managed ledger-1", "INDETERMINATE") + assert (m2["owner"], m2["lifecycle"]) == ("unknown", "INDETERMINATE") + assert m1["last_commit"] == "1" * 40 and m1["base"] is None + conn = sqlite3.connect(str(inv.e.sqlite)) + sources = [r[0] for r in conn.execute("SELECT source FROM provenance")] + assert sources == ["inventory:Eagle"], sources + for table in REGISTER_TABLES_FOR_TESTS: + assert conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] == 0, table + prov = conn.execute("SELECT commit_sha, generation FROM provenance").fetchone() + assert re.fullmatch(r"[0-9a-f]{64}", prov[0]) and prov[1] == 1 + + +REGISTER_TABLES_FOR_TESTS = ("source_files", "lanes", "log_lines", "register_lines", + "lane_aliases", "transcript_pointers") + + +def test_an_inventory_replay_changes_nothing_and_one_write_moves_one_row(inv): + """REPLAY (14(d)): a second sync of an unchanged inventory writes not one + byte - `indexed_utc` included - and one `set-lane-tree` afterwards upserts + the one row it moved.""" + assert inv.sync().returncode == 0 + before = dump(inv.e.sqlite, with_utc=True) + time.sleep(1.1) + again = inv.sync() + assert again.returncode == 0, again.stderr + assert "nothing to write" in again.stdout, again.stdout + assert dump(inv.e.sqlite, with_utc=True) == before + (inv.run_tree / "new.txt").write_text("a fresh change\n") + inv.ok("set-lane-tree", "repoA-1", str(inv.run_tree), "--checkout", str(inv.repo)) + third = inv.sync() + assert "1 upserted, 0 removed" in third.stdout, third.stdout + assert inv.rows()[f"repoA-1:{inv.run_tree}"]["dirty_count"] == 1 + + +def test_a_tree_the_inventory_no_longer_names_leaves_the_table(inv): + """ROWS LEAVE WITH THEIR RECORD (#161): a tree removed from the inventory - + at landing, or by a sweep's disposition - leaves the table in the next + sync, and nothing else moves.""" + assert inv.sync().returncode == 0 + sidecar = [f for f in (inv.lsr / "repoA-1" / "trees").iterdir() + if "w-dirty" in f.name] + assert len(sidecar) == 1 + sidecar[0].unlink() + res = inv.sync() + assert res.returncode == 0, res.stderr + assert "0 upserted, 1 removed" in res.stdout, res.stdout + assert f"repoA-1:{inv.dirty_tree}" not in inv.rows() + assert f"repoA-1:{inv.run_tree}" in inv.rows() + + +def test_a_wiped_inventory_reconciles_row_for_row(inv): + """WIPED-INDEX RECONCILE (14(d)): the rebuild equals a synced table, row for + row; and a row changed by hand in the store is found by what it SAYS and + put right.""" + assert inv.sync().returncode == 0 + synced = dump(inv.e.sqlite) + inv.e.sqlite.unlink() + dry = inv.e.index("reconcile", "--source", "inventory", "--dry-run") + assert dry.returncode == 0, dry.stderr + assert not inv.e.sqlite.exists(), "a dry run created the store" + res = inv.e.index("reconcile", "--source", "inventory") + assert res.returncode == 0, res.stderr + assert re.search(r"worktrees\s+0 kept\s+5 upserted", res.stdout), res.stdout + assert dump(inv.e.sqlite) == synced + conn = sqlite3.connect(str(inv.e.sqlite)) + conn.execute("UPDATE worktrees SET dirty_count = 0, lifecycle = 'SWAPPED' " + "WHERE path = ?", (str(inv.dirty_tree),)) + conn.commit() + conn.close() + assert "nothing to write" in inv.sync().stdout + fixed = inv.e.index("reconcile", "--source", "inventory") + assert re.search(r"worktrees\s+4 kept\s+1 upserted", fixed.stdout), fixed.stdout + righted = dump(inv.e.sqlite) + assert righted["worktrees"] == synced["worktrees"] + # the same table, so the same fingerprint - and a sync count that went UP + assert [r[:2] for r in righted["provenance"]] == [r[:2] for r in synced["provenance"]] + assert righted["provenance"][0][2] == synced["provenance"][0][2] + 1 + + +def test_an_inventorys_provenance_only_moves_forward_per_lane(inv): + """MONOTONIC PROVENANCE (14(d)), PER LANE: a lane whose lifecycle snapshot + reads BELOW the generation its rows were written at - one moved aside and + begun again - is written nothing by `sync`, which says why; the other + lanes sync; and `reconcile` takes the lane as it now stands.""" + assert inv.sync().returncode == 0 + snap = inv.lsr / "repoA-2" / "lane-state.yaml" + snap.rename(snap.with_suffix(".aside")) + inv.ok("set-lane-state", "repoA-2", "RUNNING", "--operation", "op-test-again") + (inv.run_tree / "moved.txt").write_text("repoA-1 moves on\n") + inv.ok("set-lane-tree", "repoA-1", str(inv.run_tree), "--checkout", str(inv.repo)) + res = inv.sync() + assert res.returncode == 0, res.stderr + assert "lane repoA-2: its rows are kept as they are" in res.stdout, res.stdout + assert "only moves forward" in res.stdout + assert "1 upserted, 0 removed" in res.stdout, res.stdout + rows = inv.rows() + assert rows[f"repoA-2:{inv.a2_tree}"]["generation"] == 2 + assert rows[f"repoA-2:{inv.a2_tree}"]["lifecycle"] == "SWAPPING" + assert rows[f"repoA-1:{inv.run_tree}"]["dirty_count"] == 1 + rec = inv.e.index("reconcile", "--source", "inventory") + assert rec.returncode == 0, rec.stderr + assert "generation went back from 2 to 1" in rec.stdout, rec.stdout + rows = inv.rows() + assert (rows[f"repoA-2:{inv.a2_tree}"]["generation"], + rows[f"repoA-2:{inv.a2_tree}"]["lifecycle"]) == (1, "RUNNING") + + +def test_the_inventory_owner_column_is_the_seams_answer(inv): + """THE SEAM (14(e), ruling 2026-10-04): a valid marker's leftovers index as + `managed `, a malformed one's as `unknown` - never `legacy` - and a + plain row's as `legacy`; and only a legacy lane's lifecycle is pronounced, + the others' being `INDETERMINATE`, as `lane-reconcile` pronounces nothing + on them either. The indexer carries no parser of the marker.""" + assert inv.sync().returncode == 0 + owners = {(r["lane"], r["owner"], r["lifecycle"]) for r in inv.rows().values()} + assert ("repoM-1", "managed ledger-1", "INDETERMINATE") in owners + assert ("repoM-2", "unknown", "INDETERMINATE") in owners + assert ("repoA-1", "legacy", "RUNNING") in owners + assert ("repoA-2", "legacy", "SWAPPING") in owners + rec = inv.e.edit("lane-reconcile", "repoM-1") + assert "managed-owned" in rec.stdout + source = INDEXER.read_text(encoding="utf-8") + for vocabulary in ("mode=managed", "managed-owner", "bound-lane"): + assert vocabulary not in source + + +def _wt(inv: Inventory, *args: str, env: dict | None = None) -> subprocess.CompletedProcess: + return inv.e.run("bash", LANES, "--worktrees", *args, env=env) + + +def test_lanes_worktrees_shows_one_running_tree_and_one_dirty_one(inv): + """`lanes --worktrees ` (#161): every open worktree of the lane with + its owner, branch, lifecycle, dirty and unpushed counts, when it was last + seen and whether the writer is live - and a footer that sends a person to + the act that reads the disk.""" + res = _wt(inv, "repoA-1") + assert res.returncode == 0, res.stderr + lines = res.stdout.splitlines() + assert lines[0].split()[:6] == ["WORKSTATION", "LANE", "OWNER", "LIFECYCLE", "WRITER", "BRANCH"] + run = next(l for l in lines if l.endswith(str(inv.run_tree))) + dirty = next(l for l in lines if l.endswith(str(inv.dirty_tree))) + assert run.split()[:8] == ["Eagle", "repoA-1", "legacy", "RUNNING", "none", "feat/run", "0", "0"] + assert dirty.split()[:8] == ["Eagle", "repoA-1", "legacy", "RUNNING", "none", "feat/dirty", "1", "1"] + assert UTC.search(run) + assert "2 worktree(s) in 1 lane(s) on 1 workstation(s)" in res.stdout + assert "lanes-edit.sh lane-reconcile " in res.stdout + every = _wt(inv) + assert every.returncode == 0 + assert "5 worktree(s) in 4 lane(s)" in every.stdout + live = next(l for l in every.stdout.splitlines() if l.endswith(str(inv.a2_tree))) + assert live.split()[3:5] == ["SWAPPING", "live"], live + none = _wt(inv, "repoB-1") + assert none.returncode == 8 and "no worktree is inventoried for lane repoB-1" in none.stdout + for bad in (("repoA-1", "--all"), ("repoA-1", "repoA-2"), ("--here",), ("--closed",)): + refused = _wt(inv, *bad) + assert refused.returncode == 64, (bad, refused.stderr) + + +def test_the_index_read_of_the_worktrees_is_the_source_read(inv): + """`lanes --index --worktrees` ANSWERS WHAT THE SIDECARS ANSWER where the + table is current, byte for byte, and says so on stderr; a write the index + has not followed is the index's lag, said by the time it was synced.""" + assert inv.sync().returncode == 0 + for args in ((), ("repoA-1",), ("--all",)): + src = _wt(inv, *args) + idx = _wt(inv, *args, "--index") + assert (idx.returncode, idx.stdout) == (src.returncode, src.stdout), args + assert re.search(r"read: index \(sqlite\) at inventory:Eagle synced " + UTC.pattern, + idx.stderr), idx.stderr + assert "read: " not in src.stderr + (inv.run_tree / "unseen.txt").write_text("the index has not followed this\n") + inv.ok("set-lane-tree", "repoA-1", str(inv.run_tree), "--checkout", str(inv.repo)) + src = _wt(inv, "repoA-1") + idx = _wt(inv, "repoA-1", "--index") + assert src.stdout != idx.stdout + + +@pytest.mark.parametrize("kind,setup", _fallback_setups(), ids=[k for k, _ in _fallback_setups()]) +def test_each_worktrees_index_failure_falls_back_to_the_sidecars(inv, kind, setup): + """FALLBACK (14(b)) for the table too: missing, unreachable, wiped (no + inventory source), of an unknown schema, refused, or switched off - each + says `read: sources (index )` and returns the sidecars' own answer.""" + env = setup(inv.e) + if kind == "wiped": + assert inv.sync().returncode == 0 + conn = sqlite3.connect(str(inv.e.sqlite)) + conn.execute("DELETE FROM provenance") + conn.commit() + conn.close() + for args in ((), ("repoA-1",)): + src = _wt(inv, *args, env=env) + idx = _wt(inv, *args, "--index", env=env) + assert (idx.returncode, idx.stdout) == (src.returncode, src.stdout), (kind, args) + assert f"read: sources (index {kind}" in idx.stderr, (kind, idx.stderr) + + +#: THE INVENTORY'S ACTS AND THE READS THEY MAKE (#161): its two writers, which +#: nudge the indexer, and the reads an act makes of the inventory for itself. +INVENTORY_ACTS = [ + ("set-lane-tree", "repoA-1", "", "--checkout", ""), + ("set-lane-state", "repoA-1", "SWAPPING", "--expect", "RUNNING", "--operation", "op-acts"), + ("lane-trees", "repoA-1"), + ("lane-state", "repoA-2"), + ("lane-reconcile", "repoA-1"), + ("lane-reconcile", "repoA-2"), + ("lane-reconcile", "repoM-1"), + ("lane-tree-now", ""), +] + + +def _inv_act(inv: Inventory, act: tuple, extra: dict) -> tuple: + argv = [a.replace("", str(inv.run_tree)).replace("", str(inv.repo)) + .replace("", str(inv.dirty_tree)) for a in act] + res = inv.e.edit(*argv, env=extra) + # A LANE'S TREES ARE LISTED IN THE ORDER THEIR RECORDS' NAMES SORT, and a + # record's name begins with a `cksum` of its path - which differs between + # two sandboxes by construction. So the lines are compared as a set of + # lines, each of them whole. + out = "\n".join(sorted(inv.norm(res.stdout).split("\n"))) + return (res.returncode, out, inv.norm(res.stderr)) + + +def test_offline_no_read_the_inventory_acts_are_byte_identical_against_an_unreachable_index(tmp_path): + """OFFLINE NO-READ (14(b), #161): with an indexer installed and configured + at a Postgres nobody can reach, the inventory writers and `lane-reconcile` + answer byte for byte as with no index at all - and the writers' nudges DID + reach for that store, `sync --source inventory`, after they had answered.""" + control = Inventory(tmp_path / "control") + indexed = Inventory(tmp_path / "indexed") + try: + indexed.e.install_indexer() + fake = indexed.root / "pgfake" + fake.mkdir() + psql_log = indexed.root / "psql.log" + (fake / "psql").write_text( + f"#!/bin/sh\nprintf '%s\\n' \"$*\" >> {psql_log}\n" + "echo 'psql: error: connection to server at \"lanes-index.invalid\" failed' >&2\nexit 2\n") + (fake / "psql").chmod(0o755) + pw = indexed.e.config / "pgpass" + pw.write_text("*:*:*:*:never-read\n") + pw.chmod(0o600) + indexed.e.write_config(f"url=postgresql://lanes_writer@lanes-index.invalid/qa\npassfile={pw}\n") + extra = {"PATH": indexed.e.path_with(indexed.e.indexbin, fake), "LANES_INDEX": None} + for act in INVENTORY_ACTS: + a = _inv_act(control, act, {}) + b = _inv_act(indexed, act, extra) + assert a == b, (act, a, b) + assert _wait_for(psql_log, 15.0), "no inventory nudge ever reached for the configured store" + assert "lanes_writer" in psql_log.read_text() + finally: + control.close() + indexed.close() + + +def test_a_poisoned_worktrees_table_changes_no_inventory_acts_answer(tmp_path): + """POISONED INDEX (14(b), #161): wrong `worktrees` rows - a forged tree, a + forged lifecycle, forged counts and a forged owner - change no inventory + act's answer, `lane-reconcile` included; and the poison is REAL: `lanes + --index --worktrees` reads it.""" + control = Inventory(tmp_path / "control") + poisoned = Inventory(tmp_path / "poisoned") + try: + assert poisoned.sync().returncode == 0 + conn = sqlite3.connect(str(poisoned.e.sqlite)) + conn.execute("UPDATE worktrees SET lifecycle = 'SWAPPED', dirty_count = 99, " + "unpushed_count = 99, owner = 'managed evil', writer_live = 1") + conn.execute("INSERT INTO worktrees (source, record_id, workstation, lane, owner, path, " + "branch, lifecycle, digest, provenance, indexed_utc, schema_version) VALUES " + "('inventory:Eagle', 'repoA-1:/forged', 'Eagle', 'repoA-1', 'legacy', " + "'/forged', 'feat/forged', 'RUNNING', 'x', '0', 'now', 1)") + conn.commit() + conn.close() + for act in INVENTORY_ACTS: + a = _inv_act(control, act, {}) + b = _inv_act(poisoned, act, {"LANES_INDEX": None}) + assert a == b, (act, a, b) + seen = _wt(poisoned, "--index", env={"LANES_INDEX": None}) + assert "read: index (sqlite)" in seen.stderr + assert "/forged" in seen.stdout and "managed evil" in seen.stdout, \ + "the poison never reached the index read: vacuous" + finally: + control.close() + poisoned.close() + + +def test_an_inventory_write_nudges_and_a_read_never_does(inv): + """THE NUDGE (#161): `set-lane-tree` and `set-lane-state` start the same + detached `lanes-index sync` a landed push does, asked for the inventory - + stdin closed, never waited, the writer not delayed - while the reads an act + makes of the inventory start none, and `LANES_INDEX=off` stops it.""" + stubdir = inv.root / "stub" + stubdir.mkdir() + (stubdir / "lanes-index").write_text(STUB) + (stubdir / "lanes-index").chmod(0o755) + mark = inv.root / "stubmark" + path = inv.e.path_with(stubdir) + pid_file = Path(str(mark) + ".pid") + for read in (("lane-trees", "repoA-1"), ("lane-reconcile", "repoA-1"), ("lane-state", "repoA-1"), + ("worktrees", "--all")): + res = inv.e.edit(*read, env={"PATH": path, "LANES_INDEX": None, + "LANES_INDEX_STUB_MARK": str(mark)}) + assert res.returncode == 0, (read, res.stderr) + assert not _wait_for(pid_file, 1.0), "a read nudged the indexer" + off = inv.e.edit("set-lane-tree", "repoA-1", str(inv.run_tree), "--checkout", str(inv.repo), + env={"PATH": path, "LANES_INDEX": "off", "LANES_INDEX_STUB_MARK": str(mark)}) + assert off.returncode == 0, off.stderr + assert not _wait_for(pid_file, 1.0), "LANES_INDEX=off still nudged" + for write in (("set-lane-tree", "repoA-1", str(inv.run_tree), "--checkout", str(inv.repo)), + ("set-lane-state", "repoA-1", "SWAPPING", "--operation", "op-nudge")): + for leftover in stubdir.parent.glob("stubmark.*"): + leftover.unlink() + t0 = time.monotonic() + res = inv.e.edit(*write, env={"PATH": path, "LANES_INDEX": None, + "LANES_INDEX_STUB_MARK": str(mark)}, timeout=25) + took = time.monotonic() - t0 + assert res.returncode == 0, (write, res.stderr) + assert _wait_for(pid_file, 10.0), f"{write[0]} never nudged the indexer" + pid = int(pid_file.read_text().strip()) + try: + os.kill(pid, 0) + assert took < 20, f"{write[0]} waited on its nudge ({took:.1f}s)" + assert _wait_for(Path(str(mark) + ".argv"), 5.0) + assert Path(str(mark) + ".argv").read_text().strip() == "sync --source inventory" + assert _wait_for(Path(str(mark) + ".stdin"), 5.0) + assert Path(str(mark) + ".stdin").read_text().strip() == "eof" + finally: + try: + os.kill(pid, 15) + except OSError: + pass + + +def test_a_pull_request_is_asked_of_gh_best_effort(inv): + """`pr` (#161): the newest pull request whose head is the tree's branch, + asked of `gh` at sync where GitHub may be asked - a fake `gh` here - and + NULL wherever it is not, or does not answer.""" + inv._g(inv.repo, "remote", "set-url", "origin", "https://github.com/opensoft/repoA.git") + fake = inv.root / "ghfake" + fake.mkdir() + calls = inv.root / "gh.calls" + (fake / "gh").write_text( + f"#!/bin/sh\nprintf '%s\\n' \"$*\" >> {calls}\n" + "case \"$*\" in *feat/dirty*) echo '[{\"number\":42,\"state\":\"OPEN\"}]' ;; " + "*feat/run*) echo '[{\"number\":7,\"state\":\"MERGED\"}]' ;; *) echo '[]' ;; esac\n") + (fake / "gh").chmod(0o755) + env = {"PATH": inv.e.path_with(fake), "LANES_NO_GITHUB": None} + res = inv.sync(env=env) + assert res.returncode == 0, res.stderr + rows = inv.rows() + assert (rows[f"repoA-1:{inv.dirty_tree}"]["pr_number"], + rows[f"repoA-1:{inv.dirty_tree}"]["pr_state"]) == (42, "OPEN") + assert (rows[f"repoA-1:{inv.run_tree}"]["pr_number"], + rows[f"repoA-1:{inv.run_tree}"]["pr_state"]) == (7, "MERGED") + assert rows[f"repoA-2:{inv.a2_tree}"]["pr_number"] is None + assert "--repo opensoft/repoA --head feat/dirty" in calls.read_text() + # A MERGED answer is final and is not asked again; an OPEN one is. + calls.unlink() + assert inv.sync(env=env).returncode == 0 + asked = calls.read_text() + assert "feat/dirty" in asked and "feat/run" not in asked, asked + + +def test_status_names_the_inventory_and_a_mark_names_its_source(inv, monkeypatch): + """`status` prints this workstation's inventory beside the register; and a + register sync holding the lock when an inventory nudge arrives syncs the + INVENTORY before it lets go - the mark names its source.""" + st = inv.e.index("status") + assert "source inventory:Eagle" in st.stdout + assert "--source inventory` builds it" in st.stdout + assert inv.sync().returncode == 0 + st = inv.e.index("status") + assert re.search(r"provenance: inventory@[0-9a-f]{12} \(sync 1,", st.stdout), st.stdout + assert "rows: 5 worktree(s)" in st.stdout + _as_estate(monkeypatch, inv.e) + mod = load_indexer() + runs = [] + monkeypatch.setattr(mod, "run_index", lambda *a: runs.append(("register",) + a) or 0) + monkeypatch.setattr(mod, "run_inventory", lambda *a: runs.append(("inventory",) + a) or 0) + real_release = mod.SyncLock.release + released = [] + + def release_after_an_inventory_mark(self): + if not released and self.handle is not None: + self.mark_rerun("inventory") + released.append(1) + real_release(self) + + monkeypatch.setattr(mod.SyncLock, "release", release_after_an_inventory_mark) + assert mod.cmd_sync(None) == 0 + assert [r[0] for r in runs] == ["register", "inventory"], runs + lock = mod.SyncLock() + assert not lock.has_rerun() + + +def test_the_inventory_source_is_this_workstations_alone(inv): + """Another workstation's inventory is on its own disk and is refused; and a + container with no LANES_WORKSTATION files nothing under its own id + (Amendment 11, R-A11-14).""" + other = inv.e.index("sync", "--source", "inventory:Raven") + assert other.returncode == 2 and "this workstation's inventory is inventory:Eagle" in other.stderr + assert not inv.e.sqlite.exists() + nameless = inv.e.index("sync", "--source", "inventory", + env={"LANES_WORKSTATION": None, "LANES_IN_CONTAINER": "1"}) + assert nameless.returncode == 2, (nameless.stdout, nameless.stderr) + assert "LANES_WORKSTATION" in nameless.stderr + assert not inv.e.sqlite.exists() + reg = inv.e.index("export", "--out", inv.root, "--source", "inventory") + assert reg.returncode == 2 + both = inv.e.index("export", "--out", inv.root, "--worktrees", "--source", "inventory") + assert both.returncode == 64 From 3c76b5681dd3bc66af620f64112e4a7d658e661f Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:19:44 +0000 Subject: [PATCH 07/12] Leave a lane bound elsewhere with its liveness unknown `worktrees` read `writer_live` as the listing's LIVE and wrote 0 for every other lane. For a lane whose last STARTED or RESUMED bound it on another host, this workstation's session records say nothing about its writer, and Amendment 18(b) pronounces liveness only from inside the binding: from here it is unknown, never dead. The listing's column 13 says where a lane is bound, so `elsewhere` now leaves the field empty - NULL in the index, `unknown` in `lanes --worktrees` - exactly as unreadable session records already did. A test binds a fixture lane on Raven and reads both. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lanes-edit.sh | 15 +++++++++++---- tests/test_lanes_index.py | 22 ++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/lanes-edit.sh b/lanes-edit.sh index 6307d262..faddccf8 100755 --- a/lanes-edit.sh +++ b/lanes-edit.sh @@ -12610,8 +12610,9 @@ EOF # IT READS WHAT #97 WROTE AND NOTHING THAT IT DID NOT. The lanes are the # listing's own (`lanes_rows --all --closed`, so a closed or dormant lane's # leftovers are not hidden), and so is `writer_live` — the listing's LIVE, out -# of this workstation's session records, empty where those could not be read -# (Amendment 7(d): never "no"). The trees are `lane_trees_list`'s sidecars and +# of this workstation's session records, and EMPTY where those could not be +# read or the lane is bound elsewhere (Amendment 7(d) and 18(b): never "no" on a +# read nobody could make). The trees are `lane_trees_list`'s sidecars and # NO DIRECTORY IS WALKED for a tree no sidecar names: that is # `lane-reconcile`'s act, and it is an act. `owner` is `lane_is_managed_owned`'s # answer — the read `managed-projection` makes — as `legacy`, `managed ` @@ -12650,7 +12651,7 @@ worktree_rows() { # [] fi case "$wr_lrc" in 0) : ;; 8) return 8 ;; *) return 1 ;; esac wr_tab="$(printf '\t')" - while IFS="$wr_tab" read -r wr_l wr_st wr_w wr_pf wr_win wr_sid wr_dir wr_rest; do + while IFS="$wr_tab" read -r wr_l wr_st wr_w wr_pf wr_win wr_sid wr_dir wr_obj wr_age wr_restart wr_home wr_fk wr_loc wr_rest; do [ -n "${wr_l:-}" ] || continue # ONE CONTROL-ROOT READ FOR A LANE WITH NO TREES, which is nearly every lane # in the estate: the sidecars are listed only where a `trees` directory is. @@ -12676,9 +12677,15 @@ worktree_rows() { # [] wr_trees=""; wr_trc=0 wr_trees="$(lane_trees_list "$wr_l" 2>/dev/null)" || wr_trc=$? case "$wr_trc" in 0) : ;; *) continue ;; esac + # LIVE IS PRONOUNCED, NOT-LIVE ONLY WHERE IT MAY BE (Amendment 18(b)). A + # live record here naming one of the lane's ids is `1`; no such record is + # `0` only where this workstation could read its records AND the lane is not + # bound elsewhere (the listing's column 13) — from outside a binding, + # liveness is UNKNOWN, never dead, and that is the empty field. wr_live=0 - [ "$wr_st" = LIVE ] && wr_live=1 + [ "${wr_loc:-none}" = elsewhere ] && wr_live="" [ "$wr_live_ok" = 1 ] || wr_live="" + [ "$wr_st" = LIVE ] && wr_live=1 wr_own=""; wr_orc=0 wr_own="$(lane_is_managed_owned "$wr_l" 2>/dev/null)" || wr_orc=$? wr_own="$(printf '%s\n' "$wr_own" | head -n 1)" diff --git a/tests/test_lanes_index.py b/tests/test_lanes_index.py index 584c517f..40296aa2 100644 --- a/tests/test_lanes_index.py +++ b/tests/test_lanes_index.py @@ -2057,3 +2057,25 @@ def test_the_inventory_source_is_this_workstations_alone(inv): assert reg.returncode == 2 both = inv.e.index("export", "--out", inv.root, "--worktrees", "--source", "inventory") assert both.returncode == 64 + + +def test_a_lane_bound_elsewhere_has_no_liveness_pronounced_here(inv): + """AMENDMENT 18(b) FOR `writer_live`: a lane whose last STARTED/RESUMED bound + it on another host cannot be pronounced not-live from here - its session + records are that host's - so the field is NULL, `unknown` in the listing, + and never the `0` that reads as *nobody is writing there*.""" + log = inv.e.wip / "lanes" / "log" / "repoA-1.md" + log.write_text(log.read_text(encoding="utf-8") + + f"RESUMED — lane repoA-1, session {UUID_A1B}@Raven, 2026-10-05T00:00:00Z, " + f"lane:repoA-1 → host raven; container none; window rsess:1 @9; os macos\n", + encoding="utf-8") + inv.e._git("add", "-A") + inv.e._git("commit", "-q", "-m", "repoA-1 is bound on Raven now") + inv.e._git("push", "-q", "origin", "main") + assert inv.sync().returncode == 0 + rows = inv.rows() + assert rows[f"repoA-1:{inv.run_tree}"]["writer_live"] is None + assert rows[f"repoA-2:{inv.a2_tree}"]["writer_live"] == 1 + listed = _wt(inv, "repoA-1") + run = next(l for l in listed.stdout.splitlines() if l.endswith(str(inv.run_tree))) + assert run.split()[4] == "unknown", run From fee2aa30ddd455af0d0d6c2564b8df60cf34132e Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:19:50 +0000 Subject: [PATCH 08/12] Say in the manual when writer_live is unknown The column table said only that unreadable session records leave it NULL; since the previous commit a lane bound on another host does too, and a reader of the table has to know that NULL is "not established here", not "nobody is writing". Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- docs/README-lanes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README-lanes.md b/docs/README-lanes.md index a7873340..6b9e7147 100644 --- a/docs/README-lanes.md +++ b/docs/README-lanes.md @@ -3318,7 +3318,7 @@ lanes-index reconcile --source inventory [--dry-run] | `path`, `branch`, `dirty_count`, `unpushed_count`, `last_seen_utc` | the tree's sidecar: the last observation, `detached ` for a detached head, an `unknown` count as NULL. A sidecar that could not be read is `unreadable sidecar`, one of another schema `unknown schema `, each with the record's id where its path would be | | `base`, `last_commit` | two git reads of the RECORDED head, in the tree or, once it is gone, its recorded checkout: its merge-base with `origin/main` (else `main`, else `origin/HEAD`), and ` ` | | `lifecycle`, `generation`, `operation` | the lane's snapshot (`lane-state`): `RUNNING`, `SWAPPING`, `SWAPPED` or `CLOSED` for a **legacy** lane; `INDETERMINATE` for a managed or unknown one, whose lifecycle this tooling does not pronounce on, and for a snapshot that is missing, unreadable or of another schema | -| `writer_live` | the listing's own LIVE — a session record on this workstation naming one of the row's ids — and NULL where those records could not be read | +| `writer_live` | the listing's own LIVE — a session record on this workstation naming one of the row's ids — and NULL where those records could not be read or the lane is bound on another host, from where liveness is unknown and never dead (Amendment 18(b)) | | `pr_number`, `pr_state` | `gh` at sync, best effort, the newest pull request whose head is the branch; NULL wherever it was not asked (`LANES_NO_GITHUB=1`) or did not answer. A MERGED or CLOSED answer is not asked again while the branch and its last commit stand | | `provenance`, `indexed_utc` | the generation the TREE'S OWN record was filed under — behind `generation` exactly when the observation predates the lane's current transition — and the sync | From 8f6e98e9e8c82fc0ac2d81e1da72deb1ec9aeeef Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:59:29 +0000 Subject: [PATCH 09/12] Answer an inventory refusal inside the sync loop A register sync that takes the inventory's rerun mark runs the inventory sync in its own loop; a refusal there (a container with no LANES_WORKSTATION) used to propagate as an exception out of that loop. It is now said and returned as the 2 it is, exactly as `main` would say it, so the exit is the same and the loop ends the way every other failed round ends. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lanes-index | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lanes-index b/lanes-index index 94a14f60..b56b5a33 100755 --- a/lanes-index +++ b/lanes-index @@ -1985,7 +1985,13 @@ def is_inventory(requested: str | None) -> bool: def _sync_once(kind: str, requested: str | None) -> int: if kind == "inventory": - return _guarded(lambda: run_inventory("sync", requested, False), requested, kind) + # A REFUSAL HERE IS ANSWERED HERE: a register sync that took the + # inventory's mark must not lose the marks behind it to an exception. + try: + return _guarded(lambda: run_inventory("sync", requested, False), requested, kind) + except Refusal as exc: + note(f"REFUSED: {exc}") + return 2 return _guarded(lambda: run_index("sync", requested, False), requested, kind) From d42daf4a66674b7172caa78a6ed8d18669398d33 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 19:37:05 +0000 Subject: [PATCH 10/12] Hold the inventory acts to the index's three-way comparison The shell suite's Amendment 14 section runs each act three ways - as it always ran, against an indexer configured at a store nobody can reach, and over a poisoned index. It now syncs this suite's whole inventory into the poisoned store and makes those rows wrong too (a forged tree, forged lifecycles, counts and owner), proves `worktrees --index` reads the poison while the plain read does not, and adds the inventory's acts to the comparison: set-lane-tree and set-lane-state, whose nudge must reach the unreachable store, and lane-reconcile, lane-trees and lane-state, which reach for nothing. The snapshot taken between runs now carries the lanes' control root, or a writer's second run would be a generation ahead of its first. set-lane-tree writes repoRC-2's record, because repoRC-5's record of the same tree was left at a schema the writer refuses to replace. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- tests/test_lane_helpers.sh | 49 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/test_lane_helpers.sh b/tests/test_lane_helpers.sh index 3031dc37..94219469 100755 --- a/tests/test_lane_helpers.sh +++ b/tests/test_lane_helpers.sh @@ -14636,6 +14636,33 @@ run env -u LANES_INDEX PATH="$A14_BIN:$PATH" LANES_INDEX_CONFIG="$A14_CONF/poiso has "the poison is REAL: \`history --index\` reads it, so the sameness below is not vacuous" "$out" "A14 POISON" has "…and says it read the index" "$err" "read: index (sqlite)" +# AND THE INVENTORY'S TABLE (opensoft/openRepoTools#161), true first and then +# wrong: every lane's #97 sidecars on this workstation, synced into the same +# store, then a forged tree, forged lifecycles, forged counts and a forged +# owner. The acts below that write or read the inventory must not move. +run env -u LANES_INDEX LANES_INDEX_CONFIG="$A14_CONF/poisoned.conf" LANES_EDIT="$E" "$A14_BIN/lanes-index" sync --source inventory +is "lanes-index syncs this workstation's worktree inventory into the same store" "$rc" 0 +has "…and writes its rows" "$out" "inventory:Eagle (sqlite)" +python3 - "$A14_DB" <<'PY' +import sqlite3, sys +c = sqlite3.connect(sys.argv[1]) +n = c.execute("SELECT COUNT(*) FROM worktrees").fetchone()[0] +assert n > 0, "the inventory sync wrote no worktrees row: the poison below would be vacuous" +c.execute("UPDATE worktrees SET lifecycle = 'SWAPPED', dirty_count = 99, unpushed_count = 99, " + "owner = 'managed a14-poison', writer_live = 1") +c.execute("INSERT INTO worktrees (source, record_id, workstation, lane, owner, path, branch, " + "lifecycle, digest, provenance, indexed_utc, schema_version) VALUES ('inventory:Eagle', " + "'repoRC-5:/a14/forged', 'Eagle', 'repoRC-5', 'legacy', '/a14/forged', 'feat/a14-poison', " + "'RUNNING', 'x', '0', 'now', 1)") +c.commit() +PY +run env -u LANES_INDEX PATH="$A14_BIN:$PATH" LANES_INDEX_CONFIG="$A14_CONF/poisoned.conf" "$E" worktrees repoRC-5 --index +has "the inventory's poison is REAL: \`worktrees --index\` reads the forged tree" "$out" "/a14/forged" +has "…and the forged owner" "$out" "managed a14-poison" +has "…and says it read the index's table" "$err" "read: index (sqlite) at inventory:Eagle synced" +run env -u LANES_INDEX PATH="$A14_BIN:$PATH" LANES_INDEX_CONFIG="$A14_CONF/poisoned.conf" "$E" worktrees repoRC-5 +is "…while the same read WITHOUT the flag is the sidecars', and carries no poison" "$(printf '%s' "$out" | grep -c 'a14-poison' || :)" 0 + a14_norm() { printf '%s' "$1" | sed -E \ -e 's/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z//g' \ @@ -14643,6 +14670,10 @@ a14_norm() { -e 's/[0-9a-f]{7,}//g' } A14_SNAP="$SANDBOX/a14snap" +# THE LANES' CONTROL ROOT TOO (#161): the inventory writers below change it, +# and a snapshot that put the register back and left the lifecycle advanced +# would compare a generation-3 run with a generation-4 one. +A14_LSR="$HOME/projects/.lane-state" # NO GIT GC RUNS UNDER THE SNAPSHOT (CI on #164, `tests-no-submodule` at # 50594dc: "rm: cannot remove '…/origin.git': Directory not empty"). By this # point the suite has pushed thousands of commits, so a push or a commit can @@ -14681,6 +14712,7 @@ a14_snap() { cp -a "$WIP" "$A14_SNAP/wip" cp -a "$ORIGIN" "$A14_SNAP/origin" if [ -d "$CLAUDE_CONFIG_DIR/lanes" ]; then cp -a "$CLAUDE_CONFIG_DIR/lanes" "$A14_SNAP/cclanes"; fi + if [ -d "$A14_LSR" ]; then cp -a "$A14_LSR" "$A14_SNAP/lsr"; fi } a14_restore() { a14_gc_quiet @@ -14689,6 +14721,9 @@ a14_restore() { if a14_rmtree "$CLAUDE_CONFIG_DIR/lanes" && [ -d "$A14_SNAP/cclanes" ]; then cp -a "$A14_SNAP/cclanes" "$CLAUDE_CONFIG_DIR/lanes" fi + if a14_rmtree "$A14_LSR" && [ -d "$A14_SNAP/lsr" ]; then + cp -a "$A14_SNAP/lsr" "$A14_LSR" + fi } # A WRITE'S NUDGE RUNS DETACHED and may still be reading this checkout when the # snapshot goes back: wait for its one line in the fake `psql`'s log (bounded), @@ -14739,6 +14774,20 @@ a14_three "the name guard" 0 bash -c '"$1" guard < "$2"' a14-guard "$E" "$SANDBO a14_three "who (a read an act makes for itself)" 0 "$E" who opensoft/repoA#1414 a14_three "managed-projection (#97's seam)" 0 "$E" managed-projection repoMG-1 a14_three "history without the flag" 0 "$E" history repoA-1 +# THE INVENTORY'S ACTS (opensoft/openRepoTools#161): its two writers, whose +# nudge is `lanes-index sync --source inventory`, and the reads an act makes of +# the inventory for itself — `lane-reconcile` above all, which reads the +# sidecars and the disk and never the table. +# repoRC-2, and not repoRC-5: (e) above left repoRC-5's record of w1 at a +# schema this helper refuses to replace, which is a refusal and not a write. +a14_three "set-lane-tree (an inventory writer)" 1 "$E" set-lane-tree repoRC-2 "$RC_DIR/.claude/worktrees/w1" --checkout "$RC_DIR" +is "…which the control run recorded, so the comparison is of a real write" "$a14_c_rc" 0 +a14_three "set-lane-state (the lifecycle writer)" 1 "$E" set-lane-state repoRC-2 SWAPPING --operation a14-inventory-op +is "…which the control run moved" "$a14_c_rc" 0 +a14_three "lane-reconcile (the act's own read of the inventory)" 0 "$E" lane-reconcile repoRC-5 +a14_three "lane-reconcile of a lane mid-swap" 0 "$E" lane-reconcile repoRC-2 +a14_three "lane-trees" 0 "$E" lane-trees repoRC-5 +a14_three "lane-state" 0 "$E" lane-state repoRC-2 # ------------------------------------------------------- nothing real touched # From c765cba0d14471727b3754d5762ecc6b0b6b05b9 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:28:32 +0000 Subject: [PATCH 11/12] Take Copilot round one on the worktrees table Four findings, each a real defect: - writer_live: the listing's LIVE overrode the locality and readability checks, so a still-running local session of an older binding read as a live writer for a lane now bound on another host. Amendment 18(b) pronounces liveness only from inside the binding: `elsewhere` and an unreadable session read now win, and a test binds the live fixture lane on Raven. - the no-op: "nothing to write" was decided from the rows alone, so a fingerprint changed by hand with every row right was never repaired. The source row must now agree too, and a test changes it and syncs. - the sync loop returned on the first failed source, dropping every mark already taken off the disk with it. It now drains them all and exits with the first failure; a test fails the register round and sees the inventory synced. - the manual (and the listing's own comment) promised the index read was the source read byte for byte; on a store other workstations sync into it carries their rows too. The promise is now this workstation's rows, and the whole read only where no other workstation writes the store. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- docs/README-lanes.md | 6 ++++-- lanes | 7 ++++--- lanes-edit.sh | 15 +++++++++------ lanes-index | 17 ++++++++++++----- tests/test_lanes_index.py | 36 ++++++++++++++++++++++++++++++++++++ 5 files changed, 65 insertions(+), 16 deletions(-) diff --git a/docs/README-lanes.md b/docs/README-lanes.md index 6b9e7147..028741a3 100644 --- a/docs/README-lanes.md +++ b/docs/README-lanes.md @@ -3355,8 +3355,10 @@ every source a mark names before it lets go. `--index` it reads the table — every `inventory:*` source the store holds, so Raven sees Eagle's leftovers — and stderr says `read: index () at inventory: synced `, or `read: sources (index )` and the sidecars -answer. Where the table is current the two print the same bytes. Every row is a -last observation: the current truth of a lane is `lanes-edit.sh lane-reconcile +answer. Where the table is current, this workstation's rows are the same bytes +either way — and a store only this workstation syncs into (SQLite, or a +Postgres nobody else writes) prints exactly the source read, while a shared one +adds the other workstations' rows among them. Every row is a last observation: the current truth of a lane is `lanes-edit.sh lane-reconcile `, which reads the disk, and the footer says so. **What never reads it.** No act: not `lane-reconcile` — there is no diff --git a/lanes b/lanes index 1e54640c..9d54f7ca 100755 --- a/lanes +++ b/lanes @@ -289,9 +289,10 @@ done # EVERY OPEN WORKTREE, rendered from `lanes-edit.sh worktrees`' rows — the #97 # inventory as each lane's handoff last recorded it, read from this # workstation's sidecars, or with `--index` from the derived index's -# `worktrees` table, which holds every workstation that syncs into it. The two -# reads print the same bytes on stdout wherever the index is current; stderr -# says which was read. It recomputes nothing and it decides nothing: the act +# `worktrees` table, which holds every workstation that syncs into it. Where +# the index is current, this workstation's rows are the same bytes either way, +# and a store no other workstation syncs into prints exactly the source read; +# stderr says which was read. It recomputes nothing and it decides nothing: the act # that reads the disk is `lanes-edit.sh lane-reconcile `, and the footer # says so. US="$(printf '\037')" diff --git a/lanes-edit.sh b/lanes-edit.sh index faddccf8..4b71d241 100755 --- a/lanes-edit.sh +++ b/lanes-edit.sh @@ -12677,15 +12677,18 @@ worktree_rows() { # [] wr_trees=""; wr_trc=0 wr_trees="$(lane_trees_list "$wr_l" 2>/dev/null)" || wr_trc=$? case "$wr_trc" in 0) : ;; *) continue ;; esac - # LIVE IS PRONOUNCED, NOT-LIVE ONLY WHERE IT MAY BE (Amendment 18(b)). A - # live record here naming one of the lane's ids is `1`; no such record is - # `0` only where this workstation could read its records AND the lane is not - # bound elsewhere (the listing's column 13) — from outside a binding, - # liveness is UNKNOWN, never dead, and that is the empty field. + # LIVENESS IS PRONOUNCED ONLY FROM INSIDE THE BINDING (Amendment 18(b)). A + # live record here naming one of the lane's ids is `1` and no such record is + # `0` - but only where this workstation could read its records AND the + # lane's latest binding is not on another host (the listing's column 13). + # Either of those says UNKNOWN, the empty field, and it wins over the + # listing's LIVE too (Copilot round 1 on #171): a still-running local + # session of an older binding says nothing about the writer the lane's + # binding elsewhere has now. wr_live=0 + [ "$wr_st" = LIVE ] && wr_live=1 [ "${wr_loc:-none}" = elsewhere ] && wr_live="" [ "$wr_live_ok" = 1 ] || wr_live="" - [ "$wr_st" = LIVE ] && wr_live=1 wr_own=""; wr_orc=0 wr_own="$(lane_is_managed_owned "$wr_l" 2>/dev/null)" || wr_orc=$? wr_own="$(printf '%s\n' "$wr_own" | head -n 1)" diff --git a/lanes-index b/lanes-index index b56b5a33..83ccda43 100755 --- a/lanes-index +++ b/lanes-index @@ -1955,7 +1955,10 @@ def run_inventory(mode: str, requested: str | None, dry_run: bool) -> int: if dry_run: say(f" total: {n_up} to upsert, {n_del} to remove - nothing written (--dry-run)") return 0 - if old is not None and not upserts and not deletes: + # NOTHING TO WRITE ONLY WHERE THE SOURCE'S OWN ROW AGREES TOO (Copilot + # round 1 on #171): a fingerprint changed by hand, with every row right, is + # put right here rather than left disagreeing with the table it names. + if old is not None and not upserts and not deletes and old.get("commit_sha") == fingerprint: if mode == "sync": say(f"{PROG} sync: {label} - the index is current; nothing to write") else: @@ -2039,8 +2042,11 @@ def cmd_sync(requested: str | None) -> int: say(f"{PROG} sync: origin/{branch()} kept moving through {rounds} reads; " f"nothing written - the next write's nudge syncs it") result = 0 - if result != 0: - return result + # A FAILED SOURCE DOES NOT DROP THE OTHERS (Copilot round 1 on #171): + # their marks were taken off the disk with this one's, so the loop + # drains them all and the exit is the first failure. + if result != 0 and rc == 0: + rc = result if rounds >= 8: return rc finally: @@ -2292,8 +2298,9 @@ def _export_worktrees(out_dir: str, cfg: dict, unusable) -> int: return 2 except StoreUnusable as exc: return unusable(exc.kind, exc.detail) - # THE BYTE ORDER `LC_ALL=C sort` GIVES the source read, so the two are one - # stream wherever the index is current. + # THE BYTE ORDER `LC_ALL=C sort` GIVES the source read, so this + # workstation's rows are the source read's own stream wherever the index is + # current; a shared store adds other workstations' rows among them. lines.sort(key=lambda line: line.encode("utf-8")) with open(os.path.join(out_dir, "worktrees"), "w", encoding="utf-8", newline="") as handle: for line in lines: diff --git a/tests/test_lanes_index.py b/tests/test_lanes_index.py index 40296aa2..ff618482 100644 --- a/tests/test_lanes_index.py +++ b/tests/test_lanes_index.py @@ -1721,6 +1721,17 @@ def test_a_wiped_inventory_reconciles_row_for_row(inv): assert re.search(r"worktrees\s+4 kept\s+1 upserted", fixed.stdout), fixed.stdout righted = dump(inv.e.sqlite) assert righted["worktrees"] == synced["worktrees"] + # A FINGERPRINT CHANGED BY HAND, every row right, is put right by the next + # sync rather than left naming a table that is not there (Copilot round 1 + # on #171). + conn = sqlite3.connect(str(inv.e.sqlite)) + conn.execute("UPDATE provenance SET commit_sha = ? WHERE source = 'inventory:Eagle'", ("f" * 64,)) + conn.commit() + conn.close() + healed = inv.sync() + assert healed.returncode == 0 and "0 upserted, 0 removed" in healed.stdout, healed.stdout + assert [r[:2] for r in dump(inv.e.sqlite)["provenance"]] == \ + [r[:2] for r in synced["provenance"]] # the same table, so the same fingerprint - and a sync count that went UP assert [r[:2] for r in righted["provenance"]] == [r[:2] for r in synced["provenance"]] assert righted["provenance"][0][2] == synced["provenance"][0][2] + 1 @@ -2039,6 +2050,17 @@ def release_after_an_inventory_mark(self): assert [r[0] for r in runs] == ["register", "inventory"], runs lock = mod.SyncLock() assert not lock.has_rerun() + # A FAILED SOURCE DOES NOT DROP A MARK ALREADY TAKEN (Copilot round 1 on + # #171): the register's round fails, the inventory's mark - taken off the + # disk in the same look - is still synced, and the exit is the failure. + runs.clear() + released.clear() + lock.mark_rerun("inventory") + monkeypatch.setattr(mod, "run_index", lambda *a: runs.append(("register",) + a) or 1) + monkeypatch.setattr(mod.SyncLock, "release", real_release) + assert mod.cmd_sync(None) == 1 + assert [r[0] for r in runs] == ["register", "inventory"], runs + assert not lock.has_rerun() def test_the_inventory_source_is_this_workstations_alone(inv): @@ -2079,3 +2101,17 @@ def test_a_lane_bound_elsewhere_has_no_liveness_pronounced_here(inv): listed = _wt(inv, "repoA-1") run = next(l for l in listed.stdout.splitlines() if l.endswith(str(inv.run_tree))) assert run.split()[4] == "unknown", run + # AND LOCALITY WINS OVER A LOCAL LIVE RECORD (Copilot round 1 on #171): + # repoA-2's session is still running HERE, under an id its row carries, + # and its latest binding is now on Raven - which is where its writer is + # pronounced on, not here. + log = inv.e.wip / "lanes" / "log" / "repoA-2.md" + log.write_text(log.read_text(encoding="utf-8") + + f"RESUMED — lane repoA-2, session {UUID_A2}@Raven, 2026-10-05T00:00:00Z, " + f"lane:repoA-2 → host raven; container none; window rsess:2 @9; os macos\n", + encoding="utf-8") + inv.e._git("add", "-A") + inv.e._git("commit", "-q", "-m", "repoA-2 is bound on Raven too") + inv.e._git("push", "-q", "origin", "main") + assert inv.sync().returncode == 0 + assert inv.rows()[f"repoA-2:{inv.a2_tree}"]["writer_live"] is None From f1e96c19fec4e76e7df9b744f2b7f3f004877940 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:40:50 +0000 Subject: [PATCH 12/12] Take Copilot round two on the worktrees table Three findings, each a way rows or marks could be lost: - `worktrees` read any lane-trees failure other than "no record" as an empty inventory once the trees directory was found, so the next sync removed that lane's rows. 8 stays "nothing recorded"; any other exit is now an UNREAD row, whose rows every sync keeps. A test makes a lane's trees directory unlistable and finds its rows kept and the listing saying NOT READ. - the per-lane forward fence skipped a lane that names no tree now, so a lane whose snapshot was moved aside and begun again without trees was emptied by a sync. For the few lanes whose rows would all leave, the indexer now asks `lanes-edit.sh lane-state` for the generation and holds the lane where it went back; reconcile takes it. A test does exactly that. - the eighth-round cap returned with sources still due whose marks it had already taken off the disk. It now marks them again for the next sync, as a mark it never took is left; a test keeps both sources busy to the cap and finds both marks on disk. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lanes-edit.sh | 12 ++++++- lanes-index | 35 +++++++++++++++++++ tests/test_lanes_index.py | 72 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 1 deletion(-) diff --git a/lanes-edit.sh b/lanes-edit.sh index 4b71d241..4c542850 100755 --- a/lanes-edit.sh +++ b/lanes-edit.sh @@ -12676,7 +12676,17 @@ worktree_rows() { # [] fi wr_trees=""; wr_trc=0 wr_trees="$(lane_trees_list "$wr_l" 2>/dev/null)" || wr_trc=$? - case "$wr_trc" in 0) : ;; *) continue ;; esac + # 8 IS A `trees` DIRECTORY WITH NO RECORD IN IT; ANYTHING ELSE, with the + # directory already found above, is a read that did not happen - and its + # rows are not rows this lane no longer has (Copilot round 2 on #171). + case "$wr_trc" in + 0) : ;; + 8) continue ;; + *) + wr_out="${wr_out}UNREAD$US$WS$US$wr_l${US}its trees directory $wr_root/trees is there and lane-trees could not read it (exit $wr_trc) +" + continue ;; + esac # LIVENESS IS PRONOUNCED ONLY FROM INSIDE THE BINDING (Amendment 18(b)). A # live record here naming one of the lane's ids is `1` and no such record is # `0` - but only where this workstation could read its records AND the diff --git a/lanes-index b/lanes-index index 83ccda43..2bb58ed2 100755 --- a/lanes-index +++ b/lanes-index @@ -1774,6 +1774,22 @@ def inventory_digest(record: dict) -> str: return hashlib.sha256(blob.encode("utf-8")).hexdigest() +def lane_generation(lane: str) -> tuple: + """`(generation, None)` of a lane's lifecycle snapshot, asked of `lanes-edit.sh + lane-state` - 0 where the lane has no snapshot or no control root here - or + `(None, why)` where the snapshot is there and could not be read.""" + proc = _helper_run(["lane-state", lane], 120, f"lane {lane}'s lifecycle snapshot") + if proc.returncode in (1, 8): + return 0, None + if proc.returncode != 0: + return None, f"its lifecycle snapshot could not be read (lane-state exited {proc.returncode})" + fields = dict(line.split("\t", 1) for line in proc.stdout.splitlines() if "\t" in line) + if fields.get("state") == "UNKNOWN-SCHEMA": + return None, f"its lifecycle snapshot records schema {fields.get('schema')}" + gen = fields.get("generation", "") + return (int(gen) if gen.isdigit() else 0), None + + def collect_inventory() -> tuple[dict, dict]: """Every tree the inventory names, keyed `:`, and every lane whose inventory is there and could not be read - whose rows are KEPT, never @@ -1904,6 +1920,19 @@ def run_inventory(mode: str, requested: str | None, dry_run: bool) -> int: stored_gen: dict = {} for row in stored.values(): stored_gen[row["lane"]] = max(stored_gen.get(row["lane"], 0), row["generation"] or 0) + # A LANE WHOSE ROWS WOULD ALL LEAVE IS FENCED TOO (Copilot round 2 on #171). + # It names no tree now, so it carries no generation in the read; its + # snapshot is asked for - only for these lanes, which are few - so that one + # moved aside and begun again without trees is held for `reconcile` like + # any other lane that went back, rather than emptied by a `sync`. + for lane in sorted(set(stored_gen) - set(read_gen) - set(held)): + if not stored_gen[lane]: + continue + gen, why = lane_generation(lane) + if why: + held[lane] = why + else: + read_gen[lane] = gen taken: list = [] for lane, gen in sorted(read_gen.items()): if lane in held or gen >= stored_gen.get(lane, 0): @@ -2048,6 +2077,12 @@ def cmd_sync(requested: str | None) -> int: if result != 0 and rc == 0: rc = result if rounds >= 8: + # THE CAP HANDS BACK WHAT IT DID NOT RUN (Copilot round 2 on + # #171): a source whose mark this loop took off the disk and + # never synced is marked again, for the next sync, exactly as a + # mark this loop never took is left for it. + for pending in due: + lock.mark_rerun(pending) return rc finally: lock.release() diff --git a/tests/test_lanes_index.py b/tests/test_lanes_index.py index ff618482..e0c11d9c 100644 --- a/tests/test_lanes_index.py +++ b/tests/test_lanes_index.py @@ -2115,3 +2115,75 @@ def test_a_lane_bound_elsewhere_has_no_liveness_pronounced_here(inv): inv.e._git("push", "-q", "origin", "main") assert inv.sync().returncode == 0 assert inv.rows()[f"repoA-2:{inv.a2_tree}"]["writer_live"] is None + + +def test_an_inventory_that_cannot_be_read_keeps_its_rows(inv): + """A READ THAT DID NOT HAPPEN IS NOT AN EMPTY INVENTORY (Amendment 7(d); + Copilot round 2 on #171): a lane whose `trees` directory is there and + cannot be listed is `UNREAD` - named in the listing, and its rows KEPT by a + sync rather than removed as trees it no longer has.""" + assert inv.sync().returncode == 0 + trees = inv.lsr / "repoA-1" / "trees" + trees.chmod(0) + try: + if os.access(trees, os.R_OK): + pytest.skip("this account reads a mode-000 directory (root): nothing to prove here") + res = inv.sync() + assert res.returncode == 0, res.stderr + assert "lane repoA-1: its rows are kept as they are" in res.stdout, res.stdout + assert "0 removed" in res.stdout or "nothing to write" in res.stdout, res.stdout + assert f"repoA-1:{inv.dirty_tree}" in inv.rows() + listed = _wt(inv) + assert "inventory NOT READ" in listed.stdout, listed.stdout + assert "could not be read: that is NOT a lane with no worktrees" in listed.stdout + finally: + trees.chmod(0o755) + + +def test_a_lane_that_went_back_with_no_trees_is_held_too(inv): + """THE FENCE FOR A LANE WHOSE ROWS WOULD ALL LEAVE (Copilot round 2 on + #171): its snapshot moved aside and begun again, and no tree recorded since + - a `sync` asks `lane-state` for its generation, finds it went back, and + keeps its rows; `reconcile` takes the lane as it now stands.""" + assert inv.sync().returncode == 0 + snap = inv.lsr / "repoA-2" / "lane-state.yaml" + snap.rename(snap.with_suffix(".aside")) + for sidecar in (inv.lsr / "repoA-2" / "trees").iterdir(): + sidecar.unlink() + inv.ok("set-lane-state", "repoA-2", "RUNNING", "--operation", "op-test-empty") + res = inv.sync() + assert res.returncode == 0, res.stderr + assert "lane repoA-2: its rows are kept as they are" in res.stdout, res.stdout + assert f"repoA-2:{inv.a2_tree}" in inv.rows() + rec = inv.e.index("reconcile", "--source", "inventory") + assert rec.returncode == 0, rec.stderr + assert "generation went back from 2 to 1" in rec.stdout, rec.stdout + assert f"repoA-2:{inv.a2_tree}" not in inv.rows() + + +def test_the_round_cap_hands_back_a_mark_it_took(estate, monkeypatch): + """THE CAP LOSES NO MARK (Copilot round 2 on #171): a holder that stops at + its eighth round with a source still due - its mark taken off the disk and + not yet synced - marks it again for the next sync.""" + _as_estate(monkeypatch, estate) + mod = load_indexer() + lock = mod.SyncLock() + os.makedirs(mod.state_dir(), exist_ok=True) + runs = [] + + def busy(kind): + def run(*a): + runs.append(kind) + lock.mark_rerun(kind) # every round, a write lands behind it + return 0 + return run + + monkeypatch.setattr(mod, "run_index", busy("register")) + monkeypatch.setattr(mod, "run_inventory", busy("inventory")) + lock.mark_rerun("inventory") + assert mod.cmd_sync(None) == 0 + assert len(runs) == 8, runs + assert runs[-1] == "inventory", runs + assert os.path.exists(lock.rerun_path("register")), \ + "the register's mark was taken at the cap and never handed back" + assert os.path.exists(lock.rerun_path("inventory"))