diff --git a/docs/README-lanes.md b/docs/README-lanes.md index 6a2e2668..802beca5 100644 --- a/docs/README-lanes.md +++ b/docs/README-lanes.md @@ -3473,10 +3473,17 @@ The flag is typed per invocation — **never** taken from the environment or a configuration. It replaces the **published** reads (the register, the archive, the alias table, the logs) and nothing else, so the live session records, tmux and this checkout's own files are read as they are without it. Where the index -is current the answer is **byte for byte the source read's**, the clock's -columns aside (the listing's AGE and its "last heard from origin" age, computed -the same way at the moment of the read). One line on stderr says which was -read: +is current the answer is **byte for byte the source read's**, on stderr as on +stdout — the warnings for a log's unreadable lines and the count that closes +them included (opensoft/openRepoTools#166) — the clock's columns aside (the +listing's AGE and its "last heard from origin" age, computed the same way at +the moment of the read). Stderr carries the flag's OWN lines besides: the +`read:` line below, and for `history` the `LANES_NO_FETCH=1 — not fetching …` +note, because where the index answers that arm fetches nothing in front of it. +And it is **one commit's image**: `lanes-index export` reads the provenance and +every table in ONE read snapshot — a SQLite read transaction, a Postgres +`REPEATABLE READ READ ONLY` one — so a sync landing mid-read is in the answer +whole or not at all (#166). One line on stderr says which was read: ```text read: index (sqlite) at register@1a2b3c4d5e6f; 0 commits behind @@ -3598,19 +3605,41 @@ reader, its snapshot through `lane-state`'s and its owner through that is `lane-reconcile`'s act. Nothing is recomputed: `dirty` and `unpushed` are what the handoff saw, and `LAST SEEN` says when. -**Rows follow the inventory.** Keyed by (source, `:`) and versioned -by a digest of the row and its provenance, so a replay writes nothing; a tree +**Rows follow the inventory.** Keyed by (source, `:`) — a second +tree of one lane with the same path (two with no `path:`, say) as +`:`, numbered from 2 in the read's sorted order, and a lane's +unread marker as `:unread` — and versioned by a digest of the row and +its provenance, so a replay writes nothing (a change that reorders two trees of +one path moves both keys, and that sync upserts both); a tree the inventory no longer names — removed at landing, or by a sweep's disposition — leaves in the same sync. The provenance moves only forward **per lane**: a `sync` that reads a lane's lifecycle generation below the one its rows were written at (a snapshot moved aside and begun again) writes nothing for that lane and says so, and `reconcile --source inventory` is the act that takes the lane as it now stands. A lane whose `trees` directory is there and cannot be listed -keeps its rows — it is not a lane with no worktrees. The source's own row -carries a fingerprint of the whole table as its compare-and-swap token and a -sync count that only goes up. A container with no `LANES_WORKSTATION` syncs -nothing (exit 2): a source named for a container's own id is one no restart -finds again. +keeps its rows — it is not a lane with no worktrees — and gets ONE MARKER ROW, +keyed `:unread`, with no `path` and the `branch` `unreadable inventory +()` — the key is what makes it one, because a tree's path can be empty (a +sidecar with no `path:`) and a hand-made sidecar's branch can read the same — +which the `--index` read prints as the same `inventory NOT READ` line the sidecars +give (opensoft/openRepoTools#173): a FIRST sync that cannot read a lane has no +rows to keep and is still not an empty answer, and rows held over from an +earlier sync are said to be, on that line. The marker leaves with the first +sync that reads the lane, and `lanes-index status` counts it apart from the +worktrees. The schema stays 1, so an indexer from before #173 sharing a +Postgres store reads a marker as a tree: its `export --worktrees` prints it as +a `TREE` line with an empty path and that branch, and its `status` counts it as +a worktree. An older and a newer indexer syncing ONE store each converge to their +own view of the same inventory — no duplicate rows and no growth — and they +rewrite it on every sync only where a lane has two trees with the same path, +which the older one keys as one; with none, an unreadable lane included, a sync +after the first writes nothing. A store written by #219's unlanded heads +(`f00ee58`, `ab853b1`) keeps their marker, keyed `:`, as a kept row while +that lane stays unreadable, and it clears at the first sync that reads the lane. +The source's own row carries a fingerprint of the +whole table as its compare-and-swap token and a sync count that only goes up. A +container with no `LANES_WORKSTATION` syncs nothing (exit 2): a source named for +a container's own id is one no restart finds again. **The nudge.** Every inventory write — `set-lane-state`, `set-lane-tree`, the lifecycle follow after a `STARTED`, `RESUMED`, `ENDED` or `RETIRED`, and a @@ -3628,7 +3657,9 @@ answer. Where the table is current, this workstation's rows are the same bytes either way — and a store only this workstation syncs into (SQLite, or a Postgres nobody else writes) prints exactly the source read, while a shared one adds the other workstations' rows among them. Every row is a last observation: the current truth of a lane is `lanes-edit.sh lane-reconcile -`, which reads the disk, and the footer says so. +`, which reads the disk — and the sidecars — of the workstation it runs +on and of no other, so the footer says to run it ON THE WORKSTATION the lane's +rows name (#173): Raven's listing of Eagle's rows is answered on Eagle. **What never reads it.** No act: not `lane-reconcile` — there is no `lane-reconcile --index`, by Amendment 14(b) — not `lane-start`, `lane-end`, diff --git a/lanes b/lanes index 690a46ce..d7eee152 100755 --- a/lanes +++ b/lanes @@ -365,7 +365,12 @@ EOF if [ "$wt_unread" != 0 ]; then say "$wt_unread lane(s) above have an inventory that is there and could not be read: that is NOT a lane with no worktrees." fi - say " the current truth of one lane: lanes-edit.sh lane-reconcile (it reads the disk; this listing decides nothing)" + # THE ACT IS NAMED WITH WHERE IT RUNS (opensoft/openRepoTools#173, Copilot's + # fourth review of #171). An `--index` listing holds every workstation that + # syncs into the store, so Raven lists Eagle's rows, and `lane-reconcile` + # reads only the sidecars and the disk of the workstation it is run on. The + # line is the same on both reads, so the two stay the same bytes. + say " the current truth of one lane: lanes-edit.sh lane-reconcile , run ON THE WORKSTATION its rows name (it reads that workstation's own sidecars and disk; this listing decides nothing)" exit 0 fi diff --git a/lanes-edit.sh b/lanes-edit.sh index acbd2a17..4b04260d 100755 --- a/lanes-edit.sh +++ b/lanes-edit.sh @@ -3189,7 +3189,14 @@ lane_log_events() { # indexed commit: no lines, exactly as a `git show` of a missing path. command grep -Fqx -- "$ll_rel" "$LANES_IDX_DIR/logs" 2>/dev/null || return 0 awk -F"$US" -v p="$ll_rel" '$10 == p' "$LANES_IDX_DIR/events" - awk -v p="unreadable: $ll_rel:" 'index($0, p) == 1' "$LANES_IDX_DIR/warn" >&2 2>/dev/null || : + # THAT LOG'S WARNINGS AND THE COUNT THAT CLOSES THEM (opensoft/openRepoTools#166 + # item 3): the source read runs LOG_AWK over this one file, whose END says + # how many lines it could not read. The export writes that line after each + # log's own warnings, so it is the line straight after this log's last one. + awk -v p="unreadable: $ll_rel:" ' + index($0, p) == 1 { print; mine = 1; next } + mine && /^[0-9]+ unreadable line[(]s[)] above/ { print } + { mine = 0 }' "$LANES_IDX_DIR/warn" >&2 2>/dev/null || : return 0 elif have_remote_ref; then git -C "$LANES_REPO" show "origin/$LANES_BRANCH:$ll_rel" 2>/dev/null | parse_log_stream "$ll_rel" diff --git a/lanes-index b/lanes-index index 2bb58ed2..99712937 100755 --- a/lanes-index +++ b/lanes-index @@ -38,16 +38,21 @@ worktree inventory, copied into the `worktrees` table. It is read through `lanes-edit.sh worktrees --all` - the sidecars, the lifecycle snapshot, `managed-projection`'s owner and the listing's liveness, in the helper that wrote them - and never through a parser of this file's own or a walk of the -disk. Its rows are keyed by (source, `:`) and versioned by the same -digest; a tree the inventory no longer names leaves in the same sync. Its +disk. Its rows are keyed by (source, `:`) - a second tree of one lane +with the same path `:`, and a lane's unread marker +`:unread` - and versioned by the same digest; a tree the inventory no +longer names leaves in the same sync. Its provenance moves only forward PER LANE: a sync that reads a lane's lifecycle generation below the one its rows were written at writes nothing for that lane -(`reconcile` takes the lane as it now stands, and says so), and the source's own +(`reconcile` takes the lane as it now stands, and says so); a lane whose +inventory is there and could not be read keeps its rows and gets one marker +row, with no path and a branch no git ref can be (#173), which `export +--worktrees` writes back as the source's own `UNREAD` line; and the source's own row carries a fingerprint of the whole table as its compare-and-swap token and a -sync generation that only counts up. `lanes-edit.sh` nudges `sync --source -inventory` after every inventory write. Nothing that acts reads this table -either: `lane-reconcile`, `lane-start`, `lane-end` and the sweep read the -sidecars and the disk. +sync generation that only counts up. +`lanes-edit.sh` nudges `sync --source inventory` after every inventory write. +Nothing that acts reads this table either: `lane-reconcile`, `lane-start`, +`lane-end` and the sweep read the sidecars and the disk. THE STORE (clause (c)): QA Postgres where this workstation's configuration names a `url=`, reached through `psql` and only then; otherwise one SQLite file @@ -71,6 +76,7 @@ schema - the reason is in `/why`) · 64 usage. from __future__ import annotations import datetime as _dt +import functools import hashlib import json import os @@ -726,7 +732,9 @@ TABLE_COLUMNS = { "job_summaries": ["lane", "job_id", "result_summary", "observed_utc"], # opensoft/openRepoTools#161: each lane's #97 worktree inventory under # `inventory:`, keyed by (source, lane, path) - the record id - # `:`. Amendment 14(e) left inventories to "a later word's"; #161 + # `:`, `:` for a second tree of one lane with + # the same path, and `:unread` for a lane's unread marker (#173). + # Amendment 14(e) left inventories to "a later word's"; #161 # is that word's issue. Created by schema 1 with these columns before a row # was written into it, so no store needed a new version for it. Written by # `run_inventory` and by nothing that reads the register. @@ -1121,9 +1129,42 @@ class SqliteStore: raise StoreUnusable("wiped", f"{self.path}: {exc}") def rows(self, table: str, source: str, columns: list[str]) -> list: + # A TABLE THAT CANNOT BE READ IS A STORE THAT CANNOT ANSWER, said as the + # provenance's reads say it (opensoft/openRepoTools#173, Copilot's + # fourth review of #171): a `worktrees` table dropped or damaged under + # valid provenance was a traceback, before `status` reached its + # `rows: unread` line and before `export` could write its `why`. + import sqlite3 cols = ", ".join(["record_id"] + columns) - return [dict(zip(["record_id"] + columns, r)) for r in self.conn.execute( - f"SELECT {cols} FROM {table} WHERE source = ?", (source,))] + try: + return [dict(zip(["record_id"] + columns, r)) for r in self.conn.execute( + f"SELECT {cols} FROM {table} WHERE source = ?", (source,))] + except sqlite3.Error as exc: + raise StoreUnusable("wiped", f"{self.path}: {exc}") + + def read_image(self, source: str, reads: list) -> tuple: + """`provenance(source)`, then `rows(table, source, columns)` for each + `(table, columns)` of `reads`, as ONE READ SNAPSHOT (opensoft/openRepoTools#166 + item 1). Each statement was its own snapshot, so a sync committing + between two of them made an image of two commits under a `meta` naming + the first. One read transaction holds the SHARED lock in the rollback + journal - a writer's commit waits the fraction of a second the export + takes - and in WAL mode it is a snapshot.""" + import sqlite3 + try: + self.conn.execute("BEGIN") + except sqlite3.Error as exc: + raise StoreUnusable("unreachable", f"{self.path}: {exc}") + try: + prov = self.provenance(source) + tables = [self.rows(table, source, columns) for table, columns in reads] + finally: + # A read transaction ends however its reads did; nothing was written. + try: + self.conn.execute("COMMIT") + except sqlite3.Error: + pass + return prov, tables def apply(self, source: str, old: dict | None, new_sha: str, upserts: dict, deletes: dict, generation: int | None = None) -> None: @@ -1284,15 +1325,20 @@ class PostgresStore: raise StoreUnusable("unreachable", err[-1]) return proc.stdout - def _query_many(self, queries: list[str]) -> list[list[list]]: + def _query_many(self, queries: list[str], snapshot: bool = False) -> list[list[list]]: """Several SELECTs in ONE connection, each answered as TEXT-format COPY rows; a marker line no COPY row can be (one field, a control - character) separates them.""" + character) separates them. With `snapshot`, in ONE read-only + REPEATABLE READ transaction: every SELECT reads the same snapshot.""" marker = "\x1eLANES-INDEX-NEXT" parts = [f"SET search_path TO \"{self.schema}\";"] + if snapshot: + parts.append("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY;") for query in queries: parts.append(f"COPY ({query}) TO STDOUT;") parts.append("\\echo '\\036LANES-INDEX-NEXT'") + if snapshot: + parts.append("COMMIT;") out = self._run("\n".join(parts) + "\n") results, current = [], [] for line in out.split("\n"): @@ -1338,14 +1384,8 @@ class PostgresStore: self.open_read() def provenance(self, source: str) -> dict | None: - res = self._query_many([ - f"SELECT commit_sha, generation, indexed_utc, schema_version FROM provenance " - f"WHERE source = {_pg_quote(source)}"]) - if not res or not res[0]: - return None - r = res[0][0] - return {"commit_sha": r[0], "generation": r[1], "indexed_utc": r[2], - "schema_version": r[3]} + res = self._query_many([self._provenance_query(source)]) + return self._provenance_of(res[0] if res else []) def all_provenance(self) -> list: res = self._query_many([ @@ -1355,11 +1395,31 @@ class PostgresStore: "schema_version": r[4]} for r in (res[0] if res else [])] def rows(self, table: str, source: str, columns: list[str]) -> list: - res = self._query_many([ - f"SELECT {', '.join(['record_id'] + columns)} FROM {table} " - f"WHERE source = {_pg_quote(source)}"]) + res = self._query_many([self._rows_query(table, source, columns)]) + return self._rows_of(res[0] if res else [], columns) + + @staticmethod + def _provenance_query(source: str) -> str: + return (f"SELECT commit_sha, generation, indexed_utc, schema_version FROM provenance " + f"WHERE source = {_pg_quote(source)}") + + @staticmethod + def _provenance_of(answer: list) -> dict | None: + if not answer: + return None + r = answer[0] + return {"commit_sha": r[0], "generation": r[1], "indexed_utc": r[2], + "schema_version": r[3]} + + @staticmethod + def _rows_query(table: str, source: str, columns: list[str]) -> str: + return (f"SELECT {', '.join(['record_id'] + columns)} FROM {table} " + f"WHERE source = {_pg_quote(source)}") + + @staticmethod + def _rows_of(answer: list, columns: list[str]) -> list: out = [] - for r in (res[0] if res else []): + for r in answer: rec = dict(zip(["record_id"] + columns, r)) for col in columns: if col in INTEGER_COLUMNS and rec[col] is not None: @@ -1367,6 +1427,19 @@ class PostgresStore: out.append(rec) return out + def read_image(self, source: str, reads: list) -> tuple: + """`SqliteStore.read_image`'s answer, ONE `psql` session and one + REPEATABLE READ READ ONLY transaction (opensoft/openRepoTools#166 item 1): + each `rows()` was a session of its own, and a snapshot of its own.""" + res = self._query_many([self._provenance_query(source)] + + [self._rows_query(table, source, columns) + for table, columns in reads], snapshot=True) + if len(res) != len(reads) + 1: + raise StoreUnusable("unreachable", f"psql answered {len(res)} of the " + f"{len(reads) + 1} reads of one export") + return self._provenance_of(res[0]), [self._rows_of(answer, columns) + for answer, (_table, columns) in zip(res[1:], reads)] + def apply(self, source: str, old: dict | None, new_sha: str, upserts: dict, deletes: dict, generation: int | None = None) -> None: now = utc_now() @@ -1705,9 +1778,15 @@ def _helper_run(args: list, timeout: float, what: str) -> subprocess.CompletedPr raise StoreUnusable("unread", f"`lanes-edit.sh {' '.join(args)}` could not be run: {exc}") +@functools.lru_cache(maxsize=None) def this_workstation() -> tuple[str, str]: """`lanes-edit.sh workstation`'s first two columns: the name every row of - this workstation is filed under, and where it came from.""" + this workstation is filed under, and where it came from. + + ASKED ONCE A PROCESS (the review of #219): the refusal `cmd_sync` makes + before the lock asks it, and the sync that then takes the lock asked again + for the same answer - one environment, one name. A read that failed raises, + and is not kept.""" proc = _helper_run(["workstation"], 60, "this workstation's name") first = (proc.stdout.splitlines() or [""])[0].split("\t") if proc.returncode != 0 or not first[0]: @@ -1765,6 +1844,43 @@ def tree_line(row: dict, provenance) -> str: "unpushed_count")] + [text(provenance)]) +#: THE BRANCH OF A LANE'S UNREAD MARKER (opensoft/openRepoTools#173): a +#: `worktrees` row with no path and a branch `unreadable inventory ()`, +#: saying the lane's inventory was there and could not be read at the last +#: sync, and why. A MARKER IS ITS KEY, `unread_key()` - `:` and a +#: US - and nothing else decides it (the review of #219 and its delta check): +#: no tree's key can carry a US after the colon but its own ``, while a +#: tree's path can be empty (a current-schema sidecar with no `path:`) and a +#: hand-made sidecar's branch can read like this one. +UNREAD_BRANCH = "unreadable inventory" + + +def is_unread_marker(row: dict) -> bool: + return row.get("record_id") == unread_key(row.get("lane") or "") + + +def unread_key(lane: str) -> str: + return f"{lane}:{US}unread" + + +def unread_record(workstation: str, lane: str, why: str) -> dict: + """The marker row of a lane whose inventory could not be read.""" + record = {col: None for col in TABLE_COLUMNS["worktrees"]} + record.update(workstation=workstation, lane=lane, path="", + branch=f"{UNREAD_BRANCH} ({why})", _provenance=None) + return record + + +def unread_line(row: dict) -> str: + """The `UNREAD` line a marker row came from - `UNREAD + `, as `lanes-edit.sh worktrees` prints it, so that `lanes --index + --worktrees` says NOT READ wherever the source read said it.""" + why = row.get("branch") or "" + if why.startswith(f"{UNREAD_BRANCH} (") and why.endswith(")"): + why = why[len(UNREAD_BRANCH) + 2:-1] + return US.join(["UNREAD", row.get("workstation") or "", row.get("lane") or "", why]) + + def inventory_digest(record: dict) -> str: """The register's digest, and the row's provenance with it: two observations of one tree filed under two generations are two versions.""" @@ -1791,9 +1907,10 @@ def lane_generation(lane: str) -> tuple: def collect_inventory() -> tuple[dict, dict]: - """Every tree the inventory names, keyed `:`, and every lane - whose inventory is there and could not be read - whose rows are KEPT, never - taken for a lane with no trees (Amendment 7(d)).""" + """Every tree the inventory names, keyed `:` (a second tree of + one lane with the same path `:`), and every lane whose + inventory is there and could not be read - whose rows are KEPT, never taken + for a lane with no trees (Amendment 7(d)).""" proc = _helper_run(["worktrees", "--all"], 600, "the worktree inventory") if proc.returncode == 8: return {}, {} @@ -1814,7 +1931,20 @@ def collect_inventory() -> tuple[dict, dict]: f"indexer does not read ({len(fields)} fields): {line[:120]!r} - " f"is the helper on PATH another version?") record = tree_record(fields) - trees[f"{record['lane']}:{record['path']}"] = record + # TWO TREES OF ONE LANE WITH ONE PATH ARE TWO ROWS (the review of #219): + # a current-schema sidecar with no `path:` is a tree whose path is + # EMPTY, and a second one replaced the first here. A later one is keyed + # with its place among them, after a US, which no path in a + # US-separated line can carry. The read is sorted, so a place holds + # while those trees keep their order; a change that reorders them - a + # renamed branch, say - moves both keys, and that sync upserts both. + key = f"{record['lane']}:{record['path']}" + if key in trees: + n = 2 + while f"{key}{US}{n}" in trees: + n += 1 + key = f"{key}{US}{n}" + trees[key] = record return trees, unread @@ -1947,11 +2077,30 @@ def run_inventory(mode: str, requested: str | None, dry_run: bool) -> int: f"{stored_gen[lane]} to {gen} (a snapshot moved aside or replaced); " f"reconcile takes it as it now stands") + # THE UNREAD STATE IS KEPT AS WELL AS THE ROWS (opensoft/openRepoTools#173, + # Copilot's third review of #171). A lane held because its inventory could + # not be read gets ONE marker row, which `export --worktrees` writes back as + # the source's own `UNREAD` line. Without it a FIRST sync that could not + # read a lane committed usable provenance with nothing for it, and the + # index answered "no worktree is inventoried" where the source says NOT + # READ; and rows held over from an earlier sync read as freshly read - so + # the marker says they are kept. It leaves with the first sync that reads + # its lane, a lane held for its generation included: that lane was read. + workstation = source[len(INVENTORY_PREFIX):] + markers: dict = {} + for lane, why in unread.items(): + rows_kept = sum(1 for row in stored.values() + if row["lane"] == lane and not is_unread_marker(row)) + if rows_kept: + why = (f"{why} - the {rows_kept} row(s) listed for it are kept from the last sync " + f"that read it") + markers[unread_key(lane)] = unread_record(workstation, lane, why) + fill_prs(trees, stored, mode, held) upserts, deletes, kept = {}, [], 0 - for rec_id, rec in trees.items(): + for rec_id, rec in list(trees.items()) + list(markers.items()): rec["digest"] = inventory_digest(rec) - if rec["lane"] in held: + if rec["lane"] in held and rec_id not in markers: continue row = stored.get(rec_id) if row is not None: @@ -1966,10 +2115,14 @@ def run_inventory(mode: str, requested: str | None, dry_run: bool) -> int: continue upserts[rec_id] = rec for rec_id, row in stored.items(): - if row["lane"] not in held and rec_id not in trees: + if rec_id in trees or rec_id in markers: + continue + if row["lane"] not in held or is_unread_marker(row): deletes.append(rec_id) after = {rid: rec["digest"] for rid, rec in trees.items() if rec["lane"] not in held} - after.update({rid: row["digest"] for rid, row in stored.items() if row["lane"] in held}) + after.update({rid: row["digest"] for rid, row in stored.items() + if row["lane"] in held and not is_unread_marker(row)}) + after.update({rid: rec["digest"] for rid, rec in markers.items()}) fingerprint = hashlib.sha256("\n".join( f"{rid}\t{digest}" for rid, digest in sorted(after.items())).encode("utf-8")).hexdigest() @@ -2033,6 +2186,20 @@ def cmd_sync(requested: str | None) -> int: return 0 kind = "inventory" if is_inventory(requested) else "register" asked = {kind: requested} + # A SOURCE IS REFUSED BEFORE THE LOCK IS ASKED FOR (opensoft/openRepoTools#173, + # Copilot's fourth review of #171). Behind a held lock this sync only + # leaves a mark of its KIND, and the holder syncs THIS workstation's source + # for it - so `--source inventory:Raven` on Eagle, `--source inventory` in + # a container with no name, or another register, exited 0 there instead of + # 2. A refusal leaves through `main`; a helper that cannot answer is the + # sync's own to say and record, as it was. + try: + if kind == "inventory": + inventory_source(requested) + elif requested: + resolve_source(requested, workspace_root()) + except StoreUnusable: + pass lock = SyncLock() # THE HANDSHAKE, BOTH HALVES (Codex on #164). A sync that finds the lock # held leaves its mark and then tries ONCE MORE: if that fails, somebody @@ -2196,8 +2363,14 @@ def cmd_status() -> int: say(f" provenance: inventory@{fp[:12]} (sync {prov.get('generation')}, indexed " f"{prov['indexed_utc']}, schema {prov['schema_version']})") try: - count = len(store.rows("worktrees", name, ["lane"])) - say(f" rows: {count} worktree(s)") + wt_rows = store.rows("worktrees", name, ["lane"]) + count = sum(1 for row in wt_rows if not is_unread_marker(row)) + # A LANE'S UNREAD MARKER IS NOT A WORKTREE (#173), and is said + # as what it is. + unread = len(wt_rows) - count + say(f" rows: {count} worktree(s)" + ( + f", and {unread} lane(s) whose inventory could not be read at the " + f"last sync" if unread else "")) except StoreUnusable as exc: say(f" rows: unread - {exc.kind}: {exc.detail}") rc = 1 @@ -2252,16 +2425,19 @@ def cmd_export(out_dir: str, requested: str | None, worktrees: bool = False) -> source = resolve_source(requested, root) store = open_store(cfg, "reader") store.open_read() - prov = store.provenance(source) + # ONE COMMIT'S IMAGE (opensoft/openRepoTools#166 item 1): the provenance + # and every table in one read snapshot, or a sync committing between two + # reads writes rows of two commits under a `meta` naming the first. + prov, (file_rows, lanes, reg_lines, aliases, log_rows) = store.read_image(source, [ + ("source_files", TABLE_COLUMNS["source_files"]), + ("lanes", ["row_file", "row_ordinal", "row_text"]), + ("register_lines", ["ordinal", "raw"]), + ("lane_aliases", ["ordinal", "raw"]), + ("log_lines", [c for c in TABLE_COLUMNS["log_lines"] + if c not in ("payload_fields", "raw")])]) if prov is None or not prov.get("commit_sha"): return unusable("wiped", f"no provenance for {source} in the {cfg['store']} store") - files = {r["record_id"]: r for r in store.rows("source_files", source, - TABLE_COLUMNS["source_files"])} - lanes = store.rows("lanes", source, ["row_file", "row_ordinal", "row_text"]) - reg_lines = store.rows("register_lines", source, ["ordinal", "raw"]) - aliases = store.rows("lane_aliases", source, ["ordinal", "raw"]) - log_rows = store.rows("log_lines", source, [c for c in TABLE_COLUMNS["log_lines"] - if c not in ("payload_fields", "raw")]) + files = {r["record_id"]: r for r in file_rows} except Refusal as exc: unusable("refused", str(exc)) return 2 @@ -2327,7 +2503,10 @@ def _export_worktrees(out_dir: str, cfg: dict, unusable) -> int: for prov in provs: for row in store.rows("worktrees", prov["source"], TABLE_COLUMNS["worktrees"] + ["provenance"]): - lines.append(tree_line(row, row.get("provenance"))) + # A LANE'S UNREAD MARKER (#173) is written back as the `UNREAD` + # line it was read from. + lines.append(unread_line(row) if is_unread_marker(row) + else tree_line(row, row.get("provenance"))) except Refusal as exc: unusable("refused", str(exc)) return 2 diff --git a/tests/test_lanes_index.py b/tests/test_lanes_index.py index e0c11d9c..4bbe278e 100644 --- a/tests/test_lanes_index.py +++ b/tests/test_lanes_index.py @@ -508,6 +508,61 @@ def test_the_compare_and_swap_refuses_a_provenance_that_moved(estate): assert dump(estate.sqlite, with_utc=True) == before +def _export_image(out: Path) -> dict: + return {p.name: p.read_text(encoding="utf-8") for p in sorted(out.iterdir())} + + +@pytest.mark.parametrize("journal,wait", [("wal", 60.0), ("delete", 5.0)]) +def test_an_export_is_one_commits_image_when_a_sync_lands_mid_read(estate, monkeypatch, + journal, wait): + """ONE READ SNAPSHOT (opensoft/openRepoTools#166 item 1): `export` read the + provenance and each table as statements of their own, so a sync landing + between two of them wrote rows of two commits under a `meta` naming the + first. A sync is landed after the export's FIRST table - in WAL mode it + commits at once, and the export keeps its snapshot; in the rollback + journal it waits for the export's read lock (`wait` bounds how long this + case waits for it to finish first) and lands after - and the export is the + old commit's image or the new one, never a mix of the two.""" + assert estate.index("sync").returncode == 0 + conn = sqlite3.connect(str(estate.sqlite)) + assert conn.execute(f"PRAGMA journal_mode={journal}").fetchone()[0] == journal + conn.close() + old = estate.root / "old" + old.mkdir() + assert estate.index("export", "--out", old).returncode == 0 + assert estate.note("repoA-2", "a note a sync lands mid-export").returncode == 0 + _as_estate(monkeypatch, estate) + mod = load_indexer() + real_rows = mod.SqliteStore.rows + syncs: list = [] + + def rows_then_a_sync(self, table, source, columns): + got = real_rows(self, table, source, columns) + if not syncs: + syncs.append(subprocess.Popen( + [sys.executable, str(INDEXER), "sync"], cwd=str(estate.root), + env=dict(estate.env, LANES_EDIT=str(LANES_EDIT)), stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)) + try: + syncs[0].wait(timeout=wait) + except subprocess.TimeoutExpired: + pass # its commit is waiting on this export's read lock + return got + + monkeypatch.setattr(mod.SqliteStore, "rows", rows_then_a_sync) + got = estate.root / "got" + got.mkdir() + assert mod.cmd_export(str(got), None) == 0 + out, err = syncs[0].communicate(timeout=60) + assert syncs[0].returncode == 0 and "upserted" in out, (out, err) + new = estate.root / "new" + new.mkdir() + assert estate.index("export", "--out", new).returncode == 0 + images = {"old": _export_image(old), "new": _export_image(new)} + assert images["old"] != images["new"], "the sync landed nothing: vacuous" + assert _export_image(got) in images.values(), "the export mixed two commits' rows" + + # ============================================================ the hung indexer STUB = """#!/bin/sh @@ -702,15 +757,31 @@ def test_the_index_read_is_byte_identical_to_the_source_read(estate): """`--index` ANSWERS EXACTLY WHAT THE SOURCES ANSWER where the index is current — the listing's AGE column, which is the clock's, the one permitted difference — and says so on stderr. Then a write the index has - not followed: the lag is SAID, never refused.""" + not followed: the lag is SAID, never refused. + + STDERR TOO (opensoft/openRepoTools#166 item 3): `history repoA-1` prints + LOG_AWK's warnings for its log's nine unreadable lines AND the count that + closes them, and `history repoA-1 --index` said the nine without the + count. Two lines are the flag's own and are set aside on both sides: the + `read:` line, and `history`'s no-fetch note where the index answered - the + read is the index's, so that arm fetches nothing in front of it.""" assert estate.index("sync").returncode == 0 sha12 = estate.tip()[:12] + + def told(stderr: str) -> str: + return "".join(line for line in stderr.splitlines(keepends=True) + if not line.startswith("read: ") + and "not fetching; reading origin/" not in line) + for argv in READS: src, idx = _pair(estate, argv) assert idx.returncode == src.returncode, (argv, idx.stderr) assert AGE.sub("", idx.stdout) == AGE.sub("", src.stdout), argv assert f"read: index (sqlite) at register@{sha12}; 0 commits behind" in idx.stderr, (argv, idx.stderr) assert "read: " not in src.stderr + assert told(idx.stderr) == told(src.stderr), (argv, idx.stderr, src.stderr) + src, _idx = _pair(estate, ("bash", LANES_EDIT, "history", "repoA-1")) + assert "9 unreadable line(s) above" in src.stderr, "the fixture lost its unreadable lines: vacuous" assert estate.note("repoA-1", "a note the index has not seen").returncode == 0 src, idx = _pair(estate, ("bash", LANES_EDIT, "history", "repoA-1")) assert idx.returncode == 0 @@ -990,16 +1061,77 @@ def test_a_named_configuration_that_is_not_there_and_a_store_in_a_work_tree_refu if vfile and "CREATE TABLE IF NOT EXISTS" in stdin: with open(vfile, "w") as handle: handle.write("1\n") + + +def echo(text): + # `\echo` AS psql's OWN LEXER READS IT (opensoft/openRepoTools#166 item 4; + # psqlscanslash.l's `xslashquote`): words split on blanks, a quoted run + # taken whole, and inside the quotes `''`, `\n \t \b \r \f`, octal + # `\ooo` and hex `\xhh` decoded - so the marker the indexer sends is the + # marker its parser reads back, and a wrong escape is wrong here too. + words, word, quoted, i = [], "", False, 0 + while i < len(text): + ch = text[i] + if not quoted and ch in " \t": + if word: + words.append(word) + word = "" + i += 1 + elif ch == "'" and quoted and text[i + 1:i + 2] == "'": + word, i = word + "'", i + 2 + elif ch == "'": + quoted, i = not quoted, i + 1 + elif quoted and ch == "\\" and i + 1 < len(text): + nxt = text[i + 1] + octal = len(text[i + 1:i + 4]) - len(text[i + 1:i + 4].lstrip("01234567")) + hexes = len(text[i + 2:i + 4]) - len(text[i + 2:i + 4].lstrip("0123456789abcdefABCDEF")) + if nxt in "ntbrf": + word, i = word + {"n": "\n", "t": "\t", "b": "\b", "r": "\r", "f": "\f"}[nxt], i + 2 + elif octal: + word, i = word + chr(int(text[i + 1:i + 1 + octal], 8)), i + 1 + octal + elif nxt == "x" and hexes: + word, i = word + chr(int(text[i + 2:i + 2 + hexes], 16)), i + 2 + hexes + else: + word, i = word + nxt, i + 2 + else: + word, i = word + ch, i + 1 + if word: + words.append(word) + return " ".join(words) + + for line in stdin.splitlines(): if line.startswith("COPY (SELECT version"): if version: print(version) elif line.startswith("\\echo"): - print("\x1eLANES-INDEX-NEXT") + print(echo(line[len("\\echo"):])) sys.exit(0) """ +def test_the_fake_psql_decodes_echo_as_psql_does(tmp_path): + """THE FAKE DOES NOT HARD-CODE THE MARKER (opensoft/openRepoTools#166 item + 4): it prints what psql would for each `\\echo`, so `\\036` arrives as byte + 0x1E only because psql's escapes say so - and an argument that is not the + marker is not the marker.""" + fake = tmp_path / "psql" + fake.write_text(FAKE_PSQL) + fake.chmod(0o755) + script = "\n".join([ + "\\echo '\\036LANES-INDEX-NEXT'", + "\\echo 'it''s' two words", + "\\echo 'a\\tb\\x41\\101\\q'", + "\\echo plain", + ""]) + res = subprocess.run([sys.executable, str(fake), "-d", "postgresql://x@db.invalid/qa"], + input=script, capture_output=True, text=True, check=False, + env=dict(os.environ, FAKE_PSQL_LOG=str(tmp_path / "log"))) + assert res.returncode == 0, res.stderr + assert res.stdout.split("\n") == ["\x1eLANES-INDEX-NEXT", "it's two words", "a\tbAAq", + "plain", ""], res.stdout + + def test_postgres_takes_the_writer_to_write_and_the_reader_to_read(estate): """TWO ROLES (14(c)): `sync` connects as the `url=` WRITER and `export` and `status` as the `reader_url=` READER, each with its own 0600 password @@ -1044,6 +1176,36 @@ def test_postgres_takes_the_writer_to_write_and_the_reader_to_read(estate): assert "postgres" in st.stdout +def test_a_postgres_export_is_one_repeatable_read_transaction(estate): + """ONE READ SNAPSHOT ON POSTGRES TOO (opensoft/openRepoTools#166 item 1): + each `rows()` was a `psql` session of its own. The export's provenance and + its five tables now go in ONE `psql` call, inside `BEGIN ISOLATION LEVEL + REPEATABLE READ READ ONLY` ... `COMMIT`, which is one snapshot.""" + fake = estate.root / "pgfake" + fake.mkdir() + (fake / "psql").write_text(FAKE_PSQL) + (fake / "psql").chmod(0o755) + log = estate.root / "psql.jsonl" + estate.write_config("url=postgresql://lanes_writer@db.invalid/qa\n" + "reader_url=postgresql://lanes_reader@db.invalid/qa\n") + out = estate.root / "exp" + out.mkdir() + exp = estate.index("export", "--out", out, + env={"PATH": estate.path_with(fake), "FAKE_PSQL_LOG": str(log)}) + # the fake holds no provenance, so the answer is `wiped` - read in the batch + assert exp.returncode == 3 and (out / "why").read_text().startswith("wiped"), exp.stderr + calls = [json.loads(l)["stdin"] for l in log.read_text().splitlines()] + reads = [c for c in calls if "FROM provenance" in c] + assert len(reads) == 1, calls + script = reads[0] + assert "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY;" in script, script + begin = script.index("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY;") + end = script.rindex("COMMIT;") + assert begin < script.index("FROM provenance WHERE") < end, script + for table in ("source_files", "lanes", "register_lines", "lane_aliases", "log_lines"): + assert begin < script.index(f"FROM {table} WHERE") < end, (table, script) + + def test_a_read_never_connects_with_the_writers_credential(estate): """THE READER OR NOTHING (14(c); Copilot round 1 on #164): a Postgres configuration with a writer `url=` and no `reader_url=` REFUSES the reads @@ -1828,6 +1990,12 @@ def test_the_index_read_of_the_worktrees_is_the_source_read(inv): assert re.search(r"read: index \(sqlite\) at inventory:Eagle synced " + UTC.pattern, idx.stderr), idx.stderr assert "read: " not in src.stderr + # THE FOOTER NAMES WHERE ITS ACT RUNS (opensoft/openRepoTools#173, + # Copilot's fourth review of #171): an index read lists other + # workstations' rows, and `lane-reconcile` reads only the disk and + # the sidecars of the workstation it runs on. + assert "lanes-edit.sh lane-reconcile , run ON THE WORKSTATION its rows name" \ + in idx.stdout, idx.stdout (inv.run_tree / "unseen.txt").write_text("the index has not followed this\n") inv.ok("set-lane-tree", "repoA-1", str(inv.run_tree), "--checkout", str(inv.repo)) src = _wt(inv, "repoA-1") @@ -2081,6 +2249,63 @@ def test_the_inventory_source_is_this_workstations_alone(inv): assert both.returncode == 64 +def test_a_source_is_refused_before_the_lock_is_asked_for(estate): + """REFUSED WITH THE LOCK HELD TOO (opensoft/openRepoTools#173, Copilot's + fourth review of #171): a sync that finds the lock held leaves a mark for + its source and exits 0, and the holder then syncs THIS workstation's + source for it. So another workstation's inventory, an inventory in a + container with no name, and another register are refused before the lock + is asked for - exit 2, and no mark.""" + import fcntl + state = estate.state / "openRepoTools" + state.mkdir(parents=True, exist_ok=True) + with open(state / "lanes-index.lock", "a+") as handle: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + other = estate.index("sync", "--source", "inventory:Raven") + nameless = estate.index("sync", "--source", "inventory", + env={"LANES_WORKSTATION": None, "LANES_IN_CONTAINER": "1"}) + register = estate.index("sync", "--source", "register:someone/else") + assert not list(state.glob("lanes-index.rerun*")), "a refused source left a mark" + own = estate.index("sync", "--source", "inventory") + assert other.returncode == 2, (other.stdout, other.stderr) + assert "this workstation's inventory is inventory:Eagle, not inventory:Raven" in other.stderr + assert nameless.returncode == 2 and "LANES_WORKSTATION" in nameless.stderr, nameless.stderr + assert register.returncode == 2, (register.stdout, register.stderr) + assert "not register:someone/else" in register.stderr + # NOT VACUOUS: this workstation's own source, the lock still held, marks + assert own.returncode == 0 and "another sync holds" in own.stdout, own.stdout + assert [p.name for p in state.glob("lanes-index.rerun*")] == ["lanes-index.rerun-inventory"] + + +def test_a_damaged_table_under_valid_provenance_is_unusable_not_a_traceback(estate): + """`rows()` SPEAKS `StoreUnusable` (opensoft/openRepoTools#173, Copilot's + fourth review of #171): with the inventory's provenance valid and its + `worktrees` table gone, `status` says `rows: unread` rather than printing a + traceback, and `export --worktrees` leaves by the unusable path - `why`, + exit 3 - so the read flag says why it read the sources.""" + assert estate.index("sync").returncode == 0 + conn = sqlite3.connect(str(estate.sqlite)) + conn.execute("INSERT INTO provenance (source, commit_sha, generation, indexed_utc, " + "schema_version) VALUES ('inventory:Eagle', ?, 1, '2026-10-13T00:00:00Z', 1)", + ("a" * 64,)) + conn.execute("DROP TABLE worktrees") + conn.commit() + conn.close() + st = estate.index("status") + assert "Traceback" not in st.stderr, st.stderr + assert st.returncode == 1, (st.stdout, st.stderr) + assert re.search(r"rows: +unread - wiped: .*no such table: worktrees", st.stdout), st.stdout + out = estate.root / "exp" + out.mkdir() + exp = estate.index("export", "--out", out, "--worktrees") + assert "Traceback" not in exp.stderr, exp.stderr + assert exp.returncode == 3, (exp.returncode, exp.stderr) + assert (out / "why").read_text().startswith("wiped: ") + assert "no such table: worktrees" in (out / "why").read_text() + listed = estate.run("bash", LANES, "--worktrees", "--index") + assert "read: sources (index wiped: " in listed.stderr, listed.stderr + + def test_a_lane_bound_elsewhere_has_no_liveness_pronounced_here(inv): """AMENDMENT 18(b) FOR `writer_live`: a lane whose last STARTED/RESUMED bound it on another host cannot be pronounced not-live from here - its session @@ -2136,6 +2361,134 @@ def test_an_inventory_that_cannot_be_read_keeps_its_rows(inv): listed = _wt(inv) assert "inventory NOT READ" in listed.stdout, listed.stdout assert "could not be read: that is NOT a lane with no worktrees" in listed.stdout + # AND THE INDEX SAYS THE ROWS ARE HELD (opensoft/openRepoTools#173): the + # two it kept are listed beside the NOT READ line the sidecars give, + # which says they are an earlier sync's - not rows freshly read. + idx = _wt(inv, "repoA-1", "--index") + assert idx.returncode == 0 and "read: index (sqlite)" in idx.stderr, idx.stderr + assert str(inv.run_tree) in idx.stdout and str(inv.dirty_tree) in idx.stdout, idx.stdout + assert re.search(r"inventory NOT READ: its trees directory \S+ is there and could not be " + r"listed - the 2 row\(s\) listed for it are kept from the last sync that " + r"read it", idx.stdout), idx.stdout + finally: + trees.chmod(0o755) + # READ AGAIN, THE MARK LEAVES: the index read is the source read once more. + healed = inv.sync() + assert healed.returncode == 0 and "0 upserted, 1 removed" in healed.stdout, healed.stdout + src, idx = _wt(inv, "repoA-1"), _wt(inv, "repoA-1", "--index") + assert (idx.returncode, idx.stdout) == (src.returncode, src.stdout), idx.stdout + assert "NOT READ" not in idx.stdout + + +def test_a_tree_with_no_path_is_a_tree_and_not_an_unread_marker(inv): + """A TREE'S PATH CAN BE EMPTY (the review of #219 at ab853b1): a + current-schema sidecar with a `tree:` and no `path:` is a `TREE` line + whose path is empty, and a marker read from the empty path alone exported + it as `UNREAD` and counted its lane as not read. The marker is a row with + no path AND the branch `unreadable inventory (...)`, which no git ref can + be; it has a key of its own, so holding the lane never writes it over such + a tree; and two trees of one lane with one path are two rows, as they are + two lines of the source read.""" + trees = inv.lsr / "repoM-1" / "trees" + for tid, branch in (("c1-nopath-x", "feat/x"), ("c2-nopath-y", "feat/y")): + (trees / f"{tid}.yaml").write_text("\n".join([ + "schema: 1", "lane: repoM-1", f"tree: {tid}", "checkout: unknown", + f"branch: {branch}", "head: " + "3" * 40, "upstream: none", "dirty: 0", + "unpushed: 0", "writer: none", "generation: 0", "operation: none", + "observed: 2026-10-04T00:00:00Z", ""])) + res = inv.sync() + assert res.returncode == 0, res.stderr + src, idx = _wt(inv, "repoM-1"), _wt(inv, "repoM-1", "--index") + assert "read: index (sqlite)" in idx.stderr, idx.stderr + assert "feat/x" in src.stdout and "feat/y" in src.stdout, src.stdout + assert (idx.returncode, idx.stdout) == (src.returncode, src.stdout), idx.stdout + assert "NOT READ" not in idx.stdout + st = inv.e.index("status") + assert "rows: 7 worktree(s)\n" in st.stdout, st.stdout + # HELD OVER, ALL THREE STAY, the two with no path among them. + trees.chmod(0) + try: + if os.access(trees, os.R_OK): + pytest.skip("this account reads a mode-000 directory (root): nothing to prove here") + assert inv.sync().returncode == 0 + idx = _wt(inv, "repoM-1", "--index") + assert "feat/x" in idx.stdout and "feat/y" in idx.stdout, idx.stdout + assert "the 3 row(s) listed for it are kept from the last sync" in idx.stdout, idx.stdout + finally: + trees.chmod(0o755) + healed = inv.sync() + assert healed.returncode == 0 and "0 upserted, 1 removed" in healed.stdout, healed.stdout + src, idx = _wt(inv, "repoM-1"), _wt(inv, "repoM-1", "--index") + assert (idx.returncode, idx.stdout) == (src.returncode, src.stdout), idx.stdout + + +def test_a_tree_whose_branch_reads_like_a_marker_is_a_tree(inv): + """THE MARKER IS ITS KEY (the delta check of #219 at 5bf29d9): a hand-made + sidecar with no `path:` and a branch that begins `unreadable inventory (` + is a tree no git ref names, and still a tree - exported as its `TREE` + line and counted as a worktree - because a marker is the row keyed + `unread_key()`, which no tree's key can be.""" + tid = "c3-forged-branch" + (inv.lsr / "repoM-1" / "trees" / f"{tid}.yaml").write_text("\n".join([ + "schema: 1", "lane: repoM-1", f"tree: {tid}", "checkout: unknown", + "branch: unreadable inventory (forged by hand)", "head: " + "4" * 40, + "upstream: none", "dirty: 0", "unpushed: 0", "writer: none", "generation: 0", + "operation: none", "observed: 2026-10-04T00:00:00Z", ""])) + res = inv.sync() + assert res.returncode == 0, res.stderr + src, idx = _wt(inv, "repoM-1"), _wt(inv, "repoM-1", "--index") + assert "read: index (sqlite)" in idx.stderr, idx.stderr + assert "unreadable inventory (forged by hand)" in src.stdout, src.stdout + assert (idx.returncode, idx.stdout) == (src.returncode, src.stdout), idx.stdout + assert "NOT READ" not in idx.stdout + st = inv.e.index("status") + assert "rows: 6 worktree(s)\n" in st.stdout, st.stdout + + +def test_an_inventory_sync_asks_for_the_workstation_once(estate): + """ONE `workstation` READ A SYNC (the review of #219 at ab853b1): the + refusal before the lock asks `lanes-edit.sh workstation`, and a sync that + then takes the lock asked a second time for the same answer. It is the + process's own and is asked once.""" + calls = estate.root / "helper-calls" + wrapper = estate.root / "lanes-edit-counting" + wrapper.write_text(f"#!/bin/sh\nprintf '%s\\n' \"$1\" >> {calls}\n" + f"exec bash {LANES_EDIT} \"$@\"\n") + wrapper.chmod(0o755) + res = estate.index("sync", "--source", "inventory", env={"LANES_EDIT": str(wrapper)}) + assert res.returncode == 0, (res.stdout, res.stderr) + asked = calls.read_text().splitlines() + assert "worktrees" in asked, asked # not vacuous: the sync took the lock and read + assert asked.count("workstation") == 1, asked + + +def test_a_first_sync_that_cannot_read_a_lane_says_not_read_in_the_index(inv): + """A FIRST SYNC IS NOT AN EMPTY ANSWER (opensoft/openRepoTools#173): a lane + whose `trees` directory is there and cannot be listed at the FIRST sync has + no rows to keep, and the sync committed usable provenance with nothing for + it - so `lanes --index --worktrees ` answered "no worktree is + inventoried" where the sidecars say NOT READ. The lane's marker row is + exported as the source's own `UNREAD` line, so the two reads are the same + bytes; and a replay of the same read writes nothing.""" + trees = inv.lsr / "repoA-1" / "trees" + trees.chmod(0) + try: + if os.access(trees, os.R_OK): + pytest.skip("this account reads a mode-000 directory (root): nothing to prove here") + res = inv.sync() + assert res.returncode == 0, res.stderr + for args in (("repoA-1",), ()): + src = _wt(inv, *args) + idx = _wt(inv, *args, "--index") + assert "read: index (sqlite)" in idx.stderr, (args, idx.stderr) + assert "no worktree is inventoried" not in idx.stdout, (args, idx.stdout) + assert (idx.returncode, idx.stdout) == (src.returncode, src.stdout), (args, idx.stdout) + assert "inventory NOT READ" in idx.stdout, (args, idx.stdout) + st = inv.e.index("status") + assert "rows: 3 worktree(s), and 1 lane(s) whose inventory could not be read " \ + "at the last sync" in st.stdout, st.stdout + again = inv.sync() + assert again.returncode == 0 and "nothing to write" in again.stdout, again.stdout finally: trees.chmod(0o755)