From f39d83350726a12cd96b6c61a48aa8823ec42b0e Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:05:11 +0000 Subject: [PATCH 01/11] Add lanes-index: the derived index of the lane register Amendment 14 (ratified by Brett Heap 2026-10-05, "ratify 48") keeps the register's truth where it is and adds a DERIVED index of it, written behind the sources by the indexer alone and read only by tooling that decides nothing. This is that indexer: `sync`, `reconcile [--dry-run]`, `status`, and `export`, the read flags' own reader. It reads `origin/` of the workspace checkout at one commit, never the working tree, and upserts records keyed by (source, record id) and versioned by a content digest, so a replay changes nothing. The source's provenance is compared and swapped in the same transaction as the rows, and a commit that does not descend from the indexed one writes nothing. `reconcile` compares stored rows by content, so a row changed by hand is found; on a wiped index it is the rebuild. The store is SQLite under XDG_STATE_HOME, or QA Postgres through `psql` only where `url=` is configured, with a writer and a SELECT-only reader credential. The configuration is refused when group- or world-readable, inside a git work tree or under ~/.agents/, or when a URL carries a password. The owner column is `managed-projection`'s answer (0/8/1); the indexer carries no parser of the marker. The ledger half's tables and #161's `worktrees` table are created empty. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lanes-index | 1844 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1844 insertions(+) create mode 100755 lanes-index diff --git a/lanes-index b/lanes-index new file mode 100755 index 0000000..e6d480e --- /dev/null +++ b/lanes-index @@ -0,0 +1,1844 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""lanes-index - the DERIVED INDEX of the lane register (lane-collision-protocol +Amendment 14, ratified by Brett Heap 2026-10-05, verbatim "ratify 48"). + + lanes-index sync [--source ] write behind what LANDED + lanes-index reconcile [--source ] [--dry-run] read the source whole + lanes-index status per source: store, lag, errors + lanes-index export --out [--source ] the read flags' reader + +THE REGISTER'S TRUTH STAYS WHERE IT IS (clause (a)). The workspace repository's +`lanes/LANES.md`, `lanes/log/*.md`, `lanes/aliases.tsv` and `lanes/archive/` +are the record, written only by `lanes-edit.sh`. This file keeps a COPY of what +they say, in tables a question across lanes can be asked of, and makes nothing +else true: where the index and the source disagree, the source is right and the +index is behind. + +NOTHING THAT ACTS READS IT (clause (b)). No claim, release, log line, guard, +hook, launch, row writer, sweep or migration reads this store or waits on it. +`lanes-edit.sh` starts `lanes-index sync` DETACHED after a push has landed and +its mutex is released - stdin closed, never waited, its status never read - and +two read-only words consult it behind a flag typed for that invocation: +`lanes ... --index` and `lanes-edit.sh history ... --index`, through `export`. + +WRITE-BEHIND, BY THE INDEXER ALONE, NEVER BACK (clause (d)). `sync` reads the +register as `origin/` of the workspace checkout stands - what the push +left, never the working tree - and upserts each record keyed by (source, record +id) and versioned by a digest of its content: an upsert bearing the stored +digest changes nothing, so a replay or a crash is harmless. The source's +indexed commit is compared and swapped IN THE SAME TRANSACTION as the rows it +covers, and a sync from a commit that does not descend from it writes nothing. +Nothing in any source is ever written from here. + +THE STORE (clause (c)): QA Postgres where this workstation's configuration +names a `url=`, reached through `psql` and only then; otherwise one SQLite file +on local disk. The configuration is +`${XDG_CONFIG_HOME:-$HOME/.config}/openRepoTools/lanes-index.conf`, or the file +`$LANES_INDEX_CONFIG` names: `store=`, `url=`, `passfile=`, `sqlite=`, and for +the SELECT-only role `reader_url=`/`reader_passfile=`, and `schema=`. It is +refused when it is group- or world-readable, inside a git work tree or under +`~/.agents/`, and when a URL carries a password: the secret lives only in the +0600 libpq password file `passfile=` names. + +Exit codes: 0 done · 1 a read or a write failed (nothing half-written) · 2 a +refusal (the configuration, a `--source` that is not this workstation's) · 3 +`export` only: the index cannot answer (missing, unreachable, wiped, unknown +schema - the reason is in `/why`) · 64 usage. +""" + +from __future__ import annotations + +import datetime as _dt +import hashlib +import json +import os +import re +import shutil +import stat +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor + +PROG = "lanes-index" +VERSION = "1" + +#: THE SCHEMA THIS READER AND WRITER KNOW. A store carrying another number is +#: never written by this file and never read by `export`: an older indexer +#: overwriting a newer schema's rows, or a reader guessing at them, is a second +#: way for the index to be wrong. +SCHEMA_VERSION = 1 + +US = "\x1f" +LANES_DIR = "lanes/" +REGISTER_PATH = "lanes/LANES.md" +ALIASES_PATH = "lanes/aliases.tsv" +ARCHIVE_PATH = "lanes/archive/LANES-retired.md" +LOG_PREFIX = "lanes/log/" + +UUID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") +UUID_FULL_RE = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$") +UTC_RE = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z") +SHA_RE = re.compile(r"^[0-9a-f]{40}$") +SCHEMA_NAME_RE = re.compile(r"^[a-z_][a-z0-9_]{0,62}$") +OPEN_VERBS = ("CLAIMED", "TAKEOVER", "OPENED", "LANDING", "WITHDRAWN") + + +class Refusal(Exception): + """A refusal: exit 2, nothing read and nothing written.""" + + +class StoreUnusable(Exception): + """The store cannot answer: missing, unreachable, wiped or another schema. + + `kind` is the first word of the `read: sources (index )` line.""" + + def __init__(self, kind: str, detail: str): + super().__init__(f"{kind}: {detail}") + self.kind = kind + self.detail = detail + + +class CasLost(Exception): + """The source's provenance moved under this sync: nothing was written.""" + + +def utc_now() -> str: + return _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def say(text: str = "") -> None: + sys.stdout.write(text + "\n") + + +def note(text: str) -> None: + sys.stderr.write(f"{PROG}: {text}\n") + + +# =========================================================== configuration + +def _home() -> str: + return os.environ.get("HOME") or os.path.expanduser("~") + + +def config_path() -> tuple[str, bool]: + """The configuration's path, and whether a person NAMED it. + + A named file that is not there is a refusal; the default path that is not + there is the ordinary case, and means SQLite at its default path.""" + named = os.environ.get("LANES_INDEX_CONFIG", "") + if named: + return named, True + base = os.environ.get("XDG_CONFIG_HOME") or os.path.join(_home(), ".config") + return os.path.join(base, "openRepoTools", "lanes-index.conf"), False + + +def state_dir() -> str: + base = os.environ.get("XDG_STATE_HOME") or os.path.join(_home(), ".local", "state") + return os.path.join(base, "openRepoTools") + + +def git_work_tree_of(path: str) -> str: + """The work tree `path` sits inside, or "" where it is in none. + + Asked of the nearest directory that EXISTS, because a store or a + configuration not created yet is still a file that would land inside it.""" + probe = os.path.abspath(path) + while probe and not os.path.isdir(probe): + parent = os.path.dirname(probe) + if parent == probe: + return "" + probe = parent + try: + proc = subprocess.run(["git", "-C", probe, "rev-parse", "--show-toplevel"], + capture_output=True, text=True, check=False, + stdin=subprocess.DEVNULL) + except OSError: + return "" + if proc.returncode != 0: + return "" + return proc.stdout.strip() + + +def _agents_root() -> str: + return os.environ.get("AGENT_PROTOCOL_ROOT") or os.path.join(_home(), ".agents") + + +def _under(path: str, root: str) -> bool: + try: + path_r = os.path.realpath(path) + root_r = os.path.realpath(root) + except OSError: + return False + return path_r == root_r or path_r.startswith(root_r.rstrip(os.sep) + os.sep) + + +def _url_has_password(url: str) -> bool: + """A password in a libpq URL or keyword string, however it is spelled. + + `postgresql://user:secret@host/db`, `?password=secret`, and the keyword + form `password=secret` are each a credential in a file that is not the + password file - and a URL is what ends up in an argv, a log or a process + listing.""" + low = url.strip() + m = re.match(r"^[A-Za-z][A-Za-z0-9+.-]*://([^/?#]*)", low) + if m: + userinfo = m.group(1).rpartition("@") + if userinfo[1] and ":" in userinfo[0]: + return True + query = low.split("?", 1)[1] if "?" in low else "" + for part in re.split(r"[&;]", query): + if part.split("=", 1)[0].strip().lower() == "password": + return True + return False + return bool(re.search(r"(^|\s)password\s*=", low, re.IGNORECASE)) + + +def _check_private_file(path: str, what: str) -> None: + try: + st = os.stat(path) + except OSError as exc: + raise Refusal(f"{what} {path} cannot be read: {exc.strerror or exc}") + if not stat.S_ISREG(st.st_mode): + raise Refusal(f"{what} {path} is not a regular file") + if st.st_mode & 0o077: + raise Refusal( + f"{what} {path} is mode {oct(st.st_mode & 0o777)[2:]}, so another account " + f"on this workstation can read it (Amendment 14(c): mode 0600). " + f"Fix it: chmod 600 {path}") + + +def load_config() -> dict: + """The per-workstation configuration, refused on any of clause (c)'s grounds.""" + path, named = config_path() + cfg = {"path": path, "present": False, "store": "", "url": "", "passfile": "", + "reader_url": "", "reader_passfile": "", "sqlite": "", "schema": "lanes_index"} + if not os.path.lexists(path): + if named: + raise Refusal(f"LANES_INDEX_CONFIG names {path}, which does not exist") + else: + cfg["present"] = True + _check_private_file(path, "the configuration") + tree = git_work_tree_of(os.path.dirname(os.path.abspath(path))) + if tree: + raise Refusal( + f"the configuration {path} is inside the git work tree {tree}: " + f"connection settings are per workstation and never committed " + f"(Amendment 14(c)). Move it to " + f"${{XDG_CONFIG_HOME:-$HOME/.config}}/openRepoTools/lanes-index.conf") + if _under(path, _agents_root()): + raise Refusal( + f"the configuration {path} is under {_agents_root()}, which on a " + f"workstation built from the workspace repository IS that repository " + f"(Amendment 14(c)). Move it to " + f"${{XDG_CONFIG_HOME:-$HOME/.config}}/openRepoTools/lanes-index.conf") + try: + with open(path, encoding="utf-8") as handle: + lines = handle.read().splitlines() + except (OSError, UnicodeDecodeError) as exc: + raise Refusal(f"the configuration {path} cannot be read: {exc}") + known = ("store", "url", "passfile", "reader_url", "reader_passfile", "sqlite", "schema") + for number, raw in enumerate(lines, 1): + line = raw.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + raise Refusal(f"{path}:{number} is not `key=value`: {line!r}") + key, value = (part.strip() for part in line.split("=", 1)) + if key not in known: + raise Refusal(f"{path}:{number} names `{key}=`, which this indexer does not " + f"know (it reads {', '.join(k + '=' for k in known)})") + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + value = value[1:-1] + if value.startswith("~/"): + value = os.path.join(_home(), value[2:]) + cfg[key] = value + for key in ("url", "reader_url"): + if cfg[key] and _url_has_password(cfg[key]): + raise Refusal( + f"`{key}=` in {path} carries a password. The secret lives only in the " + f"0600 libpq password file `passfile=` names (Amendment 14(c)), never in " + f"a URL a command line or a log can show") + store = cfg["store"] or ("postgres" if cfg["url"] else "sqlite") + if store not in ("sqlite", "postgres"): + raise Refusal(f"`store={cfg['store']}` in {path}: the store is `sqlite` or `postgres`") + if store == "postgres" and not cfg["url"]: + raise Refusal(f"`store=postgres` in {path} names no `url=`") + cfg["store"] = store + if not SCHEMA_NAME_RE.match(cfg["schema"]): + raise Refusal(f"`schema={cfg['schema']}` in {path} is not a plain lower-case SQL identifier") + for key in ("passfile", "reader_passfile"): + if cfg[key]: + _check_private_file(cfg[key], f"the libpq password file (`{key}=`)") + if store == "sqlite": + sqlite_path = cfg["sqlite"] or os.path.join(state_dir(), "lanes-index.sqlite") + sqlite_path = os.path.abspath(sqlite_path) + tree = git_work_tree_of(os.path.dirname(sqlite_path)) + if tree: + raise Refusal( + f"the SQLite store {sqlite_path} would be inside the git work tree {tree}; " + f"the index lives on local disk and never inside a work tree " + f"(Amendment 14(c)). Name another path with `sqlite=`") + cfg["sqlite"] = sqlite_path + return cfg + + +# =============================================================== the source + +def lanes_edit_path() -> str: + """`$LANES_EDIT`, else the `lanes-edit.sh` beside this file, else PATH - + the order `lanes` already finds its reader in.""" + named = os.environ.get("LANES_EDIT", "") + if named: + return named + here = os.path.join(os.path.dirname(os.path.realpath(__file__)), "lanes-edit.sh") + if os.access(here, os.X_OK): + return here + return shutil.which("lanes-edit.sh") or "" + + +def helper_env() -> dict: + """The environment `lanes-edit.sh` is run in from here: the caller's own, + with the fetch, GitHub and the index switched off. This process reads what + landed and reaches no network, and a helper run from inside the indexer + must never start an indexer.""" + env = dict(os.environ) + env["LANES_NO_FETCH"] = "1" + env["LANES_NO_GITHUB"] = "1" + env["LANES_INDEX"] = "off" + return env + + +def workspace_root() -> str: + helper = lanes_edit_path() + if not helper: + raise StoreUnusable("unread", "no lanes-edit.sh beside this file or on PATH, so the " + "workspace repository cannot be found (Amendment 9(a))") + proc = subprocess.run([helper, "workspace-root"], capture_output=True, text=True, + check=False, env=helper_env(), stdin=subprocess.DEVNULL) + root = proc.stdout.strip() + if proc.returncode != 0 or not root: + why = (proc.stderr.strip().splitlines() or ["no answer"])[0] + raise StoreUnusable("unread", f"`lanes-edit.sh workspace-root` exited " + f"{proc.returncode}: {why}") + return root + + +def _norm_repo(text: str) -> str: + u = text.strip() + if u.endswith(".git"): + u = u[:-4] + for prefix in ("https://github.com/", "ssh://git@github.com/", "git@github.com:", + "http://github.com/"): + if u.startswith(prefix): + return u[len(prefix):] + return u + + +def source_of(root: str) -> str: + """`register:` - the `repository:` this workstation's + `workspace.yaml` names, which is how every lane helper names the workspace + (Amendment 9(a)); its checkout's `origin` where that file does not answer.""" + yaml = os.path.join(_agents_root(), "workspace.yaml") + repo = "" + try: + with open(yaml, encoding="utf-8") as handle: + for raw in handle: + m = re.match(r"^repository:[ \t]*(.*)$", raw.rstrip("\n")) + if m: + repo = re.sub(r"[ \t]*#.*$", "", m.group(1)).strip().strip("\"'") + break + except OSError: + repo = "" + if not repo: + proc = git(root, "remote", "get-url", "origin", check=False) + repo = proc.stdout.strip() + repo = _norm_repo(repo) or os.path.basename(root) + return f"register:{repo}" + + +def branch() -> str: + return os.environ.get("LANES_BRANCH") or "main" + + +def git(root: str, *args: str, check: bool = True, data: bytes | None = None, + text: bool = True) -> subprocess.CompletedProcess: + proc = subprocess.run(["git", "-C", root, *args], capture_output=True, + check=False, input=data, text=text if data is None else False, + stdin=None if data is not None else subprocess.DEVNULL) + if check and proc.returncode != 0: + err = proc.stderr if isinstance(proc.stderr, str) else proc.stderr.decode("utf-8", "replace") + raise StoreUnusable("unread", f"git {' '.join(args)} failed in {root}: " + f"{(err.strip().splitlines() or ['no message'])[-1]}") + return proc + + +def tip_of(root: str) -> str: + proc = git(root, "rev-parse", "--verify", "-q", f"refs/remotes/origin/{branch()}^{{commit}}", + check=False) + sha = proc.stdout.strip() + if proc.returncode != 0 or not SHA_RE.match(sha): + raise StoreUnusable("unread", f"{root} has no origin/{branch()}: nothing has landed " + f"there for the index to follow") + return sha + + +def has_commit(root: str, sha: str) -> bool: + return git(root, "cat-file", "-e", f"{sha}^{{commit}}", check=False).returncode == 0 + + +def is_ancestor(root: str, old: str, new: str) -> bool: + return git(root, "merge-base", "--is-ancestor", old, new, check=False).returncode == 0 + + +def commits_between(root: str, old: str, new: str) -> int | None: + if not has_commit(root, old): + return None + proc = git(root, "rev-list", "--count", f"{old}..{new}", check=False) + try: + return int(proc.stdout.strip()) + except ValueError: + return None + + +def _decode(blob: bytes) -> str: + return blob.decode("utf-8", "replace") + + +def read_snapshot(root: str, sha: str) -> dict: + """Every file the index derives from, AS OF ONE COMMIT. + + The log list is `git ls-tree -- lanes/log/` in ITS order, which is the + order `lanes-edit.sh`'s own published read walks them in - and that order is + kept, because a reader built on the index has to produce the same stream.""" + snap = {"sha": sha, "register": None, "aliases": None, "archive": None, "logs": [], + "blobs": {}} + wanted = [] + proc = git(root, "ls-tree", "-z", sha, "--", REGISTER_PATH, ALIASES_PATH, ARCHIVE_PATH) + for entry in proc.stdout.split("\0"): + if not entry: + continue + meta, _, path = entry.partition("\t") + _mode, kind, obj = meta.split() + if kind != "blob": + continue + wanted.append((path, obj)) + logs = [] + proc = git(root, "ls-tree", "-z", sha, "--", LOG_PREFIX) + for entry in proc.stdout.split("\0"): + if not entry: + continue + meta, _, path = entry.partition("\t") + _mode, kind, obj = meta.split() + if kind != "blob": + continue + if path.rsplit("/", 1)[-1] == "README.md": + continue + logs.append((path, obj)) + wanted.append((path, obj)) + contents = cat_blobs(root, [obj for _path, obj in wanted]) + for path, obj in wanted: + text = _decode(contents.get(obj, b"")) + snap["blobs"][path] = obj + if path == REGISTER_PATH: + snap["register"] = text + elif path == ALIASES_PATH: + snap["aliases"] = text + elif path == ARCHIVE_PATH: + snap["archive"] = text + for order, (path, obj) in enumerate(logs): + snap["logs"].append((path, _decode(contents.get(obj, b"")), obj, order)) + if snap["register"] is None: + raise StoreUnusable("unread", f"{sha[:12]} carries no {REGISTER_PATH}") + return snap + + +def cat_blobs(root: str, objs: list[str]) -> dict: + """The bytes of every blob named, in ONE `git cat-file --batch`.""" + out: dict = {} + unique = list(dict.fromkeys(objs)) + if not unique: + return out + proc = git(root, "cat-file", "--batch", data=("\n".join(unique) + "\n").encode(), text=False) + buf = proc.stdout + pos = 0 + for obj in unique: + nl = buf.index(b"\n", pos) + header = buf[pos:nl].decode("ascii", "replace").split() + pos = nl + 1 + if len(header) < 3 or header[1] == "missing": + continue + size = int(header[2]) + out[obj] = buf[pos:pos + size] + pos += size + 1 + return out + + +# ============================================================== the parsers +# +# THE LOG GRAMMAR IS `lanes-edit.sh`'s `LOG_AWK`, PORTED LINE FOR LINE. A +# parser of this file's own that disagreed with that one would make `history +# --index` print a different diary from `history`, which is the one thing the +# read flag promises it does not. Every test, every trim and every reason for +# an unreadable line below is that awk's; `tests/test_lanes_index.py` holds the +# two to one output over the same logs. + +def _trim(text: str) -> str: + return text.strip(" \t") + + +def file_lane(path: str) -> str: + base = path.rsplit("/", 1)[-1] + return re.sub(r"\.[Mm][Dd]$", "", base) + + +def parse_log_line(line: str): + """None for a line the grammar skips (a comment or a blank), a dict with + `bad` for one it cannot read, else the parsed fields.""" + if re.match(r"^[ \t]*#", line) or re.match(r"^[ \t]*$", line): + return None + i = line.find(" — ") + if i < 0: + return {"bad": 'no " — " after the verb'} + verb = _trim(line[:i]) + rest = line[i + 3:] + if " " in verb: + return {"bad": f'the verb is one token: "{verb}"'} + if not rest.startswith("lane "): + return {"bad": 'no "lane " field'} + rest = rest[5:] + q = rest.find(",") + if q < 0: + return {"bad": 'no "," after the lane'} + written_lane = _trim(rest[:q]) + rest = _trim(rest[q + 1:]) + if not rest.startswith("session "): + return {"bad": 'no "session @" field'} + rest = rest[8:] + q = rest.find(",") + if q < 0: + return {"bad": 'no "," after the session'} + sess = _trim(rest[:q]) + rest = _trim(rest[q + 1:]) + a = sess.find("@") + uuid = sess[:a] if a >= 0 else sess + ws = sess[a + 1:] if a >= 0 else "" + m = UTC_RE.match(rest) + if not m: + return {"bad": "no YYYY-MM-DDTHH:MM[:SS]Z"} + utc = m.group(0) + rest = rest[m.end():] + if not rest.startswith(","): + return {"bad": 'no "," and object after the UTC'} + rest = _trim(rest[1:]) + sp = rest.find(" ") + if sp < 0: + obj, rest = rest, "" + else: + obj, rest = rest[:sp], rest[sp + 1:] + ref = payload = txt = "" + if rest != "": + if rest.startswith("— "): + txt = _trim(rest[2:]) + else: + j = rest.find(" — ") + if j >= 0: + txt = _trim(rest[j + 3:]) + rest = _trim(rest[:j]) + if rest.startswith("→ "): + ref, payload = "→", _trim(rest[2:]) + elif rest.startswith("← "): + ref, payload = "←", _trim(rest[2:]) + else: + payload = _trim(rest) + return {"verb": verb, "written_lane": written_lane, "uuid": uuid, "ws": ws, "utc": utc, + "object": obj, "ref": ref, "payload": payload, "text": txt} + + +def payload_fields(payload: str) -> dict: + """`payload_subfield`'s split: `; `-separated sub-fields, the first token + the name, a quoted value taken whole, and the FIRST of a repeated name wins + exactly as that reader's `exit` makes it.""" + fields: dict = {} + if not payload: + return fields + for piece in payload.split("; "): + name, _, value = piece.partition(" ") + if not name or name in fields: + continue + value = _trim(value) + if value.startswith('"'): + q = value.find('"', 1) + if q > 0: + value = value[1:q] + fields[name] = value + return fields + + +def parse_log(path: str, text: str) -> tuple[list, list]: + """The log's records - one per line the grammar does not skip, unreadable + ones INCLUDED - and the warnings `LOG_AWK` prints for it.""" + lane = file_lane(path) + records, warnings = [], [] + for ordinal, line in enumerate(text.split("\n"), 1): + parsed = parse_log_line(line) + if parsed is None: + continue + rec = {"lane": lane, "file": path, "ordinal": ordinal, "raw": line} + if "bad" in parsed: + rec.update(unreadable=1, unreadable_why=parsed["bad"]) + warnings.append(f"unreadable: {path}:{ordinal} ({parsed['bad']})") + else: + obj = parsed["object"] + if lane and obj.startswith("lane:"): + obj = "lane:" + lane + rec.update(unreadable=0, unreadable_why=None, verb=parsed["verb"], + written_lane=parsed["written_lane"], session=parsed["uuid"], + workstation=parsed["ws"], utc=parsed["utc"], object=obj, + ref=parsed["ref"], payload=parsed["payload"], + free_text=parsed["text"]) + records.append(rec) + if warnings: + warnings.append(f"{len(warnings)} unreadable line(s) above — an append-only log is " + f"never rewritten, so a line no parser reads is a hold no tool will " + f"mention again") + return records, warnings + + +def row_name(line: str) -> str | None: + """`ROW_AWK`'s rule: a line opening with `|` whose first backticked token + is the lane. None where the line is not a row by that rule.""" + if not line.startswith("|"): + return None + p1 = line.find("`") + if p1 < 0: + return None + rest = line[p1 + 1:] + p2 = rest.find("`") + if p2 < 0: + return None + return rest[:p2] + + +def cell_state(row: str) -> str: + """`LANES_REGISTER_INDEX_AWK`'s `cell_state`: six ` | ` from the left or it + is not knowable which text is the state cell, and then the last piece.""" + row = row.rstrip(" \t") + if not row.startswith("|") or not row.endswith("|"): + return "" + rest = row[:-1] + for _ in range(6): + i = rest.find(" | ") + if i < 0: + return "" + rest = rest[i + 3:] + if " | " in rest: + return "" + return rest + + +REGISTER_KIND_RE = re.compile(r"^(HOLD RELEASED|HOLD|LANDING|LANDED)[ \t]+(—|-)[ \t]") + + +def parse_register_line(line: str) -> dict | None: + """Rule 6's `LANDING`/`LANDED` and Amendment 10's `HOLD`/`HOLD RELEASED`. + + The kind is the line's own first word(s); every other field is read the way + `RULE6_AWK` reads it, best effort, because a `HOLD` is free text by + Amendment 10(e) and the raw line is kept beside whatever was found.""" + m = REGISTER_KIND_RE.match(line) + if not m: + return None + fields = {"kind": m.group(1), "lane": None, "repo": None, "pr": None, "tip": None, + "utc": None, "session": None} + lm = re.search(r"lane ([A-Za-z0-9._-]+)", line) + if lm: + fields["lane"] = lm.group(1) + pm = re.search(r"PR #([0-9]+)", line) or re.search(r"#([0-9]+)", line) + if pm: + fields["pr"] = pm.group(1) + rm = re.search(r"into ([A-Za-z0-9/._-]+) main", line) or \ + re.search(r"([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+) main\b", line) + if rm: + fields["repo"] = rm.group(1) + um = UTC_RE.search(line) + if um: + fields["utc"] = um.group(0) + sm = re.search(r"session ([^,@ ]+)@", line) + if sm: + fields["session"] = sm.group(1) + tm = re.search(r"→ ([0-9a-f]{7,40})\b", line) or re.search(r" main ([0-9a-f]{7,40})\b", line) + if tm: + fields["tip"] = tm.group(1) + return fields + + +# ======================================================== the records built + +TABLE_COLUMNS = { + "source_files": ["kind", "lane", "ls_order", "blob_sha", "line_count", "unreadable_count"], + "lanes": ["lane", "aliases", "home", "workstation", "started_utc", "handoff_path", + "state_phrase", "objects_cell", "session_ids", "last_session", "archived", + "owner", "row_file", "row_ordinal", "row_text"], + "log_lines": ["lane", "file", "ordinal", "verb", "session", "workstation", "utc", "object", + "ref", "payload", "payload_fields", "free_text", "written_lane", "unreadable", + "unreadable_why", "raw"], + "register_lines": ["kind", "lane", "repo", "pr", "tip", "utc", "session", "ordinal", "raw"], + "lane_aliases": ["old_name", "new_name", "utc", "ordinal", "raw"], + "transcript_pointers": ["lane", "kind", "transcript_id", "agent", "position", "workstation", + "estate", "profile_family", "resolved_store", "project_key", + "parent_uuid", "size_bytes", "sha256", "sealed_utc"], + # THE LEDGER HALF'S OWN TABLES, created EMPTY: T058 of the 001 feature + # writes them under `ledger:` (Amendment 14(g) act 4), and this + # indexer writes nothing into them. + "swap_states": ["lane", "parent_uuid", "generation", "state", "observed_utc"], + "job_summaries": ["lane", "job_id", "result_summary", "observed_utc"], + # RESERVED FOR opensoft/openRepoTools#161, created EMPTY with that issue's + # columns: each lane's #97 worktree inventory under `inventory:`, + # keyed by (source, lane, path) - here the record id `:`. Amendment + # 14(e) leaves inventories to "a later word's"; #161 is that word's issue, and + # this indexer writes nothing into the table. + "worktrees": ["workstation", "lane", "owner", "path", "branch", "base", "lifecycle", + "generation", "operation", "writer_live", "last_seen_utc", "last_commit", + "dirty_count", "unpushed_count", "pr_number", "pr_state"], +} + +INTEGER_COLUMNS = {"ls_order", "line_count", "unreadable_count", "archived", "row_ordinal", + "ordinal", "unreadable", "position", "size_bytes", "generation", + "writer_live", "dirty_count", "unpushed_count", "pr_number"} + +#: The tables `sync` and `reconcile` write. The ledger's two and #161's +#: `worktrees` are absent on purpose. +REGISTER_TABLES = ("source_files", "lanes", "log_lines", "register_lines", "lane_aliases", + "transcript_pointers") + + +def digest_of(table: str, record: dict) -> str: + content = [table] + [record.get(col) for col in TABLE_COLUMNS[table]] + blob = json.dumps(content, ensure_ascii=False, separators=(",", ":"), sort_keys=False) + return hashlib.sha256(blob.encode("utf-8")).hexdigest() + + +def build_records(snap: dict) -> dict: + """Every record of the register half at one commit, WITHOUT the owner + column: the owner is `managed-projection`'s answer, asked separately.""" + out = {table: {} for table in REGISTER_TABLES} + + def put(table: str, record_id: str, record: dict) -> None: + out[table][record_id] = record + + # ---- the files themselves + for path, kind in ((REGISTER_PATH, "register"), (ALIASES_PATH, "aliases"), + (ARCHIVE_PATH, "archive")): + text = snap.get(kind) + if text is None: + continue + put("source_files", path, {"kind": kind, "lane": None, "ls_order": None, + "blob_sha": snap["blobs"].get(path), + "line_count": len(text.split("\n")), + "unreadable_count": None}) + + # ---- the logs, every line, the unreadable ones kept (7(b)) + facts: dict = {} + first_log_of: dict = {} + for path, text, obj, order in snap["logs"]: + records, _warnings = parse_log(path, text) + lane = file_lane(path) + first_log_of.setdefault(lane.lower(), lane) + unreadable = 0 + for rec in records: + rec_id = f"{rec['lane']}:{rec['ordinal']}" + # TWO FILES WHOSE NAMES DIFFER ONLY BY CASE are two logs for one + # lane - Amendment 15(d)'s hand merge - and each keeps its lines + # under its own spelling, so neither file's lines replace the other's. + if rec_id in out["log_lines"]: + rec_id = f"{rec['file']}:{rec['ordinal']}" + if rec["unreadable"]: + unreadable += 1 + row = {col: None for col in TABLE_COLUMNS["log_lines"]} + row.update(lane=rec["lane"], file=rec["file"], ordinal=rec["ordinal"], + unreadable=1, unreadable_why=rec["unreadable_why"], raw=rec["raw"]) + else: + row = {col: rec.get(col) for col in TABLE_COLUMNS["log_lines"]} + row["payload_fields"] = json.dumps(payload_fields(rec["payload"]), + ensure_ascii=False, sort_keys=False) + put("log_lines", rec_id, row) + if rec["unreadable"]: + continue + f = facts.setdefault(lane.lower(), {"home": None, "sid": None, "paused": []}) + fork = rec["payload"].startswith("fork ") + if rec["verb"] in ("STARTED", "RESUMED") and not fork: + hm = re.search(r"home [^ ;]+", rec["payload"]) + if hm: + value = hm.group(0)[5:] + if value != "unknown": + f["home"] = value + if rec["verb"] in ("PAUSED", "RESUMED") and UUID_FULL_RE.match(rec["session"] or ""): + f["sid"] = rec["session"].lower() + if rec["verb"] == "PAUSED": + pf = payload_fields(rec["payload"]) + if "transcript" in pf: + f["paused"].append((rec["ordinal"], pf.get("transcript"), pf.get("agent"))) + put("source_files", path, {"kind": "log", "lane": lane, "ls_order": order, + "blob_sha": obj, "line_count": len(text.split("\n")), + "unreadable_count": unreadable}) + + # ---- the register: its rows and its Rule 6 / Amendment 10 lines + rows = [] + for ordinal, line in enumerate(snap["register"].split("\n"), 1): + name = row_name(line) + if name is not None: + if name != "": + rows.append((REGISTER_PATH, ordinal, name, line, 0)) + continue + fields = parse_register_line(line) + if fields is not None: + fields["ordinal"] = ordinal + fields["raw"] = line + put("register_lines", str(ordinal), fields) + if snap.get("archive") is not None: + for ordinal, line in enumerate(snap["archive"].split("\n"), 1): + name = row_name(line) + if name: + rows.append((ARCHIVE_PATH, ordinal, name, line, 1)) + + # ---- Amendment 16(e)'s alias table, every line kept + alias_map: dict = {} + if snap.get("aliases") is not None: + for ordinal, line in enumerate(snap["aliases"].split("\n"), 1): + if line.strip() == "": + continue + parts = line.split("\t") + old = _trim(parts[0]) if parts else "" + new = _trim(parts[1]) if len(parts) > 1 else "" + when = _trim(parts[2]) if len(parts) > 2 else "" + comment = bool(re.match(r"^[ \t]*#", line)) + put("lane_aliases", str(ordinal), { + "old_name": None if comment else (old or None), + "new_name": None if comment else (new or None), + "utc": None if comment else (when or None), "ordinal": ordinal, "raw": line}) + if not comment and old and new: + alias_map[old.lower()] = (old, new, ordinal) + + def resolve(name: str) -> str: + seen = set() + cur = name + while cur.lower() in alias_map and cur.lower() not in seen: + seen.add(cur.lower()) + cur = alias_map[cur.lower()][1] + return cur + + aliases_of: dict = {} + for _key, (old, _new, ordinal) in sorted(alias_map.items(), key=lambda kv: kv[1][2]): + aliases_of.setdefault(resolve(old).lower(), []).append(old) + + # ---- one record per lane: every row, then every log with no row + used_lower: set = set() + for row_file, ordinal, name, line, archived in rows: + rec_id = name + if rec_id in out["lanes"]: + rec_id = f"{name}~{'archive' if archived else 'register'}:{ordinal}" + parts = line.split("|") + session_cell = parts[2] if len(parts) > 2 else "" + ids = UUID_RE.findall(session_cell.lower()) + f = facts.get(name.lower(), {}) + state = cell_state(line) + put("lanes", rec_id, { + "lane": name, + "aliases": json.dumps(aliases_of.get(name.lower(), []), ensure_ascii=False), + "home": f.get("home"), + "workstation": _trim(parts[3]) if len(parts) > 3 else None, + "started_utc": _trim(parts[4]) if len(parts) > 4 else None, + "objects_cell": _trim(parts[5]) if len(parts) > 5 else None, + "handoff_path": _trim(parts[6]) if len(parts) > 6 else None, + "state_phrase": _trim(state) if state else None, + "session_ids": json.dumps(ids), + "last_session": ids[-1] if ids else f.get("sid"), + "archived": archived, "owner": None, + "row_file": row_file, "row_ordinal": ordinal, "row_text": line}) + used_lower.add(name.lower()) + cell_ids = ids + for position, tid in enumerate(cell_ids, 1): + put("transcript_pointers", f"{rec_id}:cell:{position}", { + "lane": name, "kind": "session-cell", "transcript_id": tid, "agent": None, + "position": position}) + for path, _text, _obj, _order in snap["logs"]: + lane = file_lane(path) + key = lane.lower() + if key in used_lower: + continue + used_lower.add(key) + f = facts.get(key, {}) + put("lanes", lane, { + "lane": lane, + "aliases": json.dumps(aliases_of.get(key, []), ensure_ascii=False), + "home": f.get("home"), "workstation": None, "started_utc": None, + "objects_cell": None, "handoff_path": None, "state_phrase": None, + "session_ids": json.dumps([]), "last_session": f.get("sid"), + "archived": 0, "owner": None, "row_file": None, "row_ordinal": None, + "row_text": None}) + for key, f in facts.items(): + lane = first_log_of.get(key, key) + for ordinal, tid, agent in f["paused"]: + put("transcript_pointers", f"{lane}:log:{ordinal}", { + "lane": lane, "kind": "paused", "transcript_id": tid, "agent": agent, + "position": ordinal}) + for table in ("transcript_pointers",): + for record in out[table].values(): + for col in TABLE_COLUMNS[table]: + record.setdefault(col, None) + return out + + +# ================================================== the owner column's reader + +def managed_projection(lane: str) -> str: + """`legacy`, `managed ` or `unknown`, AND NOTHING ELSE DECIDES IT. + + #97's own reader answers: `lanes-edit.sh managed-projection ` exits 0 + with the owner, 8 for a legacy lane, anything else for UNKNOWN. This file + carries no parser of the marker, so a malformed one can never be read here + as `legacy` (Amendment 14(e)); a reader that could not be run at all is + UNKNOWN for the same reason.""" + helper = lanes_edit_path() + if not helper: + return "unknown" + try: + proc = subprocess.run([helper, "managed-projection", lane], capture_output=True, + text=True, check=False, env=helper_env(), + stdin=subprocess.DEVNULL, timeout=120) + except (OSError, subprocess.TimeoutExpired): + return "unknown" + if proc.returncode == 0 and proc.stdout.strip(): + return f"managed {proc.stdout.strip().splitlines()[0]}" + if proc.returncode == 8: + return "legacy" + return "unknown" + + +def fill_owners(records: dict, stored_lanes: dict | None) -> int: + """Ask `managed-projection` for every lane whose row text is new or changed + (all of them where `stored_lanes` is None, which is `reconcile`), and carry + the stored answer over for a row that has not moved. An `unknown` is never + carried: it may be a read that failed, and asking again is how it clears. + Returns how many lanes were asked.""" + ask = [] + for rec_id, rec in records["lanes"].items(): + old = (stored_lanes or {}).get(rec_id) + if stored_lanes is not None and old is not None and old.get("row_text") == rec["row_text"] \ + and old.get("lane") == rec["lane"] and old.get("owner") \ + and old.get("owner") != "unknown": + rec["owner"] = old["owner"] + else: + ask.append(rec_id) + if ask: + workers = max(1, min(8, len(ask))) + with ThreadPoolExecutor(max_workers=workers) as pool: + answers = list(pool.map(lambda rid: managed_projection(records["lanes"][rid]["lane"]), + ask)) + for rec_id, owner in zip(ask, answers): + records["lanes"][rec_id]["owner"] = owner + return len(ask) + + +# =============================================================== the stores + +def ddl(dialect: str, schema: str) -> list[str]: + q = (lambda name: f'"{schema}".{name}') if dialect == "postgres" else (lambda name: name) + big = "BIGINT" if dialect == "postgres" else "INTEGER" + stmts = [ + f"CREATE TABLE IF NOT EXISTS {q('schema_version')} " + f"(version INTEGER NOT NULL, applied_utc TEXT NOT NULL)", + f"CREATE TABLE IF NOT EXISTS {q('provenance')} (source TEXT NOT NULL PRIMARY KEY, " + f"commit_sha TEXT, generation {big}, indexed_utc TEXT NOT NULL, " + f"schema_version INTEGER NOT NULL)", + ] + for table, cols in TABLE_COLUMNS.items(): + defs = ["source TEXT NOT NULL", "record_id TEXT NOT NULL"] + for col in cols: + defs.append(f"{col} {big if col in INTEGER_COLUMNS else 'TEXT'}") + defs += ["digest TEXT NOT NULL", "provenance TEXT", "indexed_utc TEXT NOT NULL", + "schema_version INTEGER NOT NULL", "PRIMARY KEY (source, record_id)"] + stmts.append(f"CREATE TABLE IF NOT EXISTS {q(table)} ({', '.join(defs)})") + # HOLDS ARE COMPUTED, NEVER STORED (14(e)): a lane holds an object while + # its own LAST line on it, in FILE ORDER - never by UTC - carries an open + # verb (Amendment 7(b)). + view = ( + f"SELECT l.source, l.lane, l.object, l.verb, l.utc, l.session, l.file, l.ordinal " + f"FROM {q('log_lines')} l WHERE l.unreadable = 0 AND l.object NOT LIKE 'lane:%' " + f"AND l.verb IN ({', '.join(repr(v) for v in OPEN_VERBS)}) " + f"AND l.ordinal = (SELECT MAX(m.ordinal) FROM {q('log_lines')} m " + f"WHERE m.source = l.source AND m.file = l.file AND m.object = l.object " + f"AND m.unreadable = 0)") + if dialect == "postgres": + stmts.append(f"CREATE OR REPLACE VIEW {q('holds')} AS {view}") + else: + stmts.append(f"CREATE VIEW IF NOT EXISTS holds AS {view}") + stmts.append( + f"INSERT INTO {q('schema_version')} (version, applied_utc) SELECT {SCHEMA_VERSION}, " + f"'{utc_now()}' WHERE NOT EXISTS (SELECT 1 FROM {q('schema_version')})") + return stmts + + +class SqliteStore: + dialect = "sqlite" + + def __init__(self, cfg: dict): + self.path = cfg["sqlite"] + self.label = "sqlite" + self.where = self.path + self.conn = None + + def open_read(self): + import sqlite3 + if not os.path.exists(self.path): + raise StoreUnusable("missing", f"{self.path} does not exist") + try: + self.conn = sqlite3.connect(f"file:{self.path}?mode=ro", uri=True, timeout=10) + self.conn.execute("SELECT 1 FROM sqlite_master LIMIT 1").fetchall() + except sqlite3.Error as exc: + raise StoreUnusable("unreachable", f"{self.path}: {exc}") + self._check_schema(writing=False) + + def open_write(self): + import sqlite3 + os.makedirs(os.path.dirname(self.path), mode=0o700, exist_ok=True) + fresh = not os.path.exists(self.path) + try: + self.conn = sqlite3.connect(self.path, timeout=30, isolation_level=None) + if fresh: + os.chmod(self.path, 0o600) + have = {r[0] for r in self.conn.execute( + "SELECT name FROM sqlite_master WHERE type IN ('table','view')")} + if "schema_version" in have: + self._check_schema(writing=True) + self.conn.execute("BEGIN IMMEDIATE") + for stmt in ddl("sqlite", ""): + self.conn.execute(stmt) + self.conn.execute("COMMIT") + except sqlite3.Error as exc: + raise StoreUnusable("unreachable", f"{self.path}: {exc}") + self._check_schema(writing=True) + + def _check_schema(self, writing: bool): + import sqlite3 + try: + rows = self.conn.execute("SELECT version FROM schema_version").fetchall() + except sqlite3.Error: + if writing: + return + raise StoreUnusable("wiped", f"{self.path} carries no schema_version table") + versions = {r[0] for r in rows} + if not versions: + if writing: + return + raise StoreUnusable("wiped", f"{self.path} carries no schema_version row") + if versions != {SCHEMA_VERSION}: + raise StoreUnusable("unknown-schema", + f"schema {sorted(versions)} in {self.path}; this " + f"{PROG} knows {SCHEMA_VERSION}") + + def provenance(self, source: str) -> dict | None: + import sqlite3 + try: + row = self.conn.execute( + "SELECT commit_sha, generation, indexed_utc, schema_version FROM provenance " + "WHERE source = ?", (source,)).fetchone() + except sqlite3.Error as exc: + raise StoreUnusable("wiped", f"{self.path}: {exc}") + if row is None: + return None + return {"commit_sha": row[0], "generation": row[1], "indexed_utc": row[2], + "schema_version": row[3]} + + def all_provenance(self) -> list: + import sqlite3 + try: + return [{"source": r[0], "commit_sha": r[1], "generation": r[2], "indexed_utc": r[3], + "schema_version": r[4]} for r in self.conn.execute( + "SELECT source, commit_sha, generation, indexed_utc, schema_version " + "FROM provenance ORDER BY source")] + except sqlite3.Error as exc: + raise StoreUnusable("wiped", f"{self.path}: {exc}") + + def rows(self, table: str, source: str, columns: list[str]) -> list: + cols = ", ".join(["record_id"] + columns) + return [dict(zip(["record_id"] + columns, r)) for r in self.conn.execute( + f"SELECT {cols} FROM {table} WHERE source = ?", (source,))] + + def apply(self, source: str, old: dict | None, new_sha: str, upserts: dict, + deletes: dict) -> None: + import sqlite3 + now = utc_now() + try: + self.conn.execute("BEGIN IMMEDIATE") + cur = self.conn.execute("SELECT commit_sha FROM provenance WHERE source = ?", + (source,)).fetchone() + want = old["commit_sha"] if old else None + if (cur[0] if cur else None) != want or (cur is None) != (old is None): + self.conn.execute("ROLLBACK") + raise CasLost(f"the provenance of {source} moved under this sync " + f"(it read {str(want)[:12]}, the store now holds " + f"{str(cur[0] if cur else None)[:12]}); nothing was written") + if cur is None: + self.conn.execute( + "INSERT INTO provenance (source, commit_sha, generation, indexed_utc, " + "schema_version) VALUES (?, ?, NULL, ?, ?)", + (source, new_sha, now, SCHEMA_VERSION)) + else: + self.conn.execute( + "UPDATE provenance SET commit_sha = ?, indexed_utc = ?, schema_version = ? " + "WHERE source = ?", (new_sha, now, SCHEMA_VERSION, source)) + for table, ids in deletes.items(): + for rec_id in ids: + self.conn.execute(f"DELETE FROM {table} WHERE source = ? AND record_id = ?", + (source, rec_id)) + for table, recs in upserts.items(): + cols = TABLE_COLUMNS[table] + allc = ["source", "record_id"] + cols + ["digest", "provenance", "indexed_utc", + "schema_version"] + sets = ", ".join(f"{c} = excluded.{c}" for c in allc[2:]) + sql = (f"INSERT INTO {table} ({', '.join(allc)}) VALUES " + f"({', '.join('?' for _ in allc)}) ON CONFLICT (source, record_id) " + f"DO UPDATE SET {sets}") + self.conn.executemany(sql, [ + [source, rec_id] + [rec.get(c) for c in cols] + + [rec["digest"], new_sha, now, SCHEMA_VERSION] + for rec_id, rec in recs.items()]) + self.conn.execute("COMMIT") + except sqlite3.Error as exc: + try: + self.conn.execute("ROLLBACK") + except sqlite3.Error: + pass + raise StoreUnusable("unreachable", f"{self.path}: {exc}") + + +def _pg_quote(value) -> str: + if value is None: + return "NULL" + if isinstance(value, int): + return str(value) + return "'" + str(value).replace("'", "''") + "'" + + +def _pg_unescape(field: str): + """One field of `COPY ... TO STDOUT` in TEXT format: `\\N` is NULL and a + backslash escapes the rest, exactly as the server wrote it.""" + if field == "\\N": + return None + if "\\" not in field: + return field + out, i, n = [], 0, len(field) + simple = {"b": "\b", "f": "\f", "n": "\n", "r": "\r", "t": "\t", "v": "\v", "\\": "\\"} + while i < n: + ch = field[i] + if ch != "\\" or i + 1 >= n: + out.append(ch) + i += 1 + continue + nxt = field[i + 1] + if nxt in simple: + out.append(simple[nxt]) + i += 2 + elif nxt in "01234567": + j = i + 1 + while j < n and j < i + 4 and field[j] in "01234567": + j += 1 + out.append(chr(int(field[i + 1:j], 8))) + i = j + elif nxt == "x": + j = i + 2 + while j < n and j < i + 4 and field[j] in "0123456789abcdefABCDEF": + j += 1 + out.append(chr(int(field[i + 2:j], 16)) if j > i + 2 else "x") + i = j + else: + out.append(nxt) + i += 2 + return "".join(out) + + +class PostgresStore: + """QA Postgres, reached ONLY through `psql` and ONLY where `url=` is + configured. Nothing here imports a driver: `psql` is libpq's own client, + and the password it needs is in the 0600 file `PGPASSFILE` names - never + in this process's arguments and never in an environment it hands on.""" + + dialect = "postgres" + + def __init__(self, cfg: dict, role: str): + self.schema = cfg["schema"] + if role == "reader" and cfg.get("reader_url"): + self.url, self.passfile = cfg["reader_url"], cfg.get("reader_passfile") or cfg["passfile"] + else: + self.url, self.passfile = cfg["url"], cfg["passfile"] + self.label = "postgres" + self.where = re.sub(r"^[A-Za-z][A-Za-z0-9+.-]*://", "", self.url).split("?", 1)[0] + self.psql = shutil.which("psql") or "" + + def _env(self) -> dict: + env = {k: v for k, v in os.environ.items() if k != "PGPASSWORD"} + if self.passfile: + env["PGPASSFILE"] = self.passfile + env.setdefault("PGCONNECT_TIMEOUT", "5") + env["PGAPPNAME"] = PROG + return env + + def _run(self, script: str) -> str: + if not self.psql: + raise StoreUnusable("unreachable", "`url=` is configured and there is no `psql` on PATH") + try: + proc = subprocess.run( + [self.psql, "--no-psqlrc", "-X", "-q", "-v", "ON_ERROR_STOP=1", "-d", self.url], + input=script, capture_output=True, text=True, check=False, env=self._env(), + timeout=300) + except subprocess.TimeoutExpired: + raise StoreUnusable("unreachable", "psql did not answer within 300 s") + except OSError as exc: + raise StoreUnusable("unreachable", f"psql could not be run: {exc}") + if proc.returncode != 0: + err = (proc.stderr.strip().splitlines() or [f"psql exited {proc.returncode}"]) + if "lanes-index: provenance moved" in proc.stderr: + raise CasLost(err[-1]) + if proc.returncode == 2: + raise StoreUnusable("unreachable", err[0]) + raise StoreUnusable("unreachable", err[-1]) + return proc.stdout + + def _query_many(self, queries: list[str]) -> list[list[list]]: + """Several SELECTs in ONE connection, each answered as TEXT-format COPY + rows; a marker line no COPY row can be (one field, a control + character) separates them.""" + marker = "\x1eLANES-INDEX-NEXT" + parts = [f"SET search_path TO \"{self.schema}\";"] + for query in queries: + parts.append(f"COPY ({query}) TO STDOUT;") + parts.append("\\echo '\\036LANES-INDEX-NEXT'") + out = self._run("\n".join(parts) + "\n") + results, current = [], [] + for line in out.split("\n"): + if line == marker: + results.append(current) + current = [] + continue + if line == "": + continue + current.append([_pg_unescape(f) for f in line.split("\t")]) + return results + + def open_read(self): + try: + res = self._query_many([ + f"SELECT version FROM \"{self.schema}\".schema_version"]) + except StoreUnusable as exc: + if "does not exist" in exc.detail: + raise StoreUnusable("wiped", exc.detail) + raise + versions = {int(r[0]) for r in res[0]} if res else set() + if not versions: + raise StoreUnusable("wiped", f"schema {self.schema} carries no schema_version row") + if versions != {SCHEMA_VERSION}: + raise StoreUnusable("unknown-schema", f"schema {sorted(versions)} in {self.schema}; " + f"this {PROG} knows {SCHEMA_VERSION}") + + def open_write(self): + script = ["SET standard_conforming_strings = on;", "BEGIN;"] + script += [s + ";" for s in ddl("postgres", self.schema)] + script.append("COMMIT;") + self._run("\n".join(script) + "\n") + self.open_read() + + def provenance(self, source: str) -> dict | None: + res = self._query_many([ + f"SELECT commit_sha, generation, indexed_utc, schema_version FROM provenance " + f"WHERE source = {_pg_quote(source)}"]) + if not res or not res[0]: + return None + r = res[0][0] + return {"commit_sha": r[0], "generation": r[1], "indexed_utc": r[2], + "schema_version": r[3]} + + def all_provenance(self) -> list: + res = self._query_many([ + "SELECT source, commit_sha, generation, indexed_utc, schema_version " + "FROM provenance ORDER BY source"]) + return [{"source": r[0], "commit_sha": r[1], "generation": r[2], "indexed_utc": r[3], + "schema_version": r[4]} for r in (res[0] if res else [])] + + def rows(self, table: str, source: str, columns: list[str]) -> list: + res = self._query_many([ + f"SELECT {', '.join(['record_id'] + columns)} FROM {table} " + f"WHERE source = {_pg_quote(source)}"]) + out = [] + for r in (res[0] if res else []): + rec = dict(zip(["record_id"] + columns, r)) + for col in columns: + if col in INTEGER_COLUMNS and rec[col] is not None: + rec[col] = int(rec[col]) + out.append(rec) + return out + + def apply(self, source: str, old: dict | None, new_sha: str, upserts: dict, + deletes: dict) -> None: + now = utc_now() + want = old["commit_sha"] if old else None + lines = ["SET standard_conforming_strings = on;", "BEGIN;", + f"SET LOCAL search_path TO \"{self.schema}\";", + # THE COMPARE-AND-SWAP, FIRST AND IN THE SAME TRANSACTION AS THE + # ROWS: the UPDATE takes the provenance row's lock, so a second + # writer reading the same old commit blocks here and then finds + # it moved, and its whole transaction goes (Amendment 14(d)). + "DO $lanes_index_cas$ DECLARE n integer; BEGIN", + f" INSERT INTO provenance (source, commit_sha, generation, indexed_utc, " + f"schema_version) VALUES ({_pg_quote(source)}, NULL, NULL, {_pg_quote(now)}, " + f"{SCHEMA_VERSION}) ON CONFLICT (source) DO NOTHING;", + f" UPDATE provenance SET commit_sha = {_pg_quote(new_sha)}, indexed_utc = " + f"{_pg_quote(now)}, schema_version = {SCHEMA_VERSION} WHERE source = " + f"{_pg_quote(source)} AND commit_sha IS NOT DISTINCT FROM {_pg_quote(want)};", + " GET DIAGNOSTICS n = ROW_COUNT;", + " IF n <> 1 THEN RAISE EXCEPTION 'lanes-index: provenance moved under this " + "sync; nothing was written'; END IF;", + "END $lanes_index_cas$;"] + for table, ids in deletes.items(): + for rec_id in ids: + lines.append(f"DELETE FROM {table} WHERE source = {_pg_quote(source)} AND " + f"record_id = {_pg_quote(rec_id)};") + for table, recs in upserts.items(): + cols = TABLE_COLUMNS[table] + allc = ["source", "record_id"] + cols + ["digest", "provenance", "indexed_utc", + "schema_version"] + sets = ", ".join(f"{c} = EXCLUDED.{c}" for c in allc[2:]) + for rec_id, rec in recs.items(): + values = [source, rec_id] + [rec.get(c) for c in cols] + \ + [rec["digest"], new_sha, now, SCHEMA_VERSION] + lines.append(f"INSERT INTO {table} ({', '.join(allc)}) VALUES " + f"({', '.join(_pg_quote(v) for v in values)}) ON CONFLICT " + f"(source, record_id) DO UPDATE SET {sets};") + lines.append("COMMIT;") + self._run("\n".join(lines) + "\n") + + +def open_store(cfg: dict, role: str): + return SqliteStore(cfg) if cfg["store"] == "sqlite" else PostgresStore(cfg, role) + + +# ============================================================ local status + +def _status_path() -> str: + return os.path.join(state_dir(), "lanes-index.status.json") + + +def read_status() -> dict: + try: + with open(_status_path(), encoding="utf-8") as handle: + data = json.load(handle) + return data if isinstance(data, dict) else {} + except (OSError, ValueError): + return {} + + +def write_status(source: str, **fields) -> None: + """This workstation's own record of its last attempt, which the store + cannot hold for an attempt that could not reach it.""" + try: + os.makedirs(state_dir(), mode=0o700, exist_ok=True) + data = read_status() + entry = data.get(source, {}) + entry.update(fields) + data[source] = entry + tmp = _status_path() + f".{os.getpid()}" + with open(tmp, "w", encoding="utf-8") as handle: + json.dump(data, handle, indent=1, sort_keys=True) + os.chmod(tmp, 0o600) + os.replace(tmp, _status_path()) + except OSError: + pass + + +class SyncLock: + """ONE SYNC AT A TIME PER WORKSTATION (14(d)). A second sync that finds the + lock held leaves a RERUN mark and returns at once - the holder syncs again + before it lets go - so a burst of writes is one or two syncs, never a + queue of them, and no nudge ever waits.""" + + def __init__(self): + self.path = os.path.join(state_dir(), "lanes-index.lock") + self.rerun = os.path.join(state_dir(), "lanes-index.rerun") + self.handle = None + + def acquire(self, wait_seconds: float) -> bool: + try: + import fcntl + except ImportError: + return True + os.makedirs(state_dir(), mode=0o700, exist_ok=True) + self.handle = open(self.path, "a+") + try: + os.chmod(self.path, 0o600) + except OSError: + pass + deadline = time.monotonic() + wait_seconds + while True: + try: + fcntl.flock(self.handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + return True + except OSError: + if time.monotonic() >= deadline: + return False + time.sleep(0.2) + + def mark_rerun(self) -> None: + try: + with open(self.rerun, "w", encoding="utf-8") as handle: + handle.write(utc_now() + "\n") + except OSError: + pass + + def take_rerun(self) -> bool: + try: + os.unlink(self.rerun) + return True + except OSError: + return False + + def release(self) -> None: + if self.handle is not None: + self.handle.close() + self.handle = None + + +# ================================================================ the acts + +def resolve_source(requested: str | None, root: str) -> str: + source = source_of(root) + if requested and requested != source: + if requested.startswith("ledger:"): + raise Refusal(f"{requested} is the managed ledger's half, which T058 of the 001 " + f"feature writes (Amendment 14(g) act 4); this indexer writes the " + f"register half only") + raise Refusal(f"this workstation's register is {source}, not {requested}") + return source + + +def diff_records(store, source: str, records: dict, verify_content: bool) -> tuple: + """The upserts and deletes that take the stored source to `records`. + + `sync` compares DIGESTS: an upsert bearing the stored digest changes + nothing. `reconcile` recomputes each stored row's digest from its CONTENT, + so a row somebody changed by hand - which still carries its old digest - is + found and put right.""" + upserts, deletes, kept = {}, {}, {} + for table in REGISTER_TABLES: + cols = TABLE_COLUMNS[table] + stored = {r["record_id"]: r for r in store.rows(table, source, + cols + ["digest"])} + for rec in records[table].values(): + rec["digest"] = digest_of(table, rec) + for rec_id, rec in records[table].items(): + old = stored.get(rec_id) + if old is not None: + have = digest_of(table, old) if verify_content else old["digest"] + if have == rec["digest"] and (not verify_content or old["digest"] == rec["digest"]): + kept[table] = kept.get(table, 0) + 1 + continue + upserts.setdefault(table, {})[rec_id] = rec + gone = [rec_id for rec_id in stored if rec_id not in records[table]] + if gone: + deletes[table] = gone + return upserts, deletes, kept + + +def stored_lanes_of(store, source: str) -> dict: + return {r["record_id"]: r for r in store.rows("lanes", source, + ["lane", "owner", "row_text"])} + + +def run_index(mode: str, requested: str | None, dry_run: bool) -> int: + cfg = load_config() + root = workspace_root() + source = resolve_source(requested, root) + tip = tip_of(root) + store = open_store(cfg, "writer") + if dry_run: + try: + store.open_read() + except StoreUnusable as exc: + if exc.kind not in ("missing", "wiped"): + raise + store = None + else: + store.open_write() + old = store.provenance(source) if store is not None else None + label = f"{source} at register@{tip[:12]} ({cfg['store']})" + if old and old.get("commit_sha"): + prev = old["commit_sha"] + if not has_commit(root, prev): + say(f"{PROG} {mode}: the index holds register@{prev[:12]}, which this checkout has " + f"not fetched - another workstation is ahead of it; nothing written") + write_status(source, last_attempt_utc=utc_now(), last_result="behind (unknown commit)") + return 0 + if prev == tip and mode == "sync": + say(f"{PROG} sync: {label} - the index is at the tip; nothing to write") + write_status(source, last_attempt_utc=utc_now(), last_sync_utc=utc_now(), + last_result="current", last_error="") + return 0 + if not is_ancestor(root, prev, tip): + behind = is_ancestor(root, tip, prev) + why = ("this checkout is BEHIND the index" if behind else + "the two have diverged - a register history rewritten under the index is " + "rebuilt by wiping the index and reconciling") + say(f"{PROG} {mode}: register@{tip[:12]} does not descend from the indexed " + f"register@{prev[:12]}: {why}. The provenance only moves forward " + f"(Amendment 14(d)); nothing written") + write_status(source, last_attempt_utc=utc_now(), + last_result="not a descendant; nothing written") + return 0 if behind else 1 + snap = read_snapshot(root, tip) + records = build_records(snap) + if store is None: + fill_owners(records, None) + upserts = {t: dict(r) for t, r in records.items() if r} + for table, recs in upserts.items(): + for rec in recs.values(): + rec["digest"] = digest_of(table, rec) + deletes, kept = {}, {} + else: + stored = None if mode == "reconcile" else stored_lanes_of(store, source) + fill_owners(records, stored) + upserts, deletes, kept = diff_records(store, source, records, + verify_content=(mode == "reconcile")) + n_up = sum(len(v) for v in upserts.values()) + n_del = sum(len(v) for v in deletes.values()) + if mode == "reconcile": + say(f"{PROG} reconcile: {label}{' - DRY RUN' if dry_run else ''}") + for table in REGISTER_TABLES: + say(f" {table:<20} {kept.get(table, 0):>6} kept {len(upserts.get(table, {})):>6} " + f"upserted {len(deletes.get(table, [])):>6} removed") + if dry_run: + say(f" total: {n_up} to upsert, {n_del} to remove - nothing written (--dry-run)") + return 0 + if old and old.get("commit_sha") == tip and not upserts and not deletes: + say(f" total: nothing differs; nothing written") + write_status(source, last_attempt_utc=utc_now(), last_sync_utc=utc_now(), + last_result="current", last_error="") + return 0 + store.apply(source, old, tip, upserts, deletes) + write_status(source, last_attempt_utc=utc_now(), last_sync_utc=utc_now(), + last_result=f"{n_up} upserted, {n_del} removed at {tip[:12]}", last_error="") + if mode == "sync": + say(f"{PROG} sync: {label} - {n_up} upserted, {n_del} removed") + else: + say(f" total: {n_up} upserted, {n_del} removed") + return 0 + + +def cmd_sync(requested: str | None) -> int: + if os.environ.get("LANES_INDEX", "") == "off": + say(f"{PROG} sync: LANES_INDEX=off - nothing indexed") + return 0 + lock = SyncLock() + if not lock.acquire(0): + lock.mark_rerun() + say(f"{PROG} sync: another sync holds {lock.path}; it will sync again before it " + f"lets go") + return 0 + try: + rounds = 0 + while True: + rounds += 1 + lock.take_rerun() + rc = _guarded(lambda: run_index("sync", requested, False), requested) + if rc != 0 or rounds >= 5 or not os.path.exists(lock.rerun): + return rc + finally: + lock.release() + + +def cmd_reconcile(requested: str | None, dry_run: bool) -> int: + if os.environ.get("LANES_INDEX", "") == "off" and not dry_run: + say(f"{PROG} reconcile: LANES_INDEX=off - nothing indexed") + return 0 + lock = SyncLock() + if not dry_run and not lock.acquire(60): + note(f"another sync has held {lock.path} for 60 s; nothing written") + return 1 + try: + return _guarded(lambda: run_index("reconcile", requested, dry_run), requested) + finally: + lock.release() + + +def _guarded(act, requested) -> int: + try: + return act() + except CasLost as exc: + note(str(exc)) + _record_error(requested, str(exc)) + return 1 + except StoreUnusable as exc: + note(f"{exc.kind}: {exc.detail} - nothing written") + _record_error(requested, f"{exc.kind}: {exc.detail}") + return 1 + + +def _record_error(requested, message: str) -> None: + try: + source = requested or source_of(workspace_root()) + except StoreUnusable: + source = requested or "register:?" + write_status(source, last_attempt_utc=utc_now(), last_error=f"{utc_now()} {message}") + + +def cmd_status() -> int: + cfg = load_config() + store = open_store(cfg, "reader") + say(f"store: {cfg['store']} {store.where} (configuration: " + f"{cfg['path'] if cfg['present'] else 'none - the defaults'})") + rc = 0 + root = "" + source = "" + tip = "" + try: + root = workspace_root() + source = source_of(root) + tip = tip_of(root) + except StoreUnusable as exc: + say(f"workspace: {exc.detail}") + rc = 1 + provs: list = [] + try: + store.open_read() + provs = store.all_provenance() + say(f"schema: {SCHEMA_VERSION}") + except StoreUnusable as exc: + say(f"index: {exc.kind} - {exc.detail}") + rc = 1 + local = read_status() + sources = [p["source"] for p in provs] + if source and source not in sources: + sources.append(source) + for name in sorted(set(sources) | set(local)): + prov = next((p for p in provs if p["source"] == name), None) + say(f"source {name}") + if prov is None: + say(" provenance: none - this source is not indexed (a wiped or never-synced " + "index; `lanes-index reconcile` builds it)") + elif prov.get("commit_sha"): + say(f" provenance: register@{prov['commit_sha'][:12]} (indexed " + f"{prov['indexed_utc']}, schema {prov['schema_version']})") + else: + say(f" provenance: generation {prov.get('generation')} (indexed " + f"{prov['indexed_utc']}, schema {prov['schema_version']})") + if name == source and tip: + say(f" tip: register@{tip[:12]} (origin/{branch()} in {root})") + if prov is not None and prov.get("commit_sha"): + lag = commits_between(root, prov["commit_sha"], tip) + say(f" lag: {'unknown (the indexed commit is not in this checkout)' if lag is None else f'{lag} commit(s)'}") + entry = local.get(name, {}) + say(f" last sync: {entry.get('last_sync_utc') or 'never, from this workstation'}" + f"{' (' + entry['last_result'] + ')' if entry.get('last_result') else ''}") + say(f" last error: {entry.get('last_error') or 'none'}") + return rc + + +def cmd_export(out_dir: str, requested: str | None) -> int: + """The read flags' reader, and nothing else's: the index's image of this + workstation's register in the shapes `lanes-edit.sh` reads from origin. + + It writes FILES into `out_dir` and prints nothing a caller parses. On + success `meta`; on any reason the index cannot answer, `why` - one line, + which `lanes-edit.sh` prints as `read: sources (index )`.""" + if not os.path.isdir(out_dir): + note(f"--out {out_dir} is not a directory") + return 64 + + def unusable(kind: str, detail: str) -> int: + with open(os.path.join(out_dir, "why"), "w", encoding="utf-8") as handle: + handle.write(f"{kind}: {detail}\n") + return 3 + + try: + cfg = load_config() + except Refusal as exc: + unusable("refused", str(exc)) + return 2 + try: + root = workspace_root() + source = resolve_source(requested, root) + store = open_store(cfg, "reader") + store.open_read() + prov = store.provenance(source) + if prov is None or not prov.get("commit_sha"): + return unusable("wiped", f"no provenance for {source} in the {cfg['store']} store") + files = {r["record_id"]: r for r in store.rows("source_files", source, + TABLE_COLUMNS["source_files"])} + lanes = store.rows("lanes", source, ["row_file", "row_ordinal", "row_text"]) + reg_lines = store.rows("register_lines", source, ["ordinal", "raw"]) + aliases = store.rows("lane_aliases", source, ["ordinal", "raw"]) + log_rows = store.rows("log_lines", source, [c for c in TABLE_COLUMNS["log_lines"] + if c not in ("payload_fields", "raw")]) + except Refusal as exc: + unusable("refused", str(exc)) + return 2 + except StoreUnusable as exc: + return unusable(exc.kind, exc.detail) + + def write(name: str, lines: list[str]) -> None: + with open(os.path.join(out_dir, name), "w", encoding="utf-8", newline="") as handle: + for line in lines: + handle.write(line + "\n") + + register = [(r["row_ordinal"], r["row_text"]) for r in lanes if r["row_file"] == REGISTER_PATH] + register += [(r["ordinal"], r["raw"]) for r in reg_lines] + write("register", [text for _o, text in sorted(register, key=lambda x: x[0])]) + if ARCHIVE_PATH in files: + write("archive", [r["row_text"] for r in sorted( + (r for r in lanes if r["row_file"] == ARCHIVE_PATH), key=lambda r: r["row_ordinal"])]) + if ALIASES_PATH in files: + write("aliases", [r["raw"] for r in sorted(aliases, key=lambda r: r["ordinal"])]) + logs = sorted((f for f in files.values() if f["kind"] == "log"), key=lambda f: f["ls_order"]) + write("logs", [f["record_id"] for f in logs]) + by_file: dict = {} + for row in log_rows: + by_file.setdefault(row["file"], []).append(row) + events, warn = [], [] + for f in logs: + bad = 0 + for row in sorted(by_file.get(f["record_id"], []), key=lambda r: r["ordinal"]): + if row["unreadable"]: + bad += 1 + warn.append(f"unreadable: {row['file']}:{row['ordinal']} ({row['unreadable_why']})") + continue + events.append(US.join([row["utc"] or "", row["lane"] or "", row["verb"] or "", + row["session"] or "", row["workstation"] or "", + row["object"] or "", row["ref"] or "", row["payload"] or "", + row["free_text"] or "", row["file"] or "", + str(row["ordinal"])])) + if bad: + warn.append(f"{bad} unreadable line(s) above — an append-only log is never " + f"rewritten, so a line no parser reads is a hold no tool will mention " + f"again") + write("events", events) + write("warn", warn) + write("meta", [f"store={cfg['store']}", f"source={source}", + f"sha={prov['commit_sha']}", f"indexed_utc={prov['indexed_utc']}"]) + return 0 + + +USAGE = f"""\ +{PROG} sync [--source ] index what LANDED, behind the source +{PROG} reconcile [--source ] [--dry-run] read the source whole; put the index right +{PROG} status per source: store, provenance, lag, errors +{PROG} export --out [--source ] the `--index` read flags' reader + +The DERIVED INDEX of the lane register (lane-collision-protocol Amendment 14). +It is written behind the sources by this command alone, read only by tooling +that decides nothing, and never a gate: no lane act reads it or waits on it. + + sync Reads `origin/` of the workspace repository as the push + left it - never the working tree - and upserts each record that + differs, keyed by (source, record id) and versioned by a digest, + with the source's provenance compared and swapped in the SAME + transaction. A sync from a commit that does not descend from the + indexed one writes nothing. `lanes-edit.sh` starts it DETACHED + after every write that landed; `LANES_INDEX=off` stops that. + reconcile Reads the source whole, compares every stored row by its + CONTENT, upserts what differs and removes what is gone: on a + wiped index, the rebuild. `--dry-run` prints the counts only. + status The store, each source's provenance against its tip, the lag in + commits, and this workstation's last sync and last error. + export Writes the index's image of the register into for + `lanes --index` and `lanes-edit.sh history --index`. Not for people. + + --source `register:`, which must be this workstation's own; + the managed ledger's `ledger:` is T058's to write. + +THE STORE: SQLite at ${{XDG_STATE_HOME:-~/.local/state}}/openRepoTools/lanes-index.sqlite, +or QA Postgres through `psql` where the configuration names a `url=`. +THE CONFIGURATION: ${{XDG_CONFIG_HOME:-~/.config}}/openRepoTools/lanes-index.conf, or the +file $LANES_INDEX_CONFIG names - mode 0600, outside every git work tree and +outside ~/.agents/, with no password in any URL: + + store=sqlite|postgres default: postgres where url= is set, else sqlite + sqlite= the SQLite file + url= the WRITER role (sync, reconcile) + passfile= the 0600 libpq password file for it + reader_url= the SELECT-only READER role (status, export) + reader_passfile= its password file, where it is not the same one + schema= the Postgres schema (default lanes_index) + +Exit: 0 done · 1 a read or write failed, nothing half-written · 2 refused · +3 export: the index cannot answer (the reason is in /why) · 64 usage.""" + + +def main(argv: list[str]) -> int: + if not argv: + sys.stderr.write(USAGE + "\n") + return 64 + if argv[0] in ("-h", "--help", "help"): + say(USAGE) + return 0 + if argv[0] == "--version": + say(f"{PROG} {VERSION} (schema {SCHEMA_VERSION})") + return 0 + verb, args = argv[0], argv[1:] + source = None + dry_run = False + out_dir = None + i = 0 + while i < len(args): + arg = args[i] + if arg in ("-h", "--help"): + say(USAGE) + return 0 + if arg == "--source" or arg.startswith("--source="): + value = arg.split("=", 1)[1] if "=" in arg else (args[i + 1] if i + 1 < len(args) else "") + if not value: + note("--source needs a value: register:") + return 64 + source = value + i += 1 if "=" in arg else 2 + continue + if arg == "--out" or arg.startswith("--out="): + value = arg.split("=", 1)[1] if "=" in arg else (args[i + 1] if i + 1 < len(args) else "") + if not value: + note("--out needs a directory") + return 64 + out_dir = value + i += 1 if "=" in arg else 2 + continue + if arg == "--dry-run" and verb == "reconcile": + dry_run = True + i += 1 + continue + note(f"unknown argument '{arg}' for {verb} - see {PROG} --help") + return 64 + try: + if verb == "sync": + return cmd_sync(source) + if verb == "reconcile": + return cmd_reconcile(source, dry_run) + if verb == "status": + if source: + note("status takes no --source; it prints every source the store holds") + return 64 + return cmd_status() + if verb == "export": + if not out_dir: + note("export needs --out ") + return 64 + return cmd_export(out_dir, source) + except Refusal as exc: + note(f"REFUSED: {exc}") + return 2 + except StoreUnusable as exc: + note(f"{exc.kind}: {exc.detail}") + return 1 + note(f"unknown subcommand '{verb}' (sync|reconcile|status|export) - see {PROG} --help") + return 64 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 78739fb7a1ed7afde39f3509bc751b63e27608da Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:05:11 +0000 Subject: [PATCH 02/11] Nudge the indexer after a landed push, and read it behind --index Amendment 14(d): after a push has landed and the mutex is released, `lanes-edit.sh` starts one detached `lanes-index sync` from `cleanup`, with stdin closed, output on /dev/null, never waited and its status never read. It is skipped where `lanes-index` is not on PATH or LANES_INDEX=off, so the suite's thousands of writes pay one `command -v` and print nothing. Amendment 14(b): `--index` on `lanes` and `history`, typed per invocation and never from the environment, swaps the eight published-read primitives for the index's image and leaves every local read alone, so the answer is the source read's byte for byte where the index is current. Stderr says which was read; any failure to answer falls back to the sources with the source read's exit. LANES_IDX_DIR is assigned empty at start-up, so no act can reach the index. `history --all` and `history --repo ` answer the cross-lane question that was refused with 64: every lane's NOTED and RULED lines on one timeline, from the sources, with or without the flag. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lanes | 38 +++++- lanes-edit.sh | 341 +++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 342 insertions(+), 37 deletions(-) diff --git a/lanes b/lanes index fbd26ed..a678f31 100755 --- a/lanes +++ b/lanes @@ -9,6 +9,7 @@ # lanes --all every lane, whatever the cwd — it clears the narrowing # lanes --here only this workstation's lanes # lanes --fetch any of the above, after refreshing from origin +# lanes --index any of the above, out of the DERIVED INDEX (Amendment 14) # # THE DEFAULT IS EVERY LANE, AND THE NARROWING IS BY CHECKOUT (A11 Addendum 4 # ruling 6, ratified "a11 addendum 4 yes" with ruling 6 answered NO, over the @@ -37,6 +38,16 @@ # nothing. It asks nothing either — a listing is output, not a picker # (ratified decision 3; decision 7 point 5 says why the two are different). # +# `--index` READS THE DERIVED INDEX, AND ONLY WHEN TYPED (lane-collision-protocol +# Amendment 14(b), ratified "ratify 48"). It is passed to the helper, whose +# published reads of the register and the logs then answer out of +# `lanes-index`'s copy; the rows, the footer and the exit are the source +# read's, and stderr says which was read — `read: index () at +# register@; commits behind`, or `read: sources (index )` where +# the index could not answer and the sources did. Never from the environment +# or a configuration: a listing a person did not ask to read from a copy is +# read from the record. +# # LOCAL BY DEFAULT, `--fetch` TO REFRESH. Every read in this family sits in # front of a launch and a person typing one word wants an answer now, so the # default passes `LANES_NO_FETCH=1` and the listing says how old the checkout's @@ -91,7 +102,7 @@ die() { printf '\n%s: REFUSED: %s\n' "$prog" "$1" >&2; exit "${2:-64}"; } usage() { cat <<'USAGE' -lanes [--fetch] [--all] [--here] [--closed] [--repo ] +lanes [--fetch] [--index] [--all] [--here] [--closed] [--repo ] [--dir ] [--prefix ] [--ws ] [--rename ] @@ -106,6 +117,13 @@ is running is never started a second time. --fetch refresh from origin first. Without it the read is LOCAL and the foot of the listing says how old this checkout's answer is: a person typing one word wants an answer now. + --index read the DERIVED INDEX instead of the published register + and logs (Amendment 14): the same rows, the same exit, and + one line on stderr saying which was read — `read: index + () at register@; commits behind`, or + `read: sources (index )` where the index cannot + answer and the sources do. Only when typed: never from the + environment or a configuration. --all every lane, whatever the cwd: it clears the checkout narrowing as well as any --repo/--dir/--ws given with it. --prefix the lanes NAMED for that repository, and the next free @@ -159,6 +177,9 @@ fi command by \`openRepoTools --install\`. Run that, or set \$LANES_EDIT." 1 do_fetch=0 +# `--index` IS PASSED THROUGH AND NEVER MIMICKED HERE: the helper owns the +# store and the fallback, exactly as it owns the network for `--fetch`. +idx_args=() closed=0 # 1 once `--closed` has been seen narrowed=0 # 1 once anything has narrowed the listing all_given=0 # 1 once `--all` has been seen, wherever in the line it was @@ -174,6 +195,7 @@ while [ $# -gt 0 ]; do case "$1" in -h | --help) usage; exit 0 ;; --fetch) do_fetch=1; shift ;; + --index) idx_args=(--index); shift ;; # `--all` CLEARS THE LABEL AS WELL AS THE FILTERS. The helper's `--all` drops # `--repo`, `--dir` and `--prefix`, so the ROWS were right — but `here_repo` # is this wrapper's own, and `lanes --all --prefix repoA11` (either order) @@ -345,11 +367,15 @@ fetch_unchecked=0 if [ "$do_fetch" = 1 ]; then fetch_err_file="$(mktemp "${TMPDIR:-/tmp}/lanes-fetch.XXXXXX" 2>/dev/null || printf '')" if [ -n "$fetch_err_file" ]; then - out="$("$LANES_EDIT" lanes --fetch ${args[@]+"${args[@]}"} 2>"$fetch_err_file")" || rc=$? + out="$("$LANES_EDIT" lanes --fetch ${idx_args[@]+"${idx_args[@]}"} ${args[@]+"${args[@]}"} 2>"$fetch_err_file")" || rc=$? fetch_err="$(cat "$fetch_err_file" 2>/dev/null || printf '')" rm -f -- "$fetch_err_file" + # THE ONE LINE THAT SAYS WHICH WAS READ reaches the person on this path + # too: the rest of this stream is rendered below only where the fetch + # fell back, and `read: …` is not about the fetch. + printf '%s\n' "$fetch_err" | command grep '^read: ' >&2 || : else - out="$("$LANES_EDIT" lanes --fetch ${args[@]+"${args[@]}"})" || rc=$? + out="$("$LANES_EDIT" lanes --fetch ${idx_args[@]+"${idx_args[@]}"} ${args[@]+"${args[@]}"})" || rc=$? fetch_unchecked=1 fi else @@ -364,13 +390,13 @@ else # stream being dropped for the sake of that sentence. local_err_file="$(mktemp "${TMPDIR:-/tmp}/lanes-local.XXXXXX" 2>/dev/null || printf '')" if [ -n "$local_err_file" ]; then - out="$("$LANES_EDIT" lanes ${args[@]+"${args[@]}"} 2>"$local_err_file")" || rc=$? + out="$("$LANES_EDIT" lanes ${idx_args[@]+"${idx_args[@]}"} ${args[@]+"${args[@]}"} 2>"$local_err_file")" || rc=$? grep -v 'not fetching; reading origin/' -- "$local_err_file" >&2 || : rm -f -- "$local_err_file" else # No capture file, so nothing is filtered and everything reaches the # person — the same answer the fetch path gives one branch up. - out="$("$LANES_EDIT" lanes ${args[@]+"${args[@]}"})" || rc=$? + out="$("$LANES_EDIT" lanes ${idx_args[@]+"${idx_args[@]}"} ${args[@]+"${args[@]}"})" || rc=$? fi fi @@ -411,7 +437,7 @@ case "$rc" in # measured defect of openRepoTools#51: under the harness `grep` is a # shell function whose status is 1 when its stdout is `/dev/null`, which # is exactly what `-q` makes it). - if "$LANES_EDIT" lanes --closed ${args[@]+"${args[@]}"} 2>/dev/null | command grep -q .; then e_hidden=1; fi + if "$LANES_EDIT" lanes --closed ${idx_args[@]+"${idx_args[@]}"} ${args[@]+"${args[@]}"} 2>/dev/null | command grep -q .; then e_hidden=1; fi if [ "$e_hidden" = 1 ]; then say "no lane of $here_repo is LISTED — and that is not the same as none being recorded: closed and dormant lanes are left out (Amendment 19(b))." say "" diff --git a/lanes-edit.sh b/lanes-edit.sh index 3c9bf5f..8eb0877 100755 --- a/lanes-edit.sh +++ b/lanes-edit.sh @@ -88,7 +88,8 @@ # lanes-edit.sh set-row-state " · " # lanes-edit.sh log NOTED lane: "" # lanes-edit.sh log RULED lane: [→ ] "" -# lanes-edit.sh history [--since ] +# lanes-edit.sh history [--since ] [--index] +# lanes-edit.sh history --all | --repo [--since ] [--index] # lanes-edit.sh migrate-state-cells [--yes] # # The `state` column holds ONE PHRASE — ` · · ` — and @@ -117,6 +118,21 @@ # register. It REFUSES when no cell holds a ` · ` entry, so a second run is a # no-op that says so. # +# AMENDMENT 14 — THE DERIVED INDEX (`lanes-index`), WHICH NOTHING HERE ACTS ON +# lanes-edit.sh lanes … --index # the listing, out of the index +# lanes-edit.sh history … --index # the diary, out of the index +# +# The register's truth stays where it is. `lanes-index` (placed by +# `openRepoTools --install`) keeps a DERIVED copy — SQLite, or QA Postgres +# where this workstation is configured for it — written BEHIND the sources: +# after every write whose push LANDED, and after the mutex is released, this +# file starts one DETACHED `lanes-index sync` (stdin closed, never waited, +# its status never read; skipped where `lanes-index` is not on PATH or +# `LANES_INDEX=off`). No act reads the index or waits on it. `--index` is the +# only way a read consults it, typed per invocation, and it says on stderr +# which it read: `read: index () at register@; commits +# behind`, or `read: sources (index )` and the sources answer. +# # AMENDMENT 7 — the per-lane object log (`lanes/log/.md`) # lanes-edit.sh log [→|← ] [""] [--text ""] # lanes-edit.sh claim [--force] [--no-github] [--home owner/repo] @@ -570,6 +586,9 @@ # ref as it already stands here. # LANES_STALE_HOURS Rule 1's stale threshold, reused (default: 4) # LANES_DEBUG non-empty: a refusal also prints its exit code +# LANES_INDEX=off no `lanes-index sync` after a write, and `--index` reads +# the sources (Amendment 14). It never turns a read ON: +# only a typed `--index` does # # Dependencies: bash, git, coreutils. No sed/awk substitution on the payload — # every edit is computed with bash string operations, so `/`, `&`, `\` and `|` @@ -1042,6 +1061,13 @@ lanes_binding_probe() { lanes_binding_probe NO_GIT="${LANES_NO_GIT:-0}" LOCK="$LANES_DIR/.lanes-edit.lock" +# AMENDMENT 14 — TWO VALUES ONLY THIS PROCESS SETS, AND NEVER THE ENVIRONMENT. +# The index is consulted behind a flag typed for the invocation, "never by +# default, environment or configuration" (clause (b)), and the nudge follows a +# push THIS process made (clause (d)). Both are assigned here unconditionally, +# so an exported variable of either name changes nothing. +LANES_IDX_DIR="" +LANES_INDEX_NUDGE=0 LOCK_HELD=0 LANES_PATH="${LANES_PATH:-$(git -C "${LANES_DIR:-.}" rev-parse --show-prefix 2>/dev/null || :)${LANES_FILE##*/}}" @@ -1161,6 +1187,12 @@ cleanup() { [ -n "${RL_SNAP:-}" ] && [ -d "${RL_SNAP:-}" ] && rm -rf -- "$RL_SNAP" [ -n "${TMPD:-}" ] && [ -d "${TMPD:-}" ] && rm -rf -- "$TMPD" [ -n "${SE_CACHE_FILE:-}" ] && rm -f -- "$SE_CACHE_FILE" "$SE_CACHE_FILE".* 2>/dev/null + [ -n "${LANES_IDX_DIR:-}" ] && [ -d "${LANES_IDX_DIR:-}" ] && rm -rf -- "$LANES_IDX_DIR" + # AMENDMENT 14(d) — THE NUDGE IS THE LAST THING THIS PROCESS DOES: after the + # lock is released at the top of this function, and only where `commit_push` recorded a push + # that landed. Tested here rather than inside the function, because `cleanup` + # also runs for a process that died before `index_nudge` was ever defined. + [ "${LANES_INDEX_NUDGE:-0}" = 1 ] && index_nudge return 0 } # AND A SIGNAL ACTUALLY STOPS IT (Amendment 8, ruling (h)). `trap cleanup EXIT @@ -1875,6 +1907,7 @@ EOF [ "$GIT_TIMED_OUT" = 1 ] && git_timeout_die "git push -u origin $LANES_BRANCH" die "initial push of '$LANES_BRANCH' failed" 6 fi + LANES_INDEX_NUDGE=1 note "pushed (created origin/$LANES_BRANCH)" return 0 fi @@ -1898,6 +1931,7 @@ EOF printf ' %s\n' $others >&2 note "skipping 'pull --rebase' (it refuses on any unstaged tracked change) and pushing straight out" if git_net -C "$LANES_REPO" push -q origin "$LANES_BRANCH"; then + LANES_INDEX_NUDGE=1 note "pushed origin/$LANES_BRANCH (attempt $attempt, no pull — see the warning above)" return 0 fi @@ -1917,6 +1951,7 @@ EOF "$CP_AFTER_REBASE" || return $? fi if git_net -C "$LANES_REPO" push -q origin "$LANES_BRANCH"; then + LANES_INDEX_NUDGE=1 note "pushed origin/$LANES_BRANCH (attempt $attempt)" return 0 fi @@ -1981,6 +2016,120 @@ EOF die "could not push origin/$LANES_BRANCH after 6 attempts; your commit is local — retry later" 6 } +# ====================================================== AMENDMENT 14 ======= +# +# THE DERIVED INDEX (lane-collision-protocol Amendment 14, ratified by Brett +# Heap 2026-10-05, verbatim "ratify 48"). The register's truth stays where it +# is; `lanes-index` keeps a DERIVED copy of it — SQLite on local disk, or QA +# Postgres where a workstation is configured for it — written behind the +# sources, read only by tooling that decides nothing, and never a gate. +# +# THIS FILE TOUCHES IT IN EXACTLY TWO PLACES, AND NEITHER IS ON AN ACT'S PATH. +# +# THE NUDGE (clause (d)). After a push has LANDED and the mutex is released — +# `cleanup`, on the way out, once `commit_push` has recorded a push that went +# through — ONE DETACHED `lanes-index sync`: stdin closed, stdout and stderr +# on /dev/null, descriptors 3 to 9 closed, started from a subshell that +# exits at once so this process never has a child to wait for, and its +# status never read. It carries no data: the indexer reads `origin/` +# as the push left it, never takes this file's mutex, and a slow, failing or +# absent indexer delays and fails nothing here. `LANES_INDEX=off` skips it, +# and so does a workstation with no `lanes-index` ON PATH — PATH alone, not +# the file beside this one, because a sibling in a clone is not an +# installation and the suite runs this file thousands of times from one: the +# absent case is one `command -v` and prints nothing. +# +# THE READ FLAG (clause (b)). `lanes ... --index` and `history ... --index`, +# and only when TYPED: `index_open` asks `lanes-index export` for the index's +# image of the register, and the eight PUBLISHED-READ primitives — the +# register, the archive, the alias table, the log list, a log's events, the +# events of them all — answer out of that image instead of `origin/`. +# Everything else the read does is unchanged, the LOCAL reads included (this +# checkout's own register and log files, this workstation's session records, +# tmux), which is what makes its answer byte-identical to the source read's +# where the index is current. Anything else — the indexer absent, the store +# missing, unreachable, wiped or of another schema, a configuration it +# refuses — is said in ONE line and the source read runs as it always did. +# +# NOTHING ELSE READS IT. No act, no hook, no guard, no writer and none of the +# reads an act makes for itself (`who`, `binding`, `live-holder`, `next-free`, +# `managed-projection`, `lane-reconcile`) has a path to `LANES_IDX_DIR`: it is +# assigned empty at start-up whatever the environment says, and set only by +# `index_open`, which only the two read arms call. +index_nudge() { + LANES_INDEX_NUDGE=0 + [ "${LANES_INDEX:-}" = off ] && return 0 + in_bin="$(command -v lanes-index 2>/dev/null || :)" + [ -n "$in_bin" ] || return 0 + ( "$in_bin" sync /dev/null 2>&1 3>&- 4>&- 5>&- 6>&- 7>&- 8>&- 9>&- & ) >/dev/null 2>&1 + return 0 +} + +# THE READ FLAG'S ONE DOOR. 0 with `LANES_IDX_DIR` set and `read: index (...)` +# said; 1 with `read: sources (index )` said and NOTHING set, so the caller +# runs the source read exactly as it would have without the flag — and exits as +# that read would. Both lines go to stderr: stdout is the read's own, and a +# person diffing the two answers diffs the answer. +# +# THE LAG IS SAID, NEVER REFUSED (clause (b)): `` is how many commits this +# checkout's `origin/` carries past the indexed one, and `unknown` where +# that commit is not here to count from. +index_open() { + LANES_IDX_DIR="" + if [ "${LANES_INDEX:-}" = off ]; then + printf 'read: sources (index off: LANES_INDEX=off)\n' >&2 + return 1 + fi + io_bin="" + if [ -x "$SCRIPT_DIR/lanes-index" ]; then io_bin="$SCRIPT_DIR/lanes-index" + else io_bin="$(command -v lanes-index 2>/dev/null || :)" + fi + if [ -z "$io_bin" ]; then + printf 'read: sources (index missing: no lanes-index beside %s or on PATH)\n' "$SCRIPT_DIR/lanes-edit.sh" >&2 + return 1 + fi + io_dir="$(mktemp -d "${TMPDIR:-/tmp}/lanes-index-read.XXXXXX" 2>/dev/null || printf '')" + if [ -z "$io_dir" ]; then + printf 'read: sources (index unread: no temporary directory could be made)\n' >&2 + return 1 + fi + io_rc=0 + if command -v timeout >/dev/null 2>&1; then + timeout -k 2 30 "$io_bin" export --out "$io_dir" /dev/null 2>"$io_dir/.stderr" || io_rc=$? + else + "$io_bin" export --out "$io_dir" /dev/null 2>"$io_dir/.stderr" || io_rc=$? + fi + if [ "$io_rc" != 0 ] || [ ! -f "$io_dir/meta" ]; then + io_why="" + [ -f "$io_dir/why" ] && io_why="$(head -n 1 -- "$io_dir/why" 2>/dev/null || :)" + case "$io_rc" in 124|137) io_why="unreachable: lanes-index export did not answer within 30 s" ;; esac + if [ -z "$io_why" ]; then + io_why="unread: lanes-index export exited $io_rc" + io_err="$(head -n 1 -- "$io_dir/.stderr" 2>/dev/null || :)" + [ -z "$io_err" ] || io_why="$io_why: $io_err" + fi + rm -rf -- "$io_dir" + printf 'read: sources (index %s)\n' "$io_why" >&2 + return 1 + fi + io_store="$(sed -n -e 's/^store=//p' "$io_dir/meta" | head -n 1)" + io_sha="$(sed -n -e 's/^sha=//p' "$io_dir/meta" | head -n 1)" + io_behind=unknown + if [ -n "$io_sha" ] && have_remote_ref; then + io_n="$(git -C "$LANES_REPO" rev-list --count "$io_sha..origin/$LANES_BRANCH" 2>/dev/null || :)" + case "$io_n" in ''|*[!0-9]*) : ;; *) io_behind="$io_n" ;; esac + fi + LANES_IDX_DIR="$io_dir" + # WHAT WAS READ BEFORE THE FLAG WAS HONOURED IS NOT WHAT IS READ AFTER IT: + # every published-read cache this process may already hold is dropped. + LANES_REGISTER_CACHE="" + [ -n "${SE_CACHE_FILE:-}" ] && rm -f -- "$SE_CACHE_FILE.register" + lane_alias_flush + state_events_flush + printf 'read: index (%s) at register@%s; %s commits behind\n' "${io_store:-unknown}" "${io_sha:0:12}" "$io_behind" >&2 + return 0 +} + # Prints, to stdout, a comma-joined, de-duplicated list of the lane(s) whose # row changed in the CURRENT unstaged diff of LANES.md — identified by the # first backtick token on each changed (+/-) line — or "append" for a @@ -2307,9 +2456,15 @@ log_files_named_ci() { # — every existing log file for it, whatever i # that half (Copilot round 4). log_path_ci() { # — 0 with the ONE path, 2 where two differ only by case lpc_exact="$(log_path_for "$1")" - if have_remote_ref; then + # AMENDMENT 14 — UNDER `--index` THE INDEX'S LOG LIST STANDS IN FOR + # `origin`'s tree, and nothing else in this function changes. + if [ -n "$LANES_IDX_DIR" ] || have_remote_ref; then + if [ -n "$LANES_IDX_DIR" ]; then + lpc_hits="$(awk -v want="$lpc_exact" 'BEGIN { w = tolower(want) } tolower($0) == w' "$LANES_IDX_DIR/logs" 2>/dev/null)" + else lpc_hits="$(git -C "$LANES_REPO" ls-tree --name-only "origin/$LANES_BRANCH" -- "$LANES_LOG_PREFIX" 2>/dev/null \ | awk -v want="$lpc_exact" 'BEGIN { w = tolower(want) } tolower($0) == w')" + fi lpc_n="$(printf '%s' "$lpc_hits" | grep -c . || :)" if [ "$lpc_n" -gt 1 ]; then note "lane $1 has $lpc_n object logs on origin/$LANES_BRANCH whose names differ only by case: $(printf '%s\n' "$lpc_hits" | tr '\n' ' ')" @@ -2717,6 +2872,14 @@ have_remote_ref() { # has LANDED is what other lanes can see, and a working tree can be anything — # behind by a peer's whole day, or carrying a line that never lands. remote_log_events() { + # AMENDMENT 14 — under `--index`, the index's events of every log, in the + # order and the grammar this function prints them in, and the warnings + # `LOG_AWK` would have printed for the lines it could not read. + if [ -n "$LANES_IDX_DIR" ]; then + cat -- "$LANES_IDX_DIR/warn" >&2 2>/dev/null || : + cat -- "$LANES_IDX_DIR/events" + return 0 + fi [ "$NO_GIT" = 1 ] && { log_events; return 0; } git -C "$LANES_REPO" rev-parse --verify -q "origin/$LANES_BRANCH" >/dev/null 2>&1 || { log_events; return 0; } git -C "$LANES_REPO" ls-tree --name-only "origin/$LANES_BRANCH" -- "$LANES_LOG_PREFIX" 2>/dev/null \ @@ -2806,7 +2969,14 @@ state_events_flush() { # read as "there is no log". lane_log_events() { ll_rel="$(log_path_ci "$1")" || return 2 - if have_remote_ref; then + if [ -n "$LANES_IDX_DIR" ]; then + # A LOG THE INDEX DOES NOT HOLD IS A LOG `origin` DID NOT HOLD at the + # indexed commit: no lines, exactly as a `git show` of a missing path. + command grep -Fqx -- "$ll_rel" "$LANES_IDX_DIR/logs" 2>/dev/null || return 0 + awk -F"$US" -v p="$ll_rel" '$10 == p' "$LANES_IDX_DIR/events" + awk -v p="unreadable: $ll_rel:" 'index($0, p) == 1' "$LANES_IDX_DIR/warn" >&2 2>/dev/null || : + return 0 + elif have_remote_ref; then git -C "$LANES_REPO" show "origin/$LANES_BRANCH:$ll_rel" 2>/dev/null | parse_log_stream "$ll_rel" else ll_f="$LANES_LOG_DIR/${ll_rel##*/}" @@ -2816,7 +2986,9 @@ lane_log_events() { } lane_log_exists() { lle_rel="$(log_path_ci "$1")" || return 2 - if have_remote_ref; then + if [ -n "$LANES_IDX_DIR" ]; then + command grep -Fqx -- "$lle_rel" "$LANES_IDX_DIR/logs" 2>/dev/null + elif have_remote_ref; then git -C "$LANES_REPO" cat-file -e "origin/$LANES_BRANCH:$lle_rel" 2>/dev/null else [ -f "$LANES_LOG_DIR/${lle_rel##*/}" ] @@ -2848,6 +3020,10 @@ lane_log_exists() { # subshell that wrote it, so the megabyte is rendered once per process. LANES_REGISTER_CACHE="" register_text() { + # AMENDMENT 14 — under `--index`, the register's rows and its Rule 6 and + # Amendment 10 lines as the index holds them, in register order. Not cached: + # it is one local file, and the cache below belongs to the published read. + if [ -n "$LANES_IDX_DIR" ]; then cat -- "$LANES_IDX_DIR/register"; return 0; fi if [ -n "$LANES_REGISTER_CACHE" ]; then printf '%s\n' "$LANES_REGISTER_CACHE"; return 0; fi rt_c="${SE_CACHE_FILE:+$SE_CACHE_FILE.register}" if [ -n "$rt_c" ] && [ -s "$rt_c" ]; then cat -- "$rt_c"; return 0; fi @@ -2880,7 +3056,10 @@ LANES_ARCH_NAME="LANES-retired.md" LANES_ARCH_PATH="${LANES_ARCH_PATH:-${LANES_PREFIX}archive/$LANES_ARCH_NAME}" LANES_ARCH_FILE="${LANES_ARCH_FILE:-$LANES_DIR/archive/$LANES_ARCH_NAME}" archive_text() { - if have_remote_ref && git -C "$LANES_REPO" cat-file -e "origin/$LANES_BRANCH:$LANES_ARCH_PATH" 2>/dev/null; then + # AMENDMENT 14 — under `--index`, the archive's rows where the indexed commit + # carried an archive; where it carried none, the same local fallback below. + if [ -n "$LANES_IDX_DIR" ] && [ -f "$LANES_IDX_DIR/archive" ]; then cat -- "$LANES_IDX_DIR/archive"; return 0; fi + if [ -z "$LANES_IDX_DIR" ] && have_remote_ref && git -C "$LANES_REPO" cat-file -e "origin/$LANES_BRANCH:$LANES_ARCH_PATH" 2>/dev/null; then # A published archive that cannot be rendered cannot be replaced with a # possibly stale local copy: that copy may omit a retired position. if git -C "$LANES_REPO" show "origin/$LANES_BRANCH:$LANES_ARCH_PATH" 2>/dev/null; then return 0; fi @@ -3166,6 +3345,15 @@ home_of_lane() { } known_lanes() { + if [ -n "$LANES_IDX_DIR" ]; then + while IFS= read -r kl_f; do + [ -n "$kl_f" ] || continue + kl_n="${kl_f##*/}" + case "$kl_n" in README.md) continue ;; esac + printf '%s\n' "${kl_n%.md}" + done < "$LANES_IDX_DIR/logs" + return 0 + fi if have_remote_ref; then git -C "$LANES_REPO" ls-tree --name-only "origin/$LANES_BRANCH" -- "$LANES_LOG_PREFIX" 2>/dev/null \ | while IFS= read -r kl_f; do @@ -4306,7 +4494,14 @@ lane_alias_text() { lat_t=""; LANES_ALIAS_ERR="" lat_e="${SE_CACHE_FILE:+$SE_CACHE_FILE.aliaserr}" [ -n "$lat_e" ] && rm -f -- "$lat_e" - if have_remote_ref && git -C "$LANES_REPO" cat-file -e "origin/$LANES_BRANCH:$LANES_ALIASES_PATH" 2>/dev/null; then + # AMENDMENT 14 — under `--index`, the table as the indexed commit carried it; + # where it carried none, the same local fallbacks below. + if [ -n "$LANES_IDX_DIR" ] && [ -f "$LANES_IDX_DIR/aliases" ]; then + lat_t="$(cat -- "$LANES_IDX_DIR/aliases" 2>/dev/null)" || { + lane_alias_fail "the index's copy at $LANES_IDX_DIR/aliases could not be read" + return 5 + } + elif [ -z "$LANES_IDX_DIR" ] && have_remote_ref && git -C "$LANES_REPO" cat-file -e "origin/$LANES_BRANCH:$LANES_ALIASES_PATH" 2>/dev/null; then lat_t="$(git -C "$LANES_REPO" show "origin/$LANES_BRANCH:$LANES_ALIASES_PATH" 2>/dev/null)" || { lane_alias_fail "git show origin/$LANES_BRANCH:$LANES_ALIASES_PATH failed, and the object is there" return 5 @@ -14668,41 +14863,117 @@ EOF # 8 IS "THIS LANE HAS WRITTEN NO NARRATIVE", which a lane that has never taken # a note and a lane with no log at all both are; a read that could not be made # is never 8 (R22) and leaves through the die above it. + # AMENDMENT 13's DIARY, and since AMENDMENT 14 ACROSS LANES TOO. + # + # history one lane's `NOTED` and `RULED` lines, in + # FILE ORDER — the diary the state cell used + # to be. Unchanged. + # history --all every lane's, as ONE timeline: by UTC, and + # by file and line where two share a minute. + # history --repo the lanes whose HOME is that repository — + # the `home` of their last `STARTED`/`RESUMED`, + # through the alias table, exactly as `lanes + # --repo` files them — on one timeline. + # --since with any of the three. + # --index read the DERIVED INDEX instead of the + # published logs (Amendment 14(b)), only when + # typed: stderr says `read: index () at + # register@; commits behind`, or + # `read: sources (index )` and the + # sources answer. Where the index answers it + # does not fetch: it reads no network at all, + # and the lag is said against this checkout. + # + # `history` over two lanes was refused, exit 64, on 2026-10-05 (Amendment 14, + # "Why" 3); `--all` and `--repo` are that question, and they read the + # SOURCES — the same one pass over every published log `who` makes — with or + # without the flag. history) - lane=""; hi_since="" + lane=""; hi_since=""; hi_all=0; hi_repo=""; hi_index=0 + hi_usage="history | --all | --repo [--since ] [--index]" while [ $# -gt 0 ]; do case "$1" in --since) hi_since="${2-}"; [ -n "$hi_since" ] || die "--since needs a UTC instant" 64; shift 2 ;; --since=*) hi_since="${1#--since=}"; [ -n "$hi_since" ] || die "--since needs a UTC instant" 64; shift ;; + --all) hi_all=1; shift ;; + --repo) hi_repo="${2-}"; [ -n "$hi_repo" ] || die "--repo needs a repository ($hi_usage)" 64; shift 2 ;; + --repo=*) hi_repo="${1#--repo=}"; [ -n "$hi_repo" ] || die "--repo needs a repository ($hi_usage)" 64; shift ;; + --index) hi_index=1; shift ;; --) shift ;; - -*) die "unknown option '$1' for history (history [--since ])" 64 ;; - *) [ -z "$lane" ] || die "history takes ONE lane: history [--since ]" 64; lane="$1"; shift ;; + -*) die "unknown option '$1' for history ($hi_usage)" 64 ;; + *) [ -z "$lane" ] || die "history takes ONE lane, or --all, or --repo : $hi_usage" 64; lane="$1"; shift ;; esac done - [ -n "$lane" ] || die "usage: history [--since ]" 64 - check_lane_name "$lane" + hi_sel=0 + [ -n "$lane" ] && hi_sel=$((hi_sel + 1)) + [ "$hi_all" = 1 ] && hi_sel=$((hi_sel + 1)) + [ -n "$hi_repo" ] && hi_sel=$((hi_sel + 1)) + [ "$hi_sel" = 1 ] || die "usage: $hi_usage — ONE of a lane, --all or --repo" 64 + [ -z "$lane" ] || check_lane_name "$lane" case "$hi_since" in '') : ;; [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]Z | [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9]Z) : ;; *) die "--since takes a UTC instant spelled as this log spells it, YYYY-MM-DDTHH:MM[:SS]Z — '$hi_since' is not one" 64 ;; esac + # THE INDEX FIRST, and where it answers there is no fetch: the read is the + # index's, and a network round trip in front of it is the cost the flag is + # for avoiding. Where it does not answer, the source read below is exactly + # the one this arm always made, the fetch included. + if [ "$hi_index" = 1 ] && index_open; then LANES_NO_FETCH=1; fi log_sync - lane="$(canon_lane "$lane")" || exit 2 # Amendment 15 - hi_lines=""; hi_rc=0 - hi_lines="$(lane_log_events "$lane")" || hi_rc=$? - case "$hi_rc" in - 0) : ;; - 2) exit 2 ;; - *) die "history could not read lane $lane's log (exit $hi_rc). That is NOT 'this lane has written no narrative' — a read that failed is never an answer (Amendment 7(d))." 1 ;; - esac - hi_out="$(printf '%s\n' "$hi_lines" | awk -F"$US" -v since="$hi_since" ' - function pad(u) { return (length(u) == 17) ? substr(u, 1, 16) ":00Z" : u } - $3 == "NOTED" || $3 == "RULED" { - if (since != "" && pad($1) < pad(since)) next - printf "%-20s %-5s %s%s\n", $1, $3, ($8 != "" ? $8 " — " : ""), $9 - }')" - [ -n "$hi_out" ] || exit 8 - printf '%s\n' "$hi_out" + if [ -n "$lane" ]; then + lane="$(canon_lane "$lane")" || exit 2 # Amendment 15 + hi_lines=""; hi_rc=0 + hi_lines="$(lane_log_events "$lane")" || hi_rc=$? + case "$hi_rc" in + 0) : ;; + 2) exit 2 ;; + *) die "history could not read lane $lane's log (exit $hi_rc). That is NOT 'this lane has written no narrative' — a read that failed is never an answer (Amendment 7(d))." 1 ;; + esac + hi_out="$(printf '%s\n' "$hi_lines" | awk -F"$US" -v since="$hi_since" ' + function pad(u) { return (length(u) == 17) ? substr(u, 1, 16) ":00Z" : u } + $3 == "NOTED" || $3 == "RULED" { + if (since != "" && pad($1) < pad(since)) next + printf "%-20s %-5s %s%s\n", $1, $3, ($8 != "" ? $8 " — " : ""), $9 + }')" + [ -n "$hi_out" ] || exit 8 + printf '%s\n' "$hi_out" + else + hi_ev="$(state_events)" + # `--repo`: THE LANES WHOSE HOME IS IT, keyed as every other table in + # this file is — the log's own lane, lower-cased — and a home compared + # through the alias table on BOTH sides, so `openRepoTools` and + # `opensoft/openRepoTools` are one repository here as they are to + # `lanes --repo`. A lane with no recorded home is in no repository's. + hi_filter=0; hi_keep="" + if [ -n "$hi_repo" ]; then + hi_filter=1 + hi_want="$(alias_lookup "$hi_repo" 2>/dev/null || :)"; [ -n "$hi_want" ] || hi_want="$hi_repo" + lower_into "$hi_want"; hi_want_lc="$LOWER_OUT" + while IFS="$US" read -r hi_l _hi_disp _hi_v _hi_u _hi_w _hi_d _hi_p _hi_win hi_h _hi_rest; do + [ -n "${hi_l:-}" ] && [ -n "${hi_h:-}" ] || continue + hi_hc="$(alias_lookup "$hi_h" 2>/dev/null || :)"; [ -n "$hi_hc" ] || hi_hc="$hi_h" + lower_into "$hi_hc" + [ "$LOWER_OUT" = "$hi_want_lc" ] && hi_keep="$hi_keep $hi_l" + done <`. @@ -14871,15 +15142,20 @@ EOF # is why the filter is an argument rather than three implementations. # 0 with rows, 8 with none, 64 a usage error of its own. lanes) - lns_args=(); lns_fetch=0 + lns_args=(); lns_fetch=0; lns_index=0 while [ $# -gt 0 ]; do case "$1" in - --repo|--dir|--ws|--lane|--prefix) [ -n "${2-}" ] || die "$1 needs a value (usage: lanes [--repo ] [--dir ] [--prefix ] [--ws ] [--lane ] [--here] [--all] [--closed] [--fetch])" 64 + --repo|--dir|--ws|--lane|--prefix) [ -n "${2-}" ] || die "$1 needs a value (usage: lanes [--repo ] [--dir ] [--prefix ] [--ws ] [--lane ] [--here] [--all] [--closed] [--fetch] [--index])" 64 lns_args+=("$1" "$2"); shift 2 ;; --all|--here|--closed) lns_args+=("$1"); shift ;; --fetch) lns_fetch=1; shift ;; + # AMENDMENT 14(b) — THE DERIVED INDEX, ONLY WHEN TYPED. It replaces the + # PUBLISHED reads of the register and the logs and nothing else; the + # live session records, tmux and this checkout's own files are read + # exactly as they are without it, so the rows are the same rows. + --index) lns_index=1; shift ;; --) shift ;; - *) die "unknown argument '$1' for lanes (usage: lanes [--repo ] [--dir ] [--prefix ] [--ws ] [--here] [--all] [--closed] [--fetch])" 64 ;; + *) die "unknown argument '$1' for lanes (usage: lanes [--repo ] [--dir ] [--prefix ] [--ws ] [--here] [--all] [--closed] [--fetch] [--index])" 64 ;; esac done # THE ONE READ IN THIS FILE WHOSE DEFAULT IS LOCAL (SPEC rev 4 §15). Every @@ -14905,6 +15181,9 @@ EOF if [ "$lns_fetch" = 1 ] && [ "$LOG_SYNC_FETCH" != yes ] && [ "$LOG_SYNC_FETCH" != fell-back ]; then note "--fetch was asked for and NO FETCH WAS MADE: $LOG_SYNC_FETCH — reading the logs as they stand locally" fi + # AFTER the fetch, so ` commits behind` is counted against the + # `origin/` this run has just brought up to date where it asked to. + if [ "$lns_index" = 1 ]; then index_open || :; fi # AMENDMENT 15 — `--lane ` IS A `` ARGUMENT AND GOES THROUGH THE # RESOLVER. The filter inside `lanes_rows` has always joined on the name # lower-cased, so the ROWS came back either way; what a typed spelling used @@ -14946,7 +15225,7 @@ EOF case "$lns_rc" in 0) : ;; 8) exit 8 ;; - 64) die "usage: lanes [--repo ] [--dir ] [--prefix ] [--ws ] [--here] [--all] [--closed]" 64 ;; + 64) die "usage: lanes [--repo ] [--dir ] [--prefix ] [--ws ] [--here] [--all] [--closed] [--fetch] [--index]" 64 ;; *) die "the lane listing could not be read (exit $lns_rc)" 1 ;; esac [ -n "$lns_out" ] || exit 8 From e837891600429556331440a6d229ec261d4e5ff1 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:05:11 +0000 Subject: [PATCH 03/11] Place lanes-index as the sixteenth installed file `openRepoTools --install` places `lanes-index` beside the lane helpers, at the end of INSTALLABLES so every index a test takes still names the file it did. The file count moves 15 -> 16 and the artifact count 29 -> 30 wherever they are stated, with the count history extended rather than restated; the stale receipt sentence now says 30 artifacts, 28 rows. The hygiene suite also compiles `lanes-index`, the one shipped command that is Python: a syntax error there fails every workstation's first sync. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- README.md | 16 ++--- openRepoTools | 91 ++++++++++++++++------------ tests/test_install_skill_and_hook.py | 28 ++++----- tests/test_openrepotools_command.py | 31 ++++++---- tests/test_repo_hygiene.py | 14 ++++- 6 files changed, 105 insertions(+), 77 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 15a4ab9..37cd1ea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,7 +3,7 @@ Three estate commands, `park`, `resume` and `status`; the lane tooling `lanes-edit.sh`, `lane-start`, `lane-end` and `link-estates`, which came here with their history under lane-collision-protocol Amendment 9; and the -`openRepoTools` that places all fifteen files and creates the workspace they read. +`openRepoTools` that places all sixteen files and creates the workspace they read. **The verbs add no mechanics.** They find the estate and run its own `make park` / `make resume`, which run the Speckit git extension's scripts — one implementation, ruled 2026-09-09 diff --git a/README.md b/README.md index ae2bc8f..be1d5a0 100644 --- a/README.md +++ b/README.md @@ -237,14 +237,14 @@ gh api repos/opensoft/openRepoTools/contents/openRepoTools \ -H 'Accept: application/vnd.github.raw' | bash -s -- --install ``` -It places FIFTEEN files into `~/.local/bin` — `openRepoTools`, `park`, `resume`, +It places SIXTEEN files into `~/.local/bin` — `openRepoTools`, `park`, `resume`, `status`, `lane`, `lanes`, `lane-handoff`, `lane-rename`, `lanes-edit.sh`, `lane-start`, `lane-end`, `link-estates`, the alias table `repos.tsv`, -`claude-current` and `claude-restart-check` — 755, idempotently: a +`claude-current`, `claude-restart-check` and `lanes-index` — 755, idempotently: a second run prints `already installed … (unchanged)` per file, one whose bytes have drifted prints `updated at`, and one whose bytes were right and whose MODE was not prints `(mode restored to 755)`: the mode is stamped on every artifact on every -run, whether or not the bytes moved. ALL FIFTEEN ARE IN HAND BEFORE ANY IS +run, whether or not the bytes moved. ALL SIXTEEN ARE IN HAND BEFORE ANY IS PLACED, so a fetch that failed replaces nothing and names the file it could not get. A mode stamp that FAILS is a refusal naming the file it could not mode (exit 2), never the shell's own 1: every `chmod` this command performs goes @@ -259,7 +259,7 @@ THAT EXISTS, so a bin directory that is not there yet is refused for the parent that would not take it, and a create needs a directory's search bit as well as its write bit. A refusal creates none of those directories either. `cp` follows a symlink, and an install through one leaves the command uninstalled and writes -these bytes into whatever it points at. Then a `15 of 15 placed in ` line, +these bytes into whatever it points at. Then a `16 of 16 placed in ` line, and the `export PATH=…` line if that directory is not on your `PATH`. It also places **fourteen things that are not files in that directory**: THREE @@ -288,7 +288,7 @@ changes no byte of them. An entry that runs string — a second writer of one of these hooks — a file it cannot parse, or a `hooks` that is not an object → it **refuses, prints the exact block, and places nothing at all**, because both merges are computed -with the fifteen files in hand before either is placed. An installer that +with the sixteen files in hand before either is placed. An installer that repairs a file it does not understand is how you lose a setting you meant. A SECOND WRITER REFUSES WHETHER OR NOT OUR OWN ENTRY IS BESIDE IT: the read asks for a rival BEFORE it asks whether ours is already there, so a file carrying @@ -299,7 +299,7 @@ own `UserPromptSubmit` hooks are left exactly where they are, which is why that arm keys on the VERB and not on the word anywhere in a path. It never writes a profile's own `settings.json`: the launcher owns that one. -Twenty-nine artifacts, and the count is the invariant. It was sixteen until A11 +Thirty artifacts, and the count is the invariant. It was sixteen until A11 Addendum 4 ruling 9 gave `--install` a command-file list and `commands/swap.md` in it, at the same pair of paths a skill takes — because `opensoft/workBenches#74` deletes the launcher's copy and `/swap` would otherwise be installed by nobody; @@ -313,7 +313,7 @@ respawned with a new session whose first prompt is that handoff's top block. Twenty-six until **Amendment 16** (ratified the same day) put `lane-rename` on `PATH`: a lane is renamed by one word, in one commit — the row, the object log, the handoff and `lanes/aliases.tsv` — and its old name resolves for ever -afterwards, in every reader that takes a lane name. Twenty-seven, with THIRTEEN files and a `13 of 13 placed` line, until opensoft/workBenches#119 put `claude-current` and `claude-restart-check` on `PATH` (Brett Heap, 2026-09-29, verbatim *"this work is really for openRepoTools repo"*): the Claude Code a launch starts, and the RESTART NEEDED line the status line prints. +afterwards, in every reader that takes a lane name. Twenty-seven, with THIRTEEN files and a `13 of 13 placed` line, until opensoft/workBenches#119 put `claude-current` and `claude-restart-check` on `PATH` (Brett Heap, 2026-09-29, verbatim *"this work is really for openRepoTools repo"*): the Claude Code a launch starts, and the RESTART NEEDED line the status line prints. Twenty-nine, with FIFTEEN files and a `15 of 15 placed` line, until opensoft/openRepoTools#160 put `lanes-index` on `PATH` (lane-collision-protocol Amendment 14, Brett Heap, 2026-10-05, verbatim *"ratify 48"*): the derived index of the lane register, which nothing that acts reads. **And it writes down what it placed.** A RECEIPT (#57) — one row of ` ` per regular file placed, in @@ -335,7 +335,7 @@ where `raw.githubusercontent.com` is blocked. |---|---|---| | `$OPENREPOTOOLS_REPO` | `opensoft/openRepoTools` | the `owner/name` to fetch from — a fork or a mirror, named once | | `$OPENREPOTOOLS_REF` | `main` | the ref to fetch it at | -| `$OPENREPOTOOLS_BIN_DIR` | `~/.local/bin` | where `--install` puts the fifteen, and where `lane-start` looks for `claude-current` | +| `$OPENREPOTOOLS_BIN_DIR` | `~/.local/bin` | where `--install` puts the sixteen, and where `lane-start` looks for `claude-current` | | `$OPENREPOTOOLS_DATA_DIR` | `${XDG_DATA_HOME:-~/.local/share}/openRepoTools` | where `--install` writes the receipt of what it placed | | `$AGENT_PROTOCOL_ROOT` | `~/.agents` | where `workspace.yaml` lives — the one pointer to your data | | `$CLAUDE_PROFILES_HOME` | `~/.claude-profiles` | the profiles root `--install` places the shared skills under | diff --git a/openRepoTools b/openRepoTools index 6960294..3cb5293 100755 --- a/openRepoTools +++ b/openRepoTools @@ -25,7 +25,7 @@ # change (Amendment 9(b), and R-A9-1). The identity that survives is narrower # and truer. # -# `--install` PLACES FIFTEEN FILES, not four: this file, `park`, `resume`, +# `--install` PLACES SIXTEEN FILES, not four: this file, `park`, `resume`, # `status` (openRepoShape #82; the fourth under Brett Heap's RULING of # 2026-09-10 in this repository, "lets go with a fourth file"), `lane` and # `lanes` (lane-collision-protocol Amendment 18 Addendum 1 and Amendment 11's @@ -38,16 +38,18 @@ # the alias table `repos.tsv` they read — which came here from # `opensoft/brett-wip` with their history under Amendment 9(b), because a # person's workspace repository holds their DATA and the code belongs where code -# goes — and `claude-current` and `claude-restart-check` +# goes — `claude-current` and `claude-restart-check` # (opensoft/workBenches#119, Brett Heap's home ruling of 2026-09-29: the Claude -# Code a launch starts, and the RESTART NEEDED line a running session shows). It -# also places THREE skills, `/handoff`, `/lane-swap` and `/restart`, at -# two paths each, THREE command files, `/handoff`, `/ctx` and `/swap`, at that -# same pair of paths (A11 Addendum 4 ruling 9; Amendment 17(a) and (f)), and -# merges TWO hook entries into +# Code a launch starts, and the RESTART NEEDED line a running session shows) — +# and `lanes-index` (opensoft/openRepoTools#160, lane-collision-protocol +# Amendment 14, ratified by Brett Heap 2026-10-05: the DERIVED INDEX of the lane +# register, which nothing that acts reads). It also places THREE skills, +# `/handoff`, `/lane-swap` and `/restart`, at two paths each, THREE command +# files, `/handoff`, `/ctx` and `/swap`, at that same pair of paths (A11 +# Addendum 4 ruling 9; Amendment 17(a) and (f)), and merges TWO hook entries into # `~/.claude/settings.json` — the `SessionStart` one (Amendment 9(b), A8 # Addendum 2 R-A8-5) and, since lane-collision-protocol Amendment 12 adoption -# act 3, the `UserPromptSubmit` NAME GUARD beside it: TWENTY-NINE +# act 3, the `UserPromptSubmit` NAME GUARD beside it: THIRTY # artifacts, and the count is the invariant. # # One install line for the whole toolset, so a second engineer gets the estate @@ -75,7 +77,7 @@ die() { printf '\nREFUSED: %s\n' "$1" >&2; exit "${2:-2}"; } usage() { cat <<'USAGE' -openRepoTools --install install (or update) the fifteen estate and +openRepoTools --install install (or update) the sixteen estate and lane commands into ~/.local/bin, plus the /handoff, /lane-swap and /restart skills, the /handoff, /ctx and /swap command files, the @@ -86,10 +88,10 @@ openRepoTools wip init create your workspace repository, clone it, openRepoTools --help | --version `openRepoShape` is the standard's front door and scaffolds product -repositories; this command scaffolds none. `--install` places fifteen files +repositories; this command scaffolds none. `--install` places sixteen files beside each other — this command, the two verbs, the read-only view, the four -lane words, the four lane helpers, the alias table they read and the two -Claude Code launch commands: +lane words, the four lane helpers, the alias table they read, the two Claude +Code launch commands and the derived index of the lane register: park [] commit, push and RECORD an estate's open features, so another workstation can take @@ -136,6 +138,11 @@ Claude Code launch commands: claude-restart-check one RESTART NEEDED line for a running session whose binary moved on disk, for the shared status line to print + lanes-index sync|reconcile|status + the DERIVED INDEX of the lane register + (Amendment 14): written behind the sources, + read only by lanes --index and + history --index `--install` also places THREE SKILLS at two paths each — `~/.claude-profiles/shared/skills//SKILL.md` and @@ -247,7 +254,7 @@ fetch_from_repo() { return 1 } -# THE FIFTEEN FILES `--install` PLACES. One list, so the installer, the fetch +# THE SIXTEEN FILES `--install` PLACES. One list, so the installer, the fetch # fallback and the tests cannot disagree about what a complete install is. # `openRepoTools` is first because it is the one a person types to get the # others. @@ -296,7 +303,14 @@ fetch_from_repo() { # `claude-profile` call, and `claude-restart-check` is the RESTART NEEDED line # the shared status line prints for a session whose binary moved on disk. They # are placed at the END of the list, so every index a test takes into it holds. -INSTALLABLES=(openRepoTools park resume status lane lanes lane-handoff lane-rename lanes-edit.sh lane-start lane-end link-estates repos.tsv claude-current claude-restart-check) +# SIXTEEN SINCE opensoft/openRepoTools#160 (lane-collision-protocol Amendment 14, +# ratified by Brett Heap 2026-10-05, verbatim "ratify 48"): `lanes-index` keeps +# the DERIVED INDEX of the lane register — `sync`, `reconcile`, `status` — which +# `lanes-edit.sh` nudges after a write lands and which `lanes --index` and +# `lanes-edit.sh history --index` read; nothing that acts reads it. It is the one +# command in the list that is Python rather than bash, and it is placed at the +# END like the two before it, for the same reason. +INSTALLABLES=(openRepoTools park resume status lane lanes lane-handoff lane-rename lanes-edit.sh lane-start lane-end link-estates repos.tsv claude-current claude-restart-check lanes-index) # ------------------------------------------ THE RECEIPT OF WHAT IT PLACED # @@ -351,10 +365,10 @@ INSTALLABLES=(openRepoTools park resume status lane lanes lane-handoff lane-rena # it went stale the next time a list moved (Copilot on #103, `openRepoTools:325`, # which found twenty-six where there were twenty-seven): `ARTIFACTS - HOOK_ENTRIES` # in `tests/test_openrepotools_command.py`, which is `len(INSTALLED) + 2 * -# len(SKILL_NAMES) + 2 * len(COMMAND_NAMES)`. Today that is 27 artifacts, 25 of -# them files, 25 rows — 13 + 6 + 6, since Amendment 16 put `lane-rename` in -# `INSTALLABLES` — and it moves with `INSTALLABLES`, `SKILLS` and `COMMANDS`, -# and with nothing else. +# len(SKILL_NAMES) + 2 * len(COMMAND_NAMES)`. Today that is 30 artifacts, 28 of +# them files, 28 rows — 16 + 6 + 6, since opensoft/openRepoTools#160 put +# `lanes-index` in `INSTALLABLES` — and it moves with `INSTALLABLES`, `SKILLS` +# and `COMMANDS`, and with nothing else. # # A RECEIPT THAT CANNOT BE WRITTEN IS A NOTE AND NEVER A REFUSAL (#57's own # words, and the same rule the retirement follows one function down). The @@ -968,7 +982,7 @@ mode_is() { # install_commands() { local dir="${OPENREPOTOOLS_BIN_DIR:-$HOME/.local/bin}" local name target source verb placed=0 - # ALL TWENTY-NINE IN HAND BEFORE ANY IS PLACED. The fifteen files, the + # ALL THIRTY IN HAND BEFORE ANY IS PLACED. The sixteen files, the # three skills, the three command files, and the TWO artifacts that are not # whole files — the two hook entries merged into one `settings.json` — # are all computed here, so a fetch that failed replaces nothing and a merge @@ -1067,7 +1081,7 @@ install_commands() { # ========================================================================= -# THE FOURTEEN ARTIFACTS THAT ARE NOT ONE OF THE FIFTEEN FILES +# THE FOURTEEN ARTIFACTS THAT ARE NOT ONE OF THE SIXTEEN FILES # ========================================================================= # # lane-collision-protocol Amendment 9(b), inheriting A8 Addendum 2's R-A8-5 @@ -1085,7 +1099,7 @@ install_commands() { # run and preserves hook keys. This command never writes a profile's # settings.json at all. One path, one writer, in both halves. # -# TWENTY-NINE ARTIFACTS, AND THE COUNT IS THE INVARIANT: fifteen files in the bin +# THIRTY ARTIFACTS, AND THE COUNT IS THE INVARIANT: sixteen files in the bin # directory, three skills in the shared skills directory, their three bare-run # copies, THREE COMMAND FILES at those same two paths, and TWO merged entries in # `~/.claude/settings.json`. It was twelve before Amendment 11 added a SECOND @@ -1095,8 +1109,9 @@ install_commands() { # `UserPromptSubmit` name guard beside the `SessionStart` entry, and nineteen # before Amendment 17 added `lane-handoff`, the `handoff` skill and the # `/handoff` and `/ctx` command files, twenty-six before Amendment 16 added -# `lane-rename`, and twenty-seven before opensoft/workBenches#119 added -# `claude-current` and `claude-restart-check`; the count moves +# `lane-rename`, twenty-seven before opensoft/workBenches#119 added +# `claude-current` and `claude-restart-check`, and twenty-nine before +# opensoft/openRepoTools#160 added `lanes-index`; the count moves # with the lists rather than being restated from memory. # # WHY A COMMAND FILE IS HERE AT ALL, WHEN ADOPTION ACT 6 SAYS THE LAUNCHER @@ -1196,7 +1211,7 @@ BLOCK # EVERY SKILL'S BYTES, by `--install`'s own rule: from the checkout beside this # file when run from one, else fetched at this same ref. ALL OF THEM IN HAND -# BEFORE ANY IS PLACED, which is the same all-or-nothing rule the fifteen files +# BEFORE ANY IS PLACED, which is the same all-or-nothing rule the sixteen files # already have — a new `/restart` beside a `handoff` that could not be fetched # is a half-install that reads like a whole one. collect_skills() { @@ -1262,8 +1277,8 @@ HOOK_PLAN="" # "absent" | "present" | "nofile" GUARD_PLAN="" # the same three, for Amendment 12's UserPromptSubmit entry HOOK_MERGED="" # a staged whole file, ready to move into place -# THE DESTINATIONS, PROVED WRITABLE BEFORE ANY OF THE FIFTEEN IS PLACED — every -# directory the artifacts that are not one of the fifteen files go into, derived +# THE DESTINATIONS, PROVED WRITABLE BEFORE ANY OF THE SIXTEEN IS PLACED — every +# directory the artifacts that are not one of the sixteen files go into, derived # from the two lists and NOT counted in this sentence. It said "THE THREE # DESTINATIONS" while the loop below built seven of them, which is the same # defect `collect_commands` names above: a number restated beside a list that @@ -1306,7 +1321,7 @@ plan_skill_targets() { # `~/.claude/skills/restart/SKILL.md` was followed and WRITTEN THROUGH: the # skill stays uninstalled, the target stays a link, and the bytes land # wherever it points. That is the same failure R-A9-12 was ruled on for the - # fifteen in the bin directory, one directory along, and the ruling says + # sixteen in the bin directory, one directory along, and the ruling says # nothing about which path it applies to. # # `unplaceable_kind` is asked rather than re-tested, because two copies of @@ -1346,7 +1361,7 @@ $bad rm -f -- $paths" } -# THE FIFTEEN TARGETS, PROVED PLACEABLE BEFORE ANY OF THE TWENTY-NINE IS PLACED +# THE SIXTEEN TARGETS, PROVED PLACEABLE BEFORE ANY OF THE THIRTY IS PLACED # (A9 Addendum 4, R-A9-12, ruled on F5 of the #24 review). # # `cp` FOLLOWS A SYMLINK. The loop below tests `[ -e "$target" ]`, compares @@ -1371,7 +1386,7 @@ $bad # `rm` that clears them — rather than leaving a person to remember a step. # WHAT IS AT A PATH, WHERE IT IS NOT A REGULAR FILE. # ONE implementation of R-A9-12's test, for every path this command writes: the -# fifteen in the bin directory, the six `SKILL.md`s (F-X17), the six command +# sixteen in the bin directory, the six `SKILL.md`s (F-X17), the six command # files, and `~/.claude/settings.json` (#44 round 1, `openRepoTools:878`). # Prints what is there and returns 0; returns 1 — saying nothing — where the # path is absent or is a regular file. @@ -1406,9 +1421,9 @@ path_kind() { # # This answered "a regular file" with `return 1` and asked nothing else, so # every existing file counted as one `--install` may place over — and # `plan_skill_targets` let a READ-ONLY `SKILL.md` or command file through. -# `install_commands` has already copied the fifteen PATH files by the time +# `install_commands` has already copied the sixteen PATH files by the time # `place_skill_and_hook` reaches the `cp`, so an ordinary permission failure -# there left a host with fifteen of the twenty-nine artifacts placed, on the one +# there left a host with sixteen of the thirty artifacts placed, on the one # path in this file that promises all or none. A mode is not an exotic state # either: `chmod 444` on a file a person did not want overwritten is what a # person does. @@ -1581,7 +1596,7 @@ $(guard_block_text)" # WHAT IS AT THAT PATH, BEFORE A BYTE OF IT IS READ (#44 round 1, # `openRepoTools:878`). This file carries the LAST TWO artifacts — the - # twenty-eighth and twenty-ninth — since Amendment 12 adoption act 3 put the + # twenty-ninth and thirtieth — since Amendment 12 adoption act 3 put the # name guard beside the `SessionStart` entry, and it is the only path no # walk asked about; the # `present` arm now CHMODs it, and `chmod` @@ -1781,7 +1796,7 @@ $(guard_block_text)" } # The fourteen artifacts that are not bin-directory files, in thirteen writes -# (the two hook entries go into settings.json in one), AFTER the fifteen files +# (the two hook entries go into settings.json in one), AFTER the sixteen files # and the merge are all in hand. place_skill_and_hook() { local shared_dir bare_dir settings target verb hook_tmp name @@ -1793,7 +1808,7 @@ place_skill_and_hook() { # EVERY FAILURE HERE IS A `die`, never the shell's own 1. `--install`'s # codes are this toolset's — 0 done, 1 findings printed, 2 a refusal — and # an unguarded `mkdir` or `cp` under `set -e` exits 1, which reads as a - # report of findings from a command that placed fifteen of twenty-nine artifacts. + # report of findings from a command that placed sixteen of thirty artifacts. for target in "$shared_dir/SKILL.md" "$bare_dir/SKILL.md"; do mkdir -p -- "$(dirname -- "$target")" || die "could not create $(dirname -- "$target") for the $name skill. @@ -1818,7 +1833,7 @@ place_skill_and_hook() { the skills and the hook are not." fi # 644: a SKILL.md is a document a session READS, not a command. The - # fifteen in the bin directory are stamped 755 for the reason that loop + # sixteen in the bin directory are stamped 755 for the reason that loop # gives — "a copy that is not executable is not a command" — and # that reason does not reach this file. Amendment 9(b) describes # `--install` as stamping 755 "on everything it places" while @@ -1840,7 +1855,7 @@ place_skill_and_hook() { # reason a skill has two. # # 644 for the same reason a `SKILL.md` is: a command file is a document a - # session READS. The 755 argument belongs to the fifteen in the bin + # session READS. The 755 argument belongs to the sixteen in the bin # directory, where a copy that is not executable is not a command. for name in "${COMMANDS[@]}"; do for target in "$(skills_home)/shared/commands/$name.md" "$(claude_home)/commands/$name.md"; do @@ -1884,8 +1899,8 @@ place_skill_and_hook() { # round 1, the suppressed comment at `openRepoTools:880`): a # `settings.json` this euid can write but does not own takes `jq` and # every check `plan_hook_merge` puts to it, then refuses the mode - # stamp, with the fifteen commands, the skills and the command files — - # twenty-seven of the twenty-nine artifacts — already placed. The guard that + # stamp, with the sixteen commands, the skills and the command files — + # twenty-eight of the thirty artifacts — already placed. The guard that # would catch it first is the same `[ -O ]` `unplaceable_kind`'s own # comment declines, for the same reason: no test in this suite can # make a second account to own this file, and the check would refuse a diff --git a/tests/test_install_skill_and_hook.py b/tests/test_install_skill_and_hook.py index b04f15c..99cfd5b 100644 --- a/tests/test_install_skill_and_hook.py +++ b/tests/test_install_skill_and_hook.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -"""`--install`'s artifacts that are not one of the FIFTEEN files: the three +"""`--install`'s artifacts that are not one of the SIXTEEN files: the three skills at two paths each, the three command files at two more each, and the TWO merged hook entries. @@ -17,7 +17,7 @@ on the command string. So the tests below are about the four answers that string can have — present, absent, differing, unreadable — asked of EACH entry, and about the one rule that makes a wrong answer survivable: both merges are -computed with the fifteen files in hand, BEFORE any of them is placed, so a +computed with the sixteen files in hand, BEFORE any of them is placed, so a refusal costs a whole install rather than half of one — and, since the pair, a settings file never passes through a state carrying one entry of the two. @@ -550,7 +550,7 @@ def test_a_differing_session_start_entry_refuses_and_places_nothing(tmp_path): twice. AND THE COST IS A WHOLE INSTALL, NOT HALF OF ONE: both merges are computed - with the fifteen files in hand, before any of them is placed, so the bin + with the sixteen files in hand, before any of them is placed, so the bin directory is untouched. That is the same all-or-nothing rule `--install` already had, extended to the two artifacts that are not whole files. """ @@ -663,16 +663,16 @@ def test_a_destination_that_cannot_be_written_refuses_before_anything_is_placed( Amendment 9(b) computes the merge in hand "so a merge that cannot be computed refuses having placed nothing". A filesystem offers no transaction - across twenty-nine artifacts, so nothing can make the last fourteen atomic - with the first fifteen — but the failure that actually happens is not an + across thirty artifacts, so nothing can make the last fourteen atomic + with the first sixteen — but the failure that actually happens is not an exotic one, it is a directory that is not this installer's to write, and that question can be asked in the planning phase where the refusal still costs nothing. - Without the check the run places fifteen files and some of the remaining + Without the check the run places sixteen files and some of the remaining artifacts, then dies — leaving a host with commands installed, neither hook entry, and an installer that reports the same "already - installed (unchanged)" for the fifteen on every re-run while never reaching + installed (unchanged)" for the sixteen on every re-run while never reaching the one that failed. """ if which == "shared skills": @@ -687,13 +687,13 @@ def test_a_destination_that_cannot_be_written_refuses_before_anything_is_placed( assert result.returncode == 2, result.stdout + result.stderr assert "NOTHING was installed" in result.stderr assert not bin_dir.exists() or not any(bin_dir.iterdir()), ( - "the fifteen files were placed against a destination that was never " + "the sixteen files were placed against a destination that was never " "going to take the other fourteen") finally: blocked.chmod(0o700) -# --- the fifteen targets, and what they are (R-A9-12) ---------------------- +# --- the sixteen targets, and what they are (R-A9-12) ---------------------- @NEEDS_JQ @NOT_ROOT @@ -714,7 +714,7 @@ def test_a_leaf_destination_that_exists_unwritable_refuses_before_anything_is_pl The bin directory is the assertion that tells the two apart: the refusal here is a PLANNING one and nothing is placed, where the mutant places all - fifteen files and dies on the `cp` into this same directory, which is the + sixteen files and dies on the `cp` into this same directory, which is the half-install the planning phase exists to prevent. """ blocked = tmp_path / ".claude-profiles" / "shared" / "skills" / SKILL_DIR_NAME @@ -727,7 +727,7 @@ def test_a_leaf_destination_that_exists_unwritable_refuses_before_anything_is_pl assert "is not writable" in result.stderr, result.stderr assert "NOTHING was installed" in result.stderr assert not bin_dir.exists() or not any(bin_dir.iterdir()), ( - "the fifteen files were placed against a leaf directory that was " + "the sixteen files were placed against a leaf directory that was " "never going to take the skill") finally: blocked.chmod(0o700) @@ -796,7 +796,7 @@ def test_a_symlinked_target_is_refused_and_nothing_is_written_through_it(tmp_pat def test_a_directory_where_a_command_goes_is_refused_the_same_way(tmp_path): """THE RULE IS `A REGULAR FILE`, not `not a symlink`. A directory at `$BIN/park` is the same refusal for the same reason — `cp` cannot place a - file over it, and finding that out after fourteen of the fifteen are placed is + file over it, and finding that out after fifteen of the sixteen are placed is the half-install the planning phase exists to prevent.""" bin_dir = tmp_path / ".local" / "bin" (bin_dir / "park").mkdir(parents=True) @@ -1057,7 +1057,7 @@ def test_a_symlinked_skill_target_is_refused_and_nothing_written_through_it( tmp_path, shared, name): """R-A9-12 IS ABOUT WHAT `cp` DOES, NOT ABOUT WHICH DIRECTORY (F-X17). - `plan_install_targets` refuses a symlink for every one of the fifteen files + `plan_install_targets` refuses a symlink for every one of the sixteen files in the bin directory. `plan_skill_targets` proved only the DIRECTORIES writable, and `place_skill_and_hook` then reached each `SKILL.md` with `[ -e ]`, `cmp -s` and `cp` — none of which can tell a regular file from a @@ -1091,7 +1091,7 @@ def test_a_symlinked_skill_target_is_refused_and_nothing_written_through_it( f"the refusal must print the exact `rm` that clears it:\n{result.stderr}") assert far.read_bytes() == before, f"--install wrote through the link into {far}" assert target.is_symlink(), f"{target} is no longer the link it was" - # AND IT REFUSED IN THE PLANNING PHASE: the fifteen commands never arrived + # AND IT REFUSED IN THE PLANNING PHASE: the sixteen commands never arrived # either, which is what makes `NOTHING was installed` true rather than # nearly true. assert not bin_dir.exists() or not any(bin_dir.iterdir()), ( diff --git a/tests/test_openrepotools_command.py b/tests/test_openrepotools_command.py index da0cbbe..640c6c3 100644 --- a/tests/test_openrepotools_command.py +++ b/tests/test_openrepotools_command.py @@ -61,11 +61,15 @@ #: rename is four files in one commit — which made the list THIRTEEN. #: `claude-current` and `claude-restart-check` joined it for #: opensoft/workBenches#119 (Brett Heap's home ruling of 2026-09-29), at the END -#: so every index taken below still names the file it did: the list is FIFTEEN. +#: so every index taken below still names the file it did, which made the list +#: FIFTEEN. `lanes-index` joined it for opensoft/openRepoTools#160 +#: (lane-collision-protocol Amendment 14, ratified 2026-10-05), at the END for +#: the same reason, and it is the one command in the list that is Python rather +#: than bash: the list is SIXTEEN. INSTALLED = ("openRepoTools", "park", "resume", "status", "lane", "lanes", "lane-handoff", "lane-rename", "lanes-edit.sh", "lane-start", "lane-end", "link-estates", "repos.tsv", "claude-current", - "claude-restart-check") + "claude-restart-check", "lanes-index") #: The skills `--install` also places, at two paths each, and the paths they are #: fetched from when there is no checkout to copy them out of (Amendment 9(b), @@ -87,11 +91,11 @@ COMMAND_NAMES = ("handoff", "ctx", "swap") COMMAND_PATHS = tuple(f"commands/{n}.md" for n in COMMAND_NAMES) -#: Everything a stdin install has to fetch: the fifteen files, the three skills +#: Everything a stdin install has to fetch: the sixteen files, the three skills #: and the three command files. FETCHED = INSTALLED + SKILL_PATHS + COMMAND_PATHS -#: TWENTY-NINE ARTIFACTS, AND THE COUNT IS THE INVARIANT: fifteen files in the +#: THIRTY ARTIFACTS, AND THE COUNT IS THE INVARIANT: sixteen files in the #: bin directory, three skills in the shared skills directory, their three #: bare-run copies, three command files at that same pair of destinations, and #: TWO merged entries in `~/.claude/settings.json`. Derived from the three @@ -110,7 +114,7 @@ + HOOK_ENTRIES) USAGE_LINES = ( - "openRepoTools --install install (or update) the fifteen estate and", + "openRepoTools --install install (or update) the sixteen estate and", "openRepoTools wip init create your workspace repository, clone it,", "openRepoTools --help | --version", ) @@ -123,7 +127,7 @@ WINDOWS_SKIP] #: `--install` HARD-REQUIRES `jq` SINCE lane-collision-protocol AMENDMENT 9(b): -#: two of its twenty-nine artifacts are merged entries inside a JSON file somebody +#: two of its thirty artifacts are merged entries inside a JSON file somebody #: else owns, and the clause has it refuse naming `jq` rather than rewriting #: that file by hand. So a run of `--install` on a host without `jq` is a #: REFUSAL BY DESIGN, and a test that asserts a successful placement there is @@ -221,7 +225,7 @@ def test_help_prints_every_usage_line(): def test_help_names_every_command_it_places_and_the_standards_front_door(): - """`--install` places fifteen files, and fourteen of them are commands this one + """`--install` places sixteen files, and fifteen of them are commands this one knows nothing about — so `--help` has to say what they are and where the rest is written down. A command a person has on PATH and cannot find written down is a command they will not use. @@ -238,7 +242,8 @@ def test_help_names_every_command_it_places_and_the_standards_front_door(): for line in ("park []", "resume []", "status []", "lane-start ", "lane-end ", "lanes-edit.sh ", "link-estates", "repos.tsv", - "claude-current [--porcelain]", "claude-restart-check"): + "claude-current [--porcelain]", "claude-restart-check", + "lanes-index sync|reconcile|status"): assert line in result.stdout, line assert "`openRepoShape` is the standard's front door" in result.stdout assert "this command scaffolds none" in result.stdout @@ -411,7 +416,7 @@ def test_installing_twice_changes_nothing(tmp_path): assert second.returncode == 0, second.stderr for name in INSTALLED: assert f"{name}: already installed at" in second.stdout, name - # TWENTY-NINE, not fifteen: the six skill copies, the six command-file copies + # THIRTY, not sixteen: the six skill copies, the six command-file copies # and BOTH hook entries each report `unchanged` too, and the count is the # invariant Amendment 9(b) names — derived from the three lists, never # restated, so a new skill or command moves it. It was eighteen until @@ -424,7 +429,7 @@ def test_installing_twice_changes_nothing(tmp_path): @NEEDS_JQ def test_install_replaces_a_copy_that_has_drifted(tmp_path, name): """Per file, and only the one that drifted: an install that rewrote all - fifteen every time would have nothing to say about which one was stale.""" + sixteen every time would have nothing to say about which one was stale.""" assert run_cmd("--install", home=tmp_path).returncode == 0 target = tmp_path / ".local" / "bin" / name target.write_text(target.read_text(encoding="utf-8") + "# drift\n", @@ -848,9 +853,9 @@ def test_install_writes_a_receipt_of_every_file_it_placed(tmp_path): command files at both of their destinations. The two hook entries are entries inside `~/.claude/settings.json` and not files this command placed, and the receipt carries no row for itself either. Today that is - 27 artifacts and 25 rows (13 + 6 + 6, since Amendment 16 put `lane-rename` - in the list); the number moves with those three lists and with nothing - else, which is why no assertion below spells it. + 30 artifacts and 28 rows (16 + 6 + 6, since opensoft/openRepoTools#160 put + `lanes-index` in the list); the number moves with those three lists and + with nothing else, which is why no assertion below spells it. """ result = run_cmd("--install", home=tmp_path) assert result.returncode == 0, result.stderr diff --git a/tests/test_repo_hygiene.py b/tests/test_repo_hygiene.py index fc681fa..82775c2 100644 --- a/tests/test_repo_hygiene.py +++ b/tests/test_repo_hygiene.py @@ -2696,15 +2696,23 @@ def test_every_shipped_bash_file_is_tracked_with_lf(): def test_the_python_files_compile(): - """The suite is the only Python here, and a file that does not parse is a - file that fails as a collection ERROR rather than as a test — which reads - as the suite being broken instead of as one file being wrong. + """The suite is the Python here, bar one shipped command named below, and a + file that does not parse is a file that fails as a collection ERROR rather + than as a test — which reads as the suite being broken instead of as one + file being wrong. `compile()` rather than `py_compile`: nothing is written anywhere, so this runs in a read-only checkout and leaves no `__pycache__` behind. + + opensoft/openRepoTools#160: `lanes-index` is the one shipped command that + is Python, and it has no `.py` suffix for the glob below to find, so its + text is compiled the same way — a syntax error there is a command that + fails on every workstation's first sync. """ for path in sorted((REPO / "tests").glob("*.py")): compile(path.read_text(encoding="utf-8"), str(path), "exec") + command = REPO / "lanes-index" + compile(command.read_text(encoding="utf-8"), str(command), "exec") # --- the macOS job RUNS this bash, it does not only parse it (R-A9-11) ------ From be3a9d80690ec8d2870b6bf0973742c6be41e190 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:05:11 +0000 Subject: [PATCH 04/11] Document the derived index in the lanes manual "The derived index (Amendment 14)": what it is for and what it never does, the three words and the nudge, `--index` and its two stderr lines, the configuration and its refusals, the two Postgres roles with their GRANTs, the schema (the ledger half's tables and #161's `worktrees` created empty), and the eight tests' promise. The `history` usage gains `--all` and `--repo`. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- docs/README-lanes.md | 168 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/docs/README-lanes.md b/docs/README-lanes.md index 244e1ad..9afe3cf 100644 --- a/docs/README-lanes.md +++ b/docs/README-lanes.md @@ -2324,6 +2324,8 @@ RULED — lane , session @, , lane:[ → ] LANES_LANE=my-lane lanes-edit.sh log NOTED lane:my-lane "rebased on main; the flake is the fixture's clock" LANES_LANE=my-lane lanes-edit.sh log RULED lane:my-lane → opensoft/openRepoTools#29 "Ratify as drafted (Recommended)" lanes-edit.sh history my-lane [--since ] +lanes-edit.sh history --all [--since ] # every lane, on one timeline (Amendment 14) +lanes-edit.sh history --repo # the lanes whose home it is ``` `NOTED` takes no payload; `RULED` takes one, the object it bears on, and that @@ -3127,6 +3129,172 @@ What the managed ledger itself does with an enrolled lane is not this manual's: branch `001-separate-swap-ctx-handoff`'s governance review is **PROPOSED — NOT APPROVED**, and nothing here asserts that it supersedes Amendment 17. +## The derived index (Amendment 14) + +**In force — ratified by Brett Heap 2026-10-05, verbatim *"ratify 48"*** +(brettheap/new-workstation#48; the tooling is opensoft/openRepoTools#160). +**The register's truth stays where it is.** `lanes/LANES.md`, `lanes/log/`, +`lanes/aliases.tsv` and `lanes/archive/` are the record, written only by +`lanes-edit.sh`. `lanes-index` keeps a **derived index** of them — SQLite on +local disk, or QA Postgres where this workstation is configured for it — +written behind the sources, read only by tooling that decides nothing, and +never a gate. Where the index and the source disagree, the source is right and +the index is behind. + +### What it is for, and what it never does + +It is for the questions no act asks: every repository's rows at once, every +lane's history on one timeline, a dashboard, Eagle and Raven asking one store. +It is **never read by an act**: no `claim`, `release`, `log` verb or LANES +line; not the name guard or the `SessionStart` block; not `lane`, `lane-start`, +`lane-end`, `lane-handoff`, `lane-rename`, `/restart`, `/handoff`, `/ctx`; no +row writer, sweep, archive or migration; and none of the reads an act makes for +itself (`who`, `binding`, `live-holder`, `next-free`, `managed-projection`, +`lane-reconcile`). An index that is missing, stale, unreachable, wiped or wrong +changes no exit code and no output of any of them. It never writes back: +nothing in any source is ever written from it. + +### The three words, and the one step `lanes-edit.sh` gained + +```sh +lanes-index sync [--source ] # write behind what LANDED +lanes-index reconcile [--source ] [--dry-run] # read the source whole; put the index right +lanes-index status # per source: store, provenance, lag, last error +``` + +- **`sync`** reads the register as `origin/` of the workspace checkout + stands — what the push left, never the working tree — and upserts every + record whose content differs, keyed by **(source, record id)** and versioned + by a **digest of its content**: an upsert bearing the stored digest changes + nothing, so a replay or a crash is harmless. The source's indexed commit is + compared and swapped **in the same transaction** as the rows it covers, and + a sync from a commit that does not descend from it **writes nothing** — so + two workstations syncing into one Postgres, even at once, cannot roll it + back. A record its source no longer holds leaves in the same sync. One sync + at a time per workstation: a second finds the lock held, leaves the holder a + mark to sync again, and returns at once. +- **`reconcile`** reads the source whole, compares every stored row by what it + **says** (not by the digest it carries, so a row changed by hand is found), + upserts what differs and removes what is gone: on a wiped index, the + rebuild. `--dry-run` prints the counts and writes nothing. +- **`status`** prints the store, each source's provenance against its tip, the + lag in commits, and this workstation's last sync and last error. + +**The nudge.** After every write whose push **landed**, and after its mutex is +released, `lanes-edit.sh` starts **one detached `lanes-index sync`**: stdin +closed, stdout and stderr on `/dev/null`, never waited, its status never read. +It carries no data. It is skipped where `lanes-index` is not on `PATH` (one +`command -v`, nothing printed) and wherever `LANES_INDEX=off`. A slow, failing +or absent indexer delays and fails no commit, push or swap. + +### Reading it — `--index`, only when typed + +```sh +lanes --index # any listing, out of the index +lanes-edit.sh history --index +lanes-edit.sh history --all [--since ] [--index] # every lane, one timeline +lanes-edit.sh history --repo [--index] # the lanes whose HOME it is +``` + +The flag is typed per invocation — **never** taken from the environment or a +configuration. It replaces the **published** reads (the register, the archive, +the alias table, the logs) and nothing else, so the live session records, tmux +and this checkout's own files are read as they are without it. Where the index +is current the answer is **byte for byte the source read's**, the clock's +columns aside (the listing's AGE and its "last heard from origin" age, computed +the same way at the moment of the read). One line on stderr says which was +read: + +```text +read: index (sqlite) at register@1a2b3c4d5e6f; 0 commits behind +read: sources (index unreachable: psql: error: connection to server … failed) +``` + +Lag is said, never refused. Where the index cannot answer — `lanes-index` not +installed, the store `missing`, `unreachable`, `wiped` (no provenance for this +register), of an `unknown-schema`, a configuration it `refused`, or +`LANES_INDEX=off` — the line says so and the **sources** answer, with the +source read's own exit. `history --all` and `--repo` read the sources without +the flag; `history` over two lanes used to be refused. + +### The store and its configuration + +`${XDG_CONFIG_HOME:-~/.config}/openRepoTools/lanes-index.conf`, or the file +`$LANES_INDEX_CONFIG` names — per workstation, **never committed**, never under +`~/.agents/` (which on a workstation built from the workspace repository *is* +that repository): + +```text +store=sqlite|postgres default: postgres where url= is set, else sqlite +sqlite= default ${XDG_STATE_HOME:-~/.local/state}/openRepoTools/lanes-index.sqlite +url= the WRITER role — sync and reconcile +passfile= the 0600 libpq password file for it +reader_url= the SELECT-only READER role — status and the --index reads +reader_passfile= its password file, where it is not the same one +schema= the Postgres schema (default lanes_index) +``` + +Refused (exit 2, nothing read or written): a configuration that is group- or +world-readable, inside a git work tree or under `~/.agents/`; a URL that +carries a password in any spelling; a password file that is not 0600; a SQLite +path inside a git work tree. Postgres is reached through `psql` and only where +`url=` is set — nothing imports a driver — with `PGPASSFILE` naming the +password file and an inherited `PGPASSWORD` removed. No configuration at all +is SQLite at its default path, created 0600. + +**The two roles are Postgres's to hold** (act 3, Brett Heap's on Eagle and +Raven — or nothing, and SQLite): + +```sql +CREATE ROLE lanes_index_writer LOGIN; -- the password lives in the 0600 passfile only +CREATE ROLE lanes_index_reader LOGIN; +CREATE SCHEMA lanes_index AUTHORIZATION lanes_index_writer; +GRANT USAGE ON SCHEMA lanes_index TO lanes_index_reader; +ALTER DEFAULT PRIVILEGES FOR ROLE lanes_index_writer IN SCHEMA lanes_index + GRANT SELECT ON TABLES TO lanes_index_reader; +``` + +The writer is confined to the index's schema and held by the indexer alone — +by no session, supervisor, launcher or ledger authority; a leaked writer can +damage only what a `reconcile` rebuilds. The reader may only `SELECT`. + +### What is indexed (schema 1) + +One versioned schema (`schema_version`); every row carries its `provenance` +(the register commit it was last written at), `indexed_utc` and +`schema_version`, and its `digest`. + +| table | one row per | what it holds | +|---|---|---| +| `lanes` | lane | canonical name, `aliases` (Amendment 16(e)), `home`, the `workstation / env / user` cell, `started_utc`, `handoff_path` (a pointer), `state_phrase`, `objects_cell`, `session_ids` in order, `last_session` (the resume target), `archived`, the row's own text — and **`owner`**: `legacy`, `managed ` or `unknown`, from `lanes-edit.sh managed-projection` (0, 8, 1) and never from a parser of the indexer's, so a malformed marker is never downgraded to `legacy` | +| `log_lines` | log line, by lane and file ordinal | verb, session, workstation, UTC as written, object, payload, its sub-fields as JSON, free text; a line the grammar cannot read is kept as `unreadable` with its file, line and reason, never dropped | +| `register_lines` | register line, by ordinal | Rule 6's `LANDING`/`LANDED` and Amendment 10's `HOLD`/`HOLD RELEASED`: lane, repository, PR, tip, UTC, the line itself | +| `lane_aliases` | alias-table line | `old_name`, `new_name`, UTC | +| `transcript_pointers` | pointer | a row's session-cell ids and each `transcript` a `PAUSED` names, with its `agent` — pointers only | +| `source_files` | file of the source | kind, blob, line counts — what the read flags need to answer as `origin` would | +| `holds` (a VIEW) | open hold | per lane, the last line on each object **in file order**, never by UTC, where its verb is open — computed, never stored | +| `provenance` | source | the indexed commit (or generation), UTC, schema version | + +**Created empty and written by nothing here**: `swap_states` and +`job_summaries` — the managed ledger's half, `ledger:`, which the +001 feature's T058 writes (act 4) — and `worktrees`, **reserved for +opensoft/openRepoTools#161** (each lane's #97 inventory under +`inventory:`). Not indexed at all: transcript and handoff bodies, +credentials, the ledger's claims, fences, releases and exit witnesses. + +### The tests' promise + +`tests/test_lanes_index.py`, and a section of `tests/test_lane_helpers.sh` for +the acts only that suite drives, hold the index to Amendment 14's eight: +**offline no-read** (every act byte-identical against an unreachable store), +**poisoned index** (wrong rows change no act's answer), **wiped-index +reconcile** (the rebuild equals a synced index row for row), **replay**, +**monotonic provenance**, **a hung indexer** delaying no writer beyond the fork, +**the managed seam** (valid, malformed and plain rows as `managed`, `unknown`, +`legacy`, all through `managed-projection`), and **fallback** (each failure +says `read: sources` and gives the source read's answer). The parser is held +to `LOG_AWK`'s output line for line. No test creates a real database. + ## Hand edits After **any** hand edit made with an allowed tool (python read/write, `sed -i From 852eee229a84a4c4d4a6f275af05348a5781a3c1 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:05:11 +0000 Subject: [PATCH 05/11] Test the derived index against Amendment 14's eight promises tests/test_lanes_index.py builds a whole estate in tmp_path and holds the indexer to offline no-read, poisoned index, wiped-index reconcile, replay, monotonic provenance, a hung indexer, the managed seam and fallback, plus LOG_AWK parity, byte-identical --index reads, the configuration refusals and the Postgres writer/reader split against a fake psql. No real database is created. The shell suite gains a section running lane-start --dry-run, the name guard, writes and reads three ways from one snapshot: no index, an installed indexer against an unreachable store, and a poisoned SQLite. Both the suite and conftest set LANES_INDEX=off so no other case nudges an installed indexer. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- tests/conftest.py | 8 + tests/test_lane_helpers.sh | 145 +++++ tests/test_lanes_index.py | 1067 ++++++++++++++++++++++++++++++++++++ 3 files changed, 1220 insertions(+) create mode 100644 tests/test_lanes_index.py diff --git a/tests/conftest.py b/tests/conftest.py index ce8c77e..e2c9292 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,14 @@ REPO = Path(__file__).resolve().parents[1] +#: AMENDMENT 14 — NO TEST NUDGES AN INSTALLED INDEXER BY ACCIDENT. Every +#: `lanes-edit.sh` write that pushes starts a detached `lanes-index sync` +#: wherever one is on PATH, and several suites here run with the workstation's +#: own PATH behind their fakes: the day a person installs `lanes-index`, those +#: writes would start one each. Off for the whole run; `tests/test_lanes_index.py` +#: builds every environment it runs from scratch and decides for itself. +os.environ["LANES_INDEX"] = "off" + #: The pinned openRepoShape checkout. Every path this suite reads out of the #: standard hangs off this one name, so a bump of the pin moves one line in #: `contracts/openreposhape-pin.yaml` and the gitlink beside it, and nothing diff --git a/tests/test_lane_helpers.sh b/tests/test_lane_helpers.sh index 90f1718..facf037 100755 --- a/tests/test_lane_helpers.sh +++ b/tests/test_lane_helpers.sh @@ -443,6 +443,14 @@ export TMUX="$SANDBOX/fake-tmux-socket,0,0" # it if a case switches the variable off on purpose — which exactly one does. export LANES_NO_GITHUB=1 +# AMENDMENT 14 — NO CASE HERE NUDGES AN INDEXER BY ACCIDENT. Every write in this +# file that pushes would start a detached `lanes-index sync` wherever one is on +# PATH, and the PATH above is the workstation's own with the sandbox's in front +# of it: the day a person installs `lanes-index`, a thousand writes would start +# a thousand of them. Off for the file, and the one section that is about the +# index turns it back on for its own cases. +export LANES_INDEX=off + # Amendment 8, ruling (g): the harness exports `CLAUDE_CODE_SESSION_ID` into # every shell a session runs, INCLUDING the one running this suite. It is a # holder test now, so a case that does not set it deliberately must not inherit @@ -14560,6 +14568,143 @@ is "…and no pane was respawned for a lane no read of it is unambiguous abou is "…and no log was written for either spelling" \ "$(ls "$LOGD" | grep -ci '^repohf-13\.md$' || :)" 0 +echo "== Amendment 14: the derived index is never read by an act ==" + +# lane-collision-protocol Amendment 14, ratified by Brett Heap 2026-10-05, +# verbatim "ratify 48" (opensoft/openRepoTools#160). The register's truth stays +# where it is and `lanes-index` keeps a DERIVED copy that NO ACT READS OR WAITS +# ON (clause (b)). `tests/test_lanes_index.py` holds the indexer to that over +# its own small estate; this section holds the ACTS that only this suite can +# drive — `lane-start --dry-run` and the name guard, beside a write and the +# reads every act makes — to it over THIS estate, three ways from one snapshot: +# +# control exactly as every case above ran: `LANES_INDEX=off` and no +# indexer on PATH; +# unreachable OFFLINE NO-READ — an indexer installed on PATH and configured +# at a Postgres nobody can reach (a fake `psql` that logs and +# refuses), so every write's nudge DOES reach for it; +# poisoned POISONED INDEX — a real sync of this register into SQLite, +# then WRONG rows: a forged hold, forged owners, forged states, +# forged narrative. Unreachability cannot prove an act does not +# read the index; wrong rows nothing repeats can. +# +# Each act's exit, stdout and stderr must be the same three ways, once the +# CLOCK is taken out (a stamp, a commit id, an age): the snapshot is restored +# between runs, so the register, the logs, origin and the guard's pending +# offers are where they were. +A14_BIN="$SANDBOX/a14bin"; A14_FAKE="$SANDBOX/a14fake"; A14_CONF="$SANDBOX/a14conf" +mkdir -p "$A14_BIN" "$A14_FAKE" "$A14_CONF" +cp -p "$SRC_DIR/lanes-index" "$A14_BIN/lanes-index" +chmod 755 "$A14_BIN/lanes-index" +A14_PSQL_LOG="$SANDBOX/a14-psql.log" +: > "$A14_PSQL_LOG" +cat > "$A14_FAKE/psql" <> "$A14_PSQL_LOG" +echo 'psql: error: connection to server at "lanes-index.invalid" failed' >&2 +exit 2 +FAKE +chmod 755 "$A14_FAKE/psql" +printf '*:*:*:*:never-read\n' > "$A14_CONF/pgpass" +chmod 600 "$A14_CONF/pgpass" +printf 'url=postgresql://lanes_writer@lanes-index.invalid/qa\npassfile=%s\n' "$A14_CONF/pgpass" > "$A14_CONF/unreachable.conf" +chmod 600 "$A14_CONF/unreachable.conf" +A14_DB="$SANDBOX/a14state/lanes-index.sqlite" +printf 'sqlite=%s\n' "$A14_DB" > "$A14_CONF/poisoned.conf" +chmod 600 "$A14_CONF/poisoned.conf" + +# THE POISONED STORE BEGINS AS A TRUE ONE: a note the index will carry, a sync +# of this whole register, and then the rows are made wrong. +run env LANES_LANE=repoA-1 "$E" log NOTED lane:repoA-1 "a14: the note the index carries" +is "a note for the derived index to carry is written" "$rc" 0 +run env -u LANES_INDEX LANES_INDEX_CONFIG="$A14_CONF/poisoned.conf" LANES_EDIT="$E" "$A14_BIN/lanes-index" sync +is "lanes-index syncs this suite's whole register into SQLite" "$rc" 0 +has "…and says what it wrote" "$out" "upserted" +python3 - "$A14_DB" <<'PY' +import sqlite3, sys +c = sqlite3.connect(sys.argv[1]) +c.execute("UPDATE lanes SET owner = 'managed a14-poison', state_phrase = 'LIVE · A14 POISON', " + "session_ids = '[\"99999999-9999-4999-8999-999999999999\"]', " + "last_session = '99999999-9999-4999-8999-999999999999'") +c.execute("UPDATE log_lines SET free_text = 'A14 POISON' WHERE verb IN ('NOTED', 'RULED')") +c.execute("UPDATE log_lines SET verb = 'CLAIMED', object = 'opensoft/repoA#1414' " + "WHERE rowid IN (SELECT rowid FROM log_lines WHERE verb = 'STARTED' LIMIT 5)") +c.execute("UPDATE register_lines SET kind = 'LANDING', raw = 'LANDING — A14 POISON'") +c.commit() +PY +run env -u LANES_INDEX PATH="$A14_BIN:$PATH" LANES_INDEX_CONFIG="$A14_CONF/poisoned.conf" "$E" history repoA-1 --index +has "the poison is REAL: \`history --index\` reads it, so the sameness below is not vacuous" "$out" "A14 POISON" +has "…and says it read the index" "$err" "read: index (sqlite)" + +a14_norm() { + printf '%s' "$1" | sed -E \ + -e 's/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z//g' \ + -e 's/[0-9]+h [0-9]{2}m//g' -e 's/[0-9]+[smhd] ago//g' \ + -e 's/[0-9a-f]{7,}//g' +} +A14_SNAP="$SANDBOX/a14snap" +a14_snap() { + rm -rf "$A14_SNAP"; mkdir -p "$A14_SNAP" + cp -a "$WIP" "$A14_SNAP/wip" + cp -a "$ORIGIN" "$A14_SNAP/origin" + if [ -d "$CLAUDE_CONFIG_DIR/lanes" ]; then cp -a "$CLAUDE_CONFIG_DIR/lanes" "$A14_SNAP/cclanes"; fi +} +a14_restore() { + rm -rf "$WIP" "$ORIGIN" "$CLAUDE_CONFIG_DIR/lanes" + cp -a "$A14_SNAP/wip" "$WIP" + cp -a "$A14_SNAP/origin" "$ORIGIN" + if [ -d "$A14_SNAP/cclanes" ]; then cp -a "$A14_SNAP/cclanes" "$CLAUDE_CONFIG_DIR/lanes"; fi +} +# A WRITE'S NUDGE RUNS DETACHED and may still be reading this checkout when the +# snapshot goes back: wait for its one line in the fake `psql`'s log (bounded), +# which is also the proof that it reached for the store at all. +a14_wait_psql() { # + a14_w=0 + while [ "$a14_w" -lt 150 ]; do + [ "$(awk 'END { print NR + 0 }' "$A14_PSQL_LOG")" -gt "$1" ] && return 0 + sleep 0.1; a14_w=$((a14_w + 1)) + done + return 1 +} +a14_three() { #