From 0b380adb9975a466f2a74f580d48df165aa09c69 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 20:24:00 +0000 Subject: [PATCH 01/29] Add lane-worktrees: the sweep that retires a lane's abandoned worktrees A swap that does not finish gracefully leaves its writers' trees where they stood, and nothing removed them: a person archived, verified and removed them by hand. `lane-worktrees sweep ` makes that one act, dry run by default, under one rule: nothing is deleted that is not first on origin or in a bundle under the sweeps directory, and a live writer's tree is never touched. Which trees are the lane's is #97's inventory and the disk, never the derived index (Amendment 14(b)); a tree the inventory does not name is FOREIGN and is left unless --include-foreign and a --word, and one another lane's inventory names is never taken. Who may act is #97's reconciliation, read before any write including the fetch: a lane bound elsewhere, held by another live session, managed-owned or unreadable is refused (exit 2); a lane this session holds acts only on the writer count the coordinator states with --live. "Merged" is the register's LANDED line for the branch's PR or gh's MERGED for it, never ancestry alone. Unpublished commits are pushed under the branch's own name, never its upstream's (a branch made from origin/main tracks main), or to rescue//- where origin diverged; dirty work becomes a WIP commit built through a copy of the index, so a refused push leaves the tree exactly as it was. Removal re-proves the head and the content at the moment it happens. Also --branches, --include-scratch, --include-caches, --include-sandboxes, --links, alternates and local-remote detection (LOAD-BEARING clones are never removed), sweep --expire with 90-day retention that never expires an archive whose rescue left origin, and the porcelain contract lane-end's gate (#163) reads: 0 nothing to retire, 3 something to retire, 2 refused. Part of #162. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lane-worktrees | 2537 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2537 insertions(+) create mode 100755 lane-worktrees diff --git a/lane-worktrees b/lane-worktrees new file mode 100755 index 0000000..dda13b4 --- /dev/null +++ b/lane-worktrees @@ -0,0 +1,2537 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""lane-worktrees - one act that archives, rescues and removes a lane's +abandoned worktrees, branches, scratch and killed-suite sandboxes +(opensoft/openRepoTools#162). + + lane-worktrees sweep [--dry-run | --yes] [--porcelain] + [--bundle] [--branches] [--links] + [--include-scratch] [--include-caches] [--include-sandboxes] + [--include-foreign --word ""] + [--live ... | --live none] [--fetch] [--estate ] + lane-worktrees sweep --expire [--dry-run | --yes] [--porcelain] + lane-worktrees --help | --version + +A SWAP THAT DOES NOT FINISH GRACEFULLY LEAVES ITS WRITERS' TREES WHERE THEY +STOOD, and the protocol's worktree-safety rule (a swap never commits, pushes, +stashes, resets or cleans) is right to leave them. This is the act that +retires them afterwards, in one place, with one rule: NOTHING IS DELETED THAT +IS NOT FIRST ON ORIGIN OR IN A BUNDLE under the sweeps directory, and a tree a +live writer owns is never touched. + +THE DRY RUN IS THE DEFAULT and changes nothing - no fetch, no ref, no file, +no register line (`--fetch` asks origin first; `lanes-edit.sh`'s own reads +fetch the register as they always do). `--yes` performs the table, fetching +origin first in every repository it acts in. + +WHICH TREES ARE THE LANE'S is #97's inventory (`lanes-edit.sh lane-trees`) +and the disk - never the derived index (Amendment 14 clause (b)). A tree the +inventory does not name is FOREIGN: reported, and left, unless +`--include-foreign` and a `--word`. A tree another lane's inventory names is +that lane's and is never acted on from here. + +WHO MAY ACT is #97's reconciliation (`lanes-edit.sh lane-reconcile`): a lane +bound on another host or container is UNKNOWN, never dead (Amendment 18(b)), +so every act is refused for it; a lane held by another live session is +refused; a lane whose holder is THIS session acts only on the writer count +the coordinator states with `--live` (Amendment 17 Addendum 1 (i)). + +"MERGED" is the register's LANDED line for the branch's pull request, or +`gh`'s MERGED state for it - never `git branch --merged` alone, because a +squash merge leaves no ancestry. + +The sweeps directory: ${XDG_STATE_HOME:-$HOME/.local/state}/openRepoTools/ +sweeps/// with MANIFEST.sha256, DISPOSITION.md and rescues.tsv. +`sweep --expire` lists (and with `--yes` removes) archives older than the +retention (90 days; `sweep.conf`) whose every rescue branch is still on +origin; an archive whose rescue branch is gone from origin is never expired. + +Exit codes: + 0 dry run: nothing to retire · --yes: every act completed + 3 dry run: something to retire (the `lane-end` gate's answer, #163) + 2 refused: the lane is bound elsewhere, held by another live session, + managed-owned, or a read failed - nothing was changed + 1 --yes: an act failed part-way; what was done is recorded and the + rest is left exactly as it was + 64 usage +""" + +from __future__ import annotations + +import argparse +import datetime as _dt +import getpass +import hashlib +import json +import os +import re +import shutil +import socket +import stat +import subprocess +import sys +import tarfile +import tempfile +import time + +PROG = "lane-worktrees" +VERSION = "1" +US = "\x1f" + +#: Dependency caches and bytecode: they hold no work and are rebuilt, so they +#: are removed WITHOUT archiving (`--include-caches`) and never counted as a +#: tree's ignored work. +CACHE_NAMES = ("__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", + "node_modules") +VENV_NAMES = ("venv", ".venv") + +#: Ignored BUILD OUTPUT: generated, rebuilt, and often gigabytes. A tree's +#: ignored files are archived before it goes - except caches and these. +BUILD_NAMES = ("target", "build", "dist", ".tox", ".nox", ".gradle", ".next", + ".turbo", ".cache", "coverage", "htmlcov", ".eggs") + +#: A branch with one of these names is never pushed "as is" by a sweep: its +#: unpublished commits go to a rescue branch instead. Pushing a lane's stray +#: commits onto a default branch is not a rescue. +PROTECTED_BRANCHES = ("main", "master") + +#: The register words a legacy writer may never write (ruling 2026-10-04). +RESERVED = ("managed owner", "managed binding", "mode=managed", "managed:") + +GIT_NET_TIMEOUT = 180 +GIT_TIMEOUT = 120 + +DEFAULT_CONF = { + "retention_days": 90, + "aging_days": 14, + "sandbox_min_age_minutes": 60, + "foreign_quiet_hours": 24, + "ignored_report_mb": 50, + "ignored_archive_mb": 200, +} + +UUID_RE = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" + r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$") +STAMP_RE = re.compile(r"^(\d{8}T\d{6}Z)") +SLUG_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") +LANDED_OBJ_RE = re.compile( + r"^LANDED — lane [^,]+, session [^,]+, [^,]+, " + r"([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)#(\d+)\b") +LANDED_ROW_RE = re.compile(r"^LANDED — .*PR #(\d+) into " + r"([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+) main") + + +class Usage(Exception): + """A usage error: exit 64.""" + + +class Refusal(Exception): + """A refusal before anything was changed: exit 2.""" + + +# ================================================================ utilities + +def utc_now() -> str: + return _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def utc_stamp() -> str: + return _dt.datetime.now(_dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + +def say(text: str = "") -> None: + sys.stdout.write(text + "\n") + + +def note(text: str) -> None: + sys.stderr.write(f"{PROG}: {text}\n") + + +def run(argv, cwd=None, env=None, timeout=GIT_TIMEOUT, input_text=None): + """Run `argv`; NEVER raises for a non-zero exit or a missing program. + + A program that could not be started is exit 127 with the reason on + stderr, and a timeout is 124 - the two shapes every caller here already + reads as "the read failed", which is never an answer (Amendment 7(d)).""" + try: + return subprocess.run( + [str(a) for a in argv], cwd=cwd, env=env, capture_output=True, + text=True, timeout=timeout, input=input_text, + stdin=None if input_text is not None else subprocess.DEVNULL) + except subprocess.TimeoutExpired: + return subprocess.CompletedProcess(argv, 124, "", f"timed out after {timeout}s") + except OSError as exc: + return subprocess.CompletedProcess(argv, 127, "", str(exc)) + + +def git_env() -> dict: + env = dict(os.environ) + env["GIT_TERMINAL_PROMPT"] = "0" + env.setdefault("GIT_SSH_COMMAND", "ssh -o BatchMode=yes") + env["LC_ALL"] = "C" + # A READ TAKES NO OPTIONAL LOCK: `git status` otherwise refreshes the + # index's stat cache and writes it back, and a dry run writes nothing. + env["GIT_OPTIONAL_LOCKS"] = "0" + return env + + +def git(path: str, *args: str, timeout: int = GIT_TIMEOUT): + return run(["git", "-C", path, *args], env=git_env(), timeout=timeout) + + +def git_out(path: str, *args: str): + """stdout stripped, or None where git failed.""" + proc = git(path, *args) + if proc.returncode != 0: + return None + return proc.stdout.strip() + + +def real(path: str) -> str: + try: + return os.path.realpath(path) + except OSError: + return path + + +def under(path: str, root: str) -> bool: + """`path` IS `root` or sits beneath it - compared physically.""" + p, r = real(path), real(root) + return p == r or p.startswith(r.rstrip(os.sep) + os.sep) + + +def tilde(path: str) -> str: + home = os.environ.get("HOME", "") + if home and (path == home or path.startswith(home.rstrip("/") + "/")): + return "~" + path[len(home.rstrip("/")):] + return path + + +def short(sha: str) -> str: + return sha[:12] if sha and re.match(r"^[0-9a-f]{12,}$", sha) else (sha or "-") + + +def dir_size(path: str) -> int: + total = 0 + for base, dirs, files in os.walk(path, followlinks=False): + for name in files + dirs: + try: + total += os.lstat(os.path.join(base, name)).st_size + except OSError: + pass + return total + + +def human_size(n: int) -> str: + size = float(n) + for unit in ("B", "KB", "MB", "GB"): + if size < 1024 or unit == "GB": + return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}" + size /= 1024 + return f"{n} B" + + +def rmtree(path: str) -> None: + """`shutil.rmtree` that also clears git's read-only object files. A path + that is already gone is already removed.""" + if not os.path.lexists(path): + return + + def clear(func, target, _exc): + try: + os.chmod(target, stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC) + except OSError: + pass + func(target) + shutil.rmtree(path, onerror=clear) + + +def sha256_of(path: str) -> str: + digest = hashlib.sha256() + with open(path, "rb") as fh: + for block in iter(lambda: fh.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def safe_ref_part(text: str) -> str: + """A path component folded into what a ref name takes.""" + folded = re.sub(r"[^A-Za-z0-9._-]+", "-", text).strip(".-") or "tree" + folded = re.sub(r"\.\.+", ".", folded) + if folded.endswith(".lock"): + folded = folded[:-5] + "-lock" + return folded[:80] + + +def register_text(text: str) -> str: + """A register line's free text, fenced the way `lanes-edit.sh`'s writer + fences a payload: no ` — ` (the line's own separator), no `, `, no + newline, and none of the managed-owner vocabulary a legacy writer may + never write (ruling 2026-10-04).""" + out = text.replace("\r", " ").replace("\n", " ") + out = out.replace(" — ", " - ").replace(", ", "; ") + low = out.lower() + for word in RESERVED: + if word in low: + out = re.sub("managed", "manag.ed", out, flags=re.IGNORECASE) + break + while out.startswith(("→", "←")): + out = out[1:].lstrip() + return out + + +# ============================================================ configuration + +def _home() -> str: + return os.environ.get("HOME") or os.path.expanduser("~") + + +def state_dir() -> str: + base = os.environ.get("XDG_STATE_HOME") or os.path.join(_home(), ".local", "state") + return os.path.join(base, "openRepoTools") + + +def sweeps_dir() -> str: + return os.path.join(state_dir(), "sweeps") + + +def conf_path() -> str: + named = os.environ.get("LANE_WORKTREES_CONF", "") + if named: + return named + base = os.environ.get("XDG_CONFIG_HOME") or os.path.join(_home(), ".config") + return os.path.join(base, "openRepoTools", "sweep.conf") + + +def load_conf() -> dict: + """`sweep.conf`: `key=value` lines, `#` comments. An unknown key or a + value that is not a whole number is a note and the default stands.""" + conf = dict(DEFAULT_CONF) + path = conf_path() + try: + with open(path, encoding="utf-8") as fh: + lines = fh.read().splitlines() + except OSError: + return conf + for raw in lines: + line = raw.split("#", 1)[0].strip() + if not line or "=" not in line: + continue + key, value = (part.strip() for part in line.split("=", 1)) + if key not in DEFAULT_CONF: + note(f"{path}: '{key}' is not a sweep.conf key; ignored") + continue + if not value.isdigit(): + note(f"{path}: {key}={value} is not a whole number; the default " + f"{DEFAULT_CONF[key]} stands") + continue + conf[key] = int(value) + return conf + + +# =================================================== the helper (lanes-edit) + +def lanes_edit_path() -> str: + """`$LANES_EDIT`, else the `lanes-edit.sh` beside this file, else PATH - + the order `lanes` and `lanes-index` already find it in.""" + named = os.environ.get("LANES_EDIT", "") + if named: + return named + here = os.path.join(os.path.dirname(os.path.realpath(__file__)), "lanes-edit.sh") + if os.access(here, os.X_OK): + return here + return shutil.which("lanes-edit.sh") or "" + + +class LaneFacts: + """What #97's reconciliation says about one lane, read once.""" + + def __init__(self, lane: str): + self.lane = lane + self.rc = 0 + self.root = "" + self.state = "NONE" + self.holder = "unknown" # live | none | unknown + self.holder_uuid = "" + self.holder_detail = "" + self.binding = "unknown" # here | free | gone | elsewhere | unknown + self.binding_at = "" + self.managed = "" # "" | owned | unknown + self.managed_owner = "" + self.verdict = "" + self.verdict_why = "" + self.dir = "" + self.inventory: list = [] + self.inventory_rc = 0 + self.why = "" + + +class Helper: + def __init__(self): + self.path = lanes_edit_path() + self._fetched = False + + def call(self, *args: str, env_extra: dict | None = None, fetch: bool = False, + timeout: int = 300): + if not self.path: + return subprocess.CompletedProcess(args, 127, "", "no lanes-edit.sh beside this " + "command or on PATH") + env = dict(os.environ) + env["LANES_INDEX"] = "off" + if not fetch or self._fetched: + env["LANES_NO_FETCH"] = "1" + if env_extra: + env.update(env_extra) + proc = run([self.path, *args], env=env, timeout=timeout) + if fetch: + self._fetched = True + return proc + + def lane_facts(self, lane: str) -> LaneFacts: + facts = LaneFacts(lane) + proc = self.call("lane-reconcile", lane, fetch=True) + facts.rc = proc.returncode + if proc.returncode == 0: + self._parse_reconcile(facts, proc.stdout) + elif proc.returncode == 8: + # NO CONTROL ROOT: no inventory exists, and the holder and the + # binding are asked for on their own. + facts.state = "NONE" + self._holder_binding(facts) + else: + facts.why = (f"`lanes-edit.sh lane-reconcile {lane}` exited {proc.returncode}: " + + _first_line(proc.stderr)) + return facts + dproc = self.call("lane-dir", lane) + if dproc.returncode == 0: + facts.dir = dproc.stdout.strip() + elif dproc.returncode != 8: + facts.why = (f"`lanes-edit.sh lane-dir {lane}` exited {dproc.returncode}: " + + _first_line(dproc.stderr)) + return facts + tproc = self.call("lane-trees", lane) + facts.inventory_rc = tproc.returncode + if tproc.returncode == 0: + for line in tproc.stdout.splitlines(): + fields = line.split(US) + if len(fields) < 2 or not fields[0]: + continue + keys = ("id", "path", "branch", "head", "upstream", "dirty", "unpushed", + "writer", "observed", "checkout", "generation", "operation", + "schema") + rec = dict(zip(keys, fields + [""] * (len(keys) - len(fields)))) + facts.inventory.append(rec) + elif tproc.returncode not in (8, 1) or (tproc.returncode == 1 and facts.root): + facts.why = (f"`lanes-edit.sh lane-trees {lane}` exited {tproc.returncode}: " + + _first_line(tproc.stderr)) + return facts + + def _parse_reconcile(self, facts: LaneFacts, text: str) -> None: + for line in text.splitlines(): + parts = line.split(US) + tag = parts[0] + if tag == "MANAGED": + facts.managed = "unknown" if parts[1:2] == ["unknown"] else "owned" + facts.managed_owner = parts[1] if len(parts) > 1 else "" + elif tag == "ROOT" and len(parts) > 1: + facts.root = parts[1] + elif tag == "STATE" and len(parts) > 1: + facts.state = parts[1] + elif tag == "HOLDER" and len(parts) > 1: + facts.holder = parts[1] + facts.holder_detail = parts[2] if len(parts) > 2 else "" + if facts.holder == "live": + facts.holder_uuid = facts.holder_detail.split(" ")[0] + elif tag == "BINDING" and len(parts) > 1: + facts.binding = parts[1] + facts.binding_at = parts[2] if len(parts) > 2 else "" + elif tag == "VERDICT" and len(parts) > 1: + facts.verdict = parts[1] + facts.verdict_why = parts[2] if len(parts) > 2 else "" + + def _holder_binding(self, facts: LaneFacts) -> None: + bproc = self.call("binding", facts.lane) + if bproc.returncode == 0: + cols = bproc.stdout.rstrip("\n").split("\t") + where = cols[6] if len(cols) > 6 else "elsewhere" + wstate = cols[7] if len(cols) > 7 else "unknown" + facts.binding_at = f"{cols[0]}/{cols[1]}" if len(cols) > 1 else "" + if where == "here": + facts.binding = "here" + elif wstate == "gone": + facts.binding = "gone" + else: + facts.binding = "elsewhere" + elif bproc.returncode == 8: + facts.binding = "free" + else: + facts.binding = "unknown" + hproc = self.call("live-holder", facts.lane) + if hproc.returncode == 0: + facts.holder = "live" + facts.holder_detail = hproc.stdout.strip().replace(US, " ") + facts.holder_uuid = facts.holder_detail.split(" ")[0] + elif hproc.returncode == 8: + facts.holder = "none" + else: + facts.holder = "unknown" + facts.holder_detail = _first_line(hproc.stderr) + mproc = self.call("managed-projection", facts.lane) + if mproc.returncode == 0: + facts.managed, facts.managed_owner = "owned", mproc.stdout.strip() + elif mproc.returncode not in (8, 2): + facts.managed = "unknown" + + def workspace_root(self) -> str: + proc = self.call("workspace-root") + return proc.stdout.strip() if proc.returncode == 0 else "" + + def transcript_live(self, uuid: str) -> str: + """live | none | unknown - one live process carrying that transcript, + asked once per session however many trees it recorded.""" + cache = self.__dict__.setdefault("_live", {}) + if uuid not in cache: + proc = self.call("transcript-holders", uuid) + cache[uuid] = {0: "live", 8: "none"}.get(proc.returncode, "unknown") + return cache[uuid] + + def note_line(self, lane: str, text: str): + return self.call("log", "NOTED", f"lane:{lane}", register_text(text), + env_extra={"LANES_LANE": lane}, fetch=False) + + +def _first_line(text: str) -> str: + for line in (text or "").splitlines(): + if line.strip(): + return line.strip() + return "no answer" + + +# ============================================================ liveness: who + +class ProcScan: + """Every process this account can see, and the paths it stands in or + holds open: `/proc` where there is one, `lsof` where there is not + (macOS), and every tmux pane's current path beside either. + + A scan that could not be made is `available = False`, and a tree whose + liveness rests on it is never acted on - not "nobody is there".""" + + def __init__(self): + self.entries: list = [] # (pid, comm, kind, path) + self.available = False + self.why = "" + self._env_loaded = False + self._scan() + + def _scan(self) -> None: + me = os.getpid() + if os.path.isdir("/proc/self/fd"): + self.available = True + for name in os.listdir("/proc"): + if not name.isdigit() or int(name) == me: + continue + base = "/proc/" + name + try: + with open(base + "/comm", encoding="utf-8", errors="replace") as fh: + comm = fh.read().strip() + except OSError: + continue + try: + self.entries.append((int(name), comm, "cwd", _proc_path(os.readlink(base + "/cwd")))) + except OSError: + pass + try: + fds = os.listdir(base + "/fd") + except OSError: + fds = [] + for fd in fds: + try: + target = os.readlink(base + "/fd/" + fd) + except OSError: + continue + if target.startswith("/"): + self.entries.append((int(name), comm, "fd", _proc_path(target))) + elif shutil.which("lsof"): + proc = run(["lsof", "-nP", "-w", "-F", "pcfn", "-u", str(os.getuid())], timeout=120) + if proc.stdout: + self.available = True + pid, comm, fd = 0, "", "" + for line in proc.stdout.splitlines(): + if not line: + continue + tag, value = line[0], line[1:] + if tag == "p": + pid, comm, fd = int(value or 0), "", "" + elif tag == "c": + comm = value + elif tag == "f": + fd = value + elif tag == "n" and pid and pid != me and value.startswith("/"): + self.entries.append((pid, comm, "cwd" if fd == "cwd" else "fd", value)) + else: + self.why = "lsof answered nothing: " + _first_line(proc.stderr) + else: + self.why = "this host has no /proc and no lsof, so which process stands in a tree cannot be read" + if shutil.which("tmux"): + proc = run(["tmux", "list-panes", "-a", "-F", + "#{pane_pid}\t#{session_name}:#{window_index}.#{pane_index}\t#{pane_current_path}"], + timeout=30) + if proc.returncode == 0: + for line in proc.stdout.splitlines(): + cols = line.split("\t") + if len(cols) == 3 and cols[2].startswith("/"): + pid = int(cols[0]) if cols[0].isdigit() else 0 + # PHYSICAL, like every other path here: `/proc` and `lsof` + # answer resolved paths, and a pane keeps the spelling + # the shell was given (macOS's `/var` is `/private/var`). + self.entries.append((pid, "tmux pane " + cols[1], "pane", real(cols[2]))) + + def load_environ(self) -> None: + """The path-shaped values of every readable process's environment + (HOME, TMPDIR, a suite's sandbox): Linux only, and only for the + sandbox sweep, which is where a process's ROOT may be nowhere else.""" + if self._env_loaded: + return + self._env_loaded = True + if not os.path.isdir("/proc/self"): + return + me = os.getpid() + for name in os.listdir("/proc"): + if not name.isdigit() or int(name) == me: + continue + try: + with open(f"/proc/{name}/environ", "rb") as fh: + raw = fh.read() + except OSError: + continue + for item in raw.split(b"\0"): + if b"=/" not in item: + continue + value = item.split(b"=", 1)[1].decode("utf-8", "replace") + for piece in value.split(":"): + if piece.startswith("/"): + self.entries.append((int(name), "", "env", piece)) + + def holders(self, path: str) -> list: + root = real(path) + out = [] + for pid, comm, kind, where in self.entries: + if where == root or where.startswith(root.rstrip("/") + "/"): + out.append((pid, comm, kind, where)) + return out + + +def _proc_path(target: str) -> str: + return target[:-10] if target.endswith(" (deleted)") else target + + +def pid_alive(pid: int) -> bool: + if pid <= 0: + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return False + return True + + +# ========================================================== merge evidence + +def slug_of(url: str) -> str: + u = (url or "").strip() + if u.endswith(".git"): + u = u[:-4] + for prefix in ("https://github.com/", "http://github.com/", "ssh://git@github.com/", + "git@github.com:", "git://github.com/"): + if u.startswith(prefix): + rest = u[len(prefix):].strip("/") + return rest if SLUG_RE.match(rest) else "" + return "" + + +class MergeEvidence: + """Whether a branch's work is MERGED, decided by the register's LANDED + line for its pull request or by `gh`'s state for it - and never by + `git branch --merged` alone, which a squash merge defeats.""" + + def __init__(self, helper: Helper): + self.helper = helper + self.gh_ok = (os.environ.get("LANES_NO_GITHUB", "") != "1" + and bool(shutil.which("gh"))) + self.gh_why = "" if self.gh_ok else ("LANES_NO_GITHUB=1" if os.environ.get( + "LANES_NO_GITHUB", "") == "1" else "no gh on PATH") + self._prs: dict = {} + self._landed = None + self.register_why = "" + + def landed(self) -> set: + if self._landed is not None: + return self._landed + self._landed = set() + root = self.helper.workspace_root() + if not root: + self.register_why = "the workspace repository could not be found" + return self._landed + branch = os.environ.get("LANES_BRANCH", "main") + proc = git(root, "grep", "-h", "-E", "-e", "^LANDED", f"origin/{branch}", "--", + "lanes/LANES.md", "lanes/log") + if proc.returncode not in (0, 1): + self.register_why = "the register could not be read: " + _first_line(proc.stderr) + return self._landed + for line in proc.stdout.splitlines(): + m = LANDED_OBJ_RE.match(line) + if m: + self._landed.add((m.group(1).lower(), int(m.group(2)))) + continue + m = LANDED_ROW_RE.match(line) + if m: + self._landed.add((m.group(2).lower(), int(m.group(1)))) + return self._landed + + def prs(self, slug: str): + if not slug or not self.gh_ok: + return None + if slug in self._prs: + return self._prs[slug] + env = dict(os.environ, GH_PROMPT_DISABLED="1", GH_NO_UPDATE_NOTIFIER="1") + proc = run(["gh", "pr", "list", "--repo", slug, "--state", "all", "--limit", "1000", + "--json", "number,state,mergedAt,headRefName,headRefOid,isCrossRepository"], + env=env, timeout=120) + rows = None + if proc.returncode == 0: + try: + rows = json.loads(proc.stdout or "[]") + except ValueError: + rows = None + if rows is None: + self.gh_why = f"gh pr list --repo {slug} failed: " + _first_line(proc.stderr) + self._prs[slug] = rows + return rows + + def judge(self, repo: str, slug: str, branch: str, tip: str) -> dict: + """{'state': merged|open|closed|none|unknown, 'pr': n, 'source': ..., + 'why': ...} for `branch` at `tip`.""" + rows = self.prs(slug) + if rows is None: + return {"state": "unknown", "pr": 0, "source": "", + "why": "no pull request could be read for it (" + (self.gh_why or + "not a GitHub origin") + ")"} + mine = [r for r in rows if r.get("headRefName") == branch + and not r.get("isCrossRepository")] + if not mine: + return {"state": "none", "pr": 0, "source": "gh", "why": "no pull request names it"} + landed = self.landed() + for row in sorted(mine, key=lambda r: -int(r.get("number", 0))): + number = int(row.get("number", 0)) + by_register = (slug.lower(), number) in landed + by_gh = row.get("state") == "MERGED" + if by_register or by_gh: + head = row.get("headRefOid") or "" + contained = bool(tip) and (tip == head or (bool(head) and git( + repo, "merge-base", "--is-ancestor", tip, head).returncode == 0)) + source = "register LANDED" if by_register else "gh MERGED" + if contained: + return {"state": "merged", "pr": number, "source": source, "head": head, + "why": f"PR #{number} is merged ({source}) and holds this tip"} + return {"state": "moved", "pr": number, "source": source, "head": head, + "why": f"PR #{number} is merged ({source}) but this branch has moved " + f"past its head {short(head)}"} + for row in mine: + if row.get("state") == "OPEN": + return {"state": "open", "pr": int(row["number"]), "source": "gh", + "why": f"PR #{row['number']} is open"} + return {"state": "closed", "pr": int(mine[0].get("number", 0)), "source": "gh", + "why": f"PR #{mine[0].get('number')} was closed unmerged"} + + +# ============================================================== the estate + +def estate_repos(root: str, max_depth: int = 7) -> list: + """Every repository under `root`: a walk that descends into a repository + ONLY through nested checkouts and worktree containers, so the walk is the + estate's shape and not every source directory in it.""" + found = [] + prune = set(CACHE_NAMES) | set(VENV_NAMES) | {".git", ".lane-state", ".tox"} + + def walk(d: str, depth: int, in_repo: bool) -> None: + try: + entries = list(os.scandir(d)) + except OSError: + return + is_repo = any(e.name == ".git" for e in entries) + if is_repo: + found.append(d) + if depth >= max_depth: + return + for e in entries: + if e.name in prune: + continue + try: + if e.is_symlink() or not e.is_dir(follow_symlinks=False): + continue + except OSError: + continue + if is_repo or in_repo: + if (e.name in ("worktrees", ".worktrees", ".claude") + or os.path.lexists(os.path.join(e.path, ".git"))): + walk(e.path, depth + 1, True) + else: + walk(e.path, depth + 1, False) + + if root and os.path.isdir(root): + walk(root, 0, False) + return found + + +def git_common_dir(repo: str) -> str: + dotgit = os.path.join(repo, ".git") + if os.path.isdir(dotgit): + return dotgit + try: + with open(dotgit, encoding="utf-8") as fh: + first = fh.readline().strip() + except OSError: + return "" + if not first.startswith("gitdir:"): + return "" + gitdir = first[len("gitdir:"):].strip() + if not os.path.isabs(gitdir): + gitdir = os.path.normpath(os.path.join(repo, gitdir)) + try: + with open(os.path.join(gitdir, "commondir"), encoding="utf-8") as fh: + common = fh.readline().strip() + if not os.path.isabs(common): + common = os.path.normpath(os.path.join(gitdir, common)) + return common + except OSError: + return gitdir + + +def local_remote_urls(common: str) -> list: + """(name, url) for every remote in that config whose URL is a local path.""" + out = [] + try: + with open(os.path.join(common, "config"), encoding="utf-8", errors="replace") as fh: + text = fh.read() + except OSError: + return out + section = "" + for raw in text.splitlines(): + line = raw.strip() + m = re.match(r'^\[remote\s+"([^"]+)"\]$', line) + if m: + section = m.group(1) + continue + if line.startswith("["): + section = "" + continue + if section and re.match(r"^url\s*=", line): + url = line.split("=", 1)[1].strip() + if url.startswith(("/", "./", "../", "~", "file://")): + out.append((section, url)) + return out + + +def dependents_map(repos: list) -> dict: + """{physical path a repository LEANS ON: [(dependent, how)]} - read from + `objects/info/alternates` and from remotes whose URL is a local path. A + clone something leans on is LOAD-BEARING and is never removed by a sweep.""" + deps: dict = {} + for repo in repos: + common = git_common_dir(repo) + if not common: + continue + try: + with open(os.path.join(common, "objects", "info", "alternates"), + encoding="utf-8") as fh: + for raw in fh.read().splitlines(): + alt = raw.strip() + if not alt or alt.startswith("#"): + continue + if not os.path.isabs(alt): + alt = os.path.normpath(os.path.join(common, "objects", alt)) + target = real(alt) + deps.setdefault(target, []).append((repo, "alternates " + alt)) + except OSError: + pass + for name, url in local_remote_urls(common): + path = url[len("file://"):] if url.startswith("file://") else url + path = os.path.expanduser(path) + if not os.path.isabs(path): + path = os.path.normpath(os.path.join(repo, path)) + deps.setdefault(real(path), []).append((repo, f"remote {name} -> {url}")) + return deps + + +def leaned_on(path: str, common: str, deps: dict) -> list: + """Who leans on the tree at `path` (and on its object store, for a clone).""" + out = [] + targets = [real(path)] + if common and not under(common, path): + pass + elif common: + targets.append(real(common)) + for target, who in deps.items(): + for t in targets: + if target == t or target.startswith(t.rstrip("/") + "/"): + for dependent, how in who: + if not under(dependent, path): + out.append((dependent, how)) + seen, uniq = set(), [] + for item in out: + if item not in seen: + seen.add(item) + uniq.append(item) + return uniq + + +# ================================================================ the trees + +class Tree: + def __init__(self, path: str, owner: str, foreign_kind: str = ""): + self.path = path + self.real = real(path) + self.owner = owner # inventory | foreign + self.foreign_kind = foreign_kind # registered | lane-root | clone | other-lane + self.other_lane = "" + self.sidecar: dict = {} + self.checkout = "" # the repository's main worktree + self.registered = False + self.locked = False + self.exists = os.path.isdir(path) + self.is_git = False + self.kind = "" # worktree | clone | main + self.common = "" + self.branch = "" + self.detached = False + self.unborn = False + self.head = "" + self.status_lines: list = [] + self.upstream = "" + self.origin_url = "" + self.remote_branch = "" # the name it is (or would be) pushed as + self.remote_tip = "" + self.on_origin = None # bool | None (unknown) + self.ahead = 0 + self.behind = 0 + self.lane_trailer = "" + self.merge: dict = {} + self.live = None # (True|None, why) | None = not live + self.lean: list = [] + self.guard = "" + self.read_error = "" + self.disposition = "" + self.reason = "" + self.retire = False + self.result = "" + self.rescue = "" # origin ref a rescue went to + self.bundle = "" + self.ignored_archive = "" + self.register = "" + + @property + def slice(self) -> str: + return safe_ref_part(os.path.basename(self.path.rstrip("/"))) + + def label(self) -> str: + if not self.exists: + return "(gone)" + if not self.is_git: + return "(not a checkout)" + if self.detached: + return f"(detached) {short(self.head)}" + if self.unborn: + return f"{self.branch} (unborn)" + return f"{self.branch} {short(self.head)}" + + +def worktree_registrations(checkout: str) -> list: + proc = git(checkout, "worktree", "list", "--porcelain") + if proc.returncode != 0: + return [] + out, cur = [], {} + for line in proc.stdout.splitlines() + [""]: + if not line: + if cur: + out.append(cur) + cur = {} + continue + key, _, value = line.partition(" ") + if key == "worktree": + cur = {"path": value} + elif key in ("HEAD", "branch"): + cur[key.lower()] = value + elif key in ("detached", "bare", "locked", "prunable"): + cur[key] = value or True + return out + + +def read_tree(tree: Tree) -> None: + """Everything git says about one tree, NOW. A read that failed is said, + never converted into a clean-looking value (R22, Amendment 7(d)).""" + if not tree.exists: + return + proc = git(tree.path, "rev-parse", "--show-toplevel", "--git-common-dir") + if proc.returncode != 0: + return + lines = proc.stdout.strip().splitlines() + if not lines or real(lines[0]) != tree.real: + return # a plain directory inside some other checkout + tree.is_git = True + common = lines[1] if len(lines) > 1 else "" + if common and not os.path.isabs(common): + common = os.path.normpath(os.path.join(tree.path, common)) + tree.common = common + dotgit = os.path.join(tree.path, ".git") + tree.kind = "worktree" if os.path.isfile(dotgit) else "clone" + if tree.common: + main = os.path.dirname(tree.common.rstrip("/")) if os.path.basename( + tree.common.rstrip("/")) == ".git" else "" + tree.checkout = main or tree.checkout + if tree.kind == "clone" and main and real(main) == tree.real: + tree.kind = "clone" + branch = git(tree.path, "symbolic-ref", "-q", "--short", "HEAD") + if branch.returncode == 0: + tree.branch = branch.stdout.strip() + elif branch.returncode == 1: + tree.detached = True + else: + tree.read_error = "HEAD could not be read: " + _first_line(branch.stderr) + return + head = git(tree.path, "rev-parse", "--verify", "-q", "HEAD") + if head.returncode == 0: + tree.head = head.stdout.strip() + elif tree.branch: + tree.unborn = True + else: + tree.read_error = "HEAD names no commit" + return + status = git(tree.path, "status", "--porcelain=v1", "--ignore-submodules=none", + "--untracked-files=normal") + if status.returncode != 0: + tree.read_error = "git status failed: " + _first_line(status.stderr) + return + tree.status_lines = [ln for ln in status.stdout.splitlines() if ln.strip()] + tree.origin_url = git_out(tree.path, "config", "--get", "remote.origin.url") or "" + if tree.branch: + up = git(tree.path, "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}") + tree.upstream = up.stdout.strip() if up.returncode == 0 else "" + # THE BRANCH IS PUSHED UNDER ITS OWN NAME, NEVER ITS UPSTREAM'S. A branch + # made with `worktree add -b origin/main` TRACKS origin/main, and a + # push "to its upstream" would put a lane's stray commits on main. + target = tree.branch + tree.remote_branch = target + tip = git_out(tree.path, "rev-parse", "-q", "--verify", f"refs/remotes/origin/{target}") + tree.remote_tip = tip or "" + if tree.head: + if tree.remote_tip: + counts = git_out(tree.path, "rev-list", "--left-right", "--count", + f"{tree.remote_tip}...{tree.head}") + if counts and len(counts.split()) == 2: + tree.behind, tree.ahead = (int(x) for x in counts.split()) + contains = git(tree.path, "for-each-ref", "--count=1", "--contains", tree.head, + "--format=%(refname)", "refs/remotes/origin/") + if contains.returncode == 0: + tree.on_origin = bool(contains.stdout.strip()) + msg = git_out(tree.path, "log", "-1", "--format=%B", "HEAD") or "" + trailers = re.findall(r"^Lane:\s*(\S+)\s*$", msg, re.MULTILINE) + tree.lane_trailer = trailers[-1] if trailers else "" + + +def snapshot_tree(path: str) -> str: + """The tree object `git add -A` would commit in `path` NOW, written + through a COPY of its index so the tree's own index is never touched. + Empty where it could not be made.""" + index = git_out(path, "rev-parse", "--git-path", "index") + if index is None: + return "" + if not os.path.isabs(index): + index = os.path.join(path, index) + fd, tmp = tempfile.mkstemp(prefix="lane-worktrees-index.") + os.close(fd) + try: + if os.path.exists(index): + shutil.copyfile(index, tmp) + else: + os.unlink(tmp) + env = git_env() + env["GIT_INDEX_FILE"] = tmp + if run(["git", "-C", path, "add", "-A"], env=env).returncode != 0: + return "" + proc = run(["git", "-C", path, "write-tree"], env=env) + return proc.stdout.strip() if proc.returncode == 0 else "" + finally: + try: + os.unlink(tmp) + except OSError: + pass + + +def tree_guards(tree: Tree) -> str: + """What makes a tree unsafe to remove whatever its git state says: work a + commit in it cannot carry (a submodule's own dirt or commits, a clone + nested inside it) and a registration someone locked.""" + if tree.locked: + return ("its worktree registration is LOCKED - someone pinned it on purpose; " + f"`git -C {tree.checkout or tree.path} worktree unlock {tree.path}` is theirs to run") + if tree.kind == "clone": + # A CLONE TAKES ITS BRANCHES AND ITS STASH WITH IT, which a worktree + # never does: every one of them must be on origin first. + heads = git(tree.path, "for-each-ref", "--format=%(refname:short) %(objectname)", + "refs/heads/") + if heads.returncode != 0: + return "its branches could not be read" + for line in heads.stdout.splitlines(): + name, _, sha = line.partition(" ") + on = git(tree.path, "for-each-ref", "--count=1", "--contains", sha, + "--format=%(refname)", "refs/remotes/origin/") + if on.returncode != 0 or not on.stdout.strip(): + return (f"it is a CLONE and its branch {name} holds commits origin lacks; " + "a clone's removal takes every branch with it") + stash = git(tree.path, "stash", "list") + if stash.returncode != 0 or stash.stdout.strip(): + return "it is a CLONE with a stash, which its removal would take with it" + sub = git(tree.path, "submodule", "status", "--recursive") + if sub.returncode == 0: + for line in sub.stdout.splitlines(): + if not line.strip() or line.startswith("-"): + continue + parts = line[1:].split() + if len(parts) < 2: + continue + spath = os.path.join(tree.path, parts[1]) + dirt = git_out(spath, "status", "--porcelain") + if dirt is None or dirt: + return (f"submodule {parts[1]} holds uncommitted work (a commit in the tree " + "cannot carry it) - commit it there first") + on = git(spath, "for-each-ref", "--count=1", "--contains", "HEAD", + "--format=%(refname)", "refs/remotes/") + if on.returncode != 0 or not on.stdout.strip(): + return (f"submodule {parts[1]} is at a commit no remote of it holds " + "- push it there first") + subs = set() + if sub.returncode == 0: + for line in sub.stdout.splitlines(): + parts = line[1:].split() + if len(parts) >= 2: + subs.add(os.path.normpath(os.path.join(tree.path, parts[1]))) + prune = set(CACHE_NAMES) | set(VENV_NAMES) + for base, dirs, files in os.walk(tree.path): + if base != tree.path and (".git" in dirs or ".git" in files): + if os.path.normpath(base) not in subs: + return (f"a repository is nested inside it at {base} - its work is " + "no commit of this tree's; move it out first (a lane creates " + "worktrees, never clones)") + dirs[:] = [d for d in dirs if d != ".git" and d not in prune + and os.path.normpath(os.path.join(base, d)) not in subs] + return "" + + +def ignored_work(tree: Tree) -> list: + """Ignored paths that are NOT caches: a commit cannot carry them, so they + are archived beside the bundle before the tree goes.""" + proc = git(tree.path, "ls-files", "--others", "--ignored", "--exclude-standard", + "--directory", "-z") + if proc.returncode != 0: + return [] + out = [] + for rel in proc.stdout.split("\0"): + rel = rel.rstrip("/") + if not rel: + continue + parts = rel.split("/") + if any(p in CACHE_NAMES for p in parts): + continue + full = os.path.join(tree.path, rel) + if os.path.isdir(full) and (os.path.basename(rel) in VENV_NAMES + or os.path.exists(os.path.join(full, "pyvenv.cfg"))): + continue + if any(p in BUILD_NAMES or p.endswith(".egg-info") for p in parts): + continue + if rel.endswith((".pyc", ".pyo")): + continue + out.append(rel) + return out + + +# ============================================================ the sweep + +class Options: + def __init__(self, ns): + self.yes = bool(ns.yes) + self.porcelain = bool(ns.porcelain) + self.bundle = bool(ns.bundle) + self.branches = bool(ns.branches) + self.links = bool(ns.links) + self.scratch = bool(ns.include_scratch) + self.caches = bool(ns.include_caches) + self.sandboxes = bool(ns.include_sandboxes) + self.foreign = bool(ns.include_foreign) + self.word = ns.word or "" + self.live = list(ns.live or []) + self.fetch = bool(ns.fetch) or self.yes + self.estate = ns.estate or "" + + +class Item: + """One row of the table that is not a tree: a branch, a scratch + directory, a cache, a sandbox, a broken link.""" + + def __init__(self, kind: str, path: str, disposition: str, reason: str, + retire: bool = False, detail: str = ""): + self.kind = kind + self.path = path + self.disposition = disposition + self.reason = reason + self.retire = retire + self.detail = detail + self.size = 0 + self.result = "" + self.extra: dict = {} + + +class Sweep: + def __init__(self, lane: str, opts: Options): + self.lane = lane + self.opts = opts + self.conf = load_conf() + self.helper = Helper() + self.facts = LaneFacts(lane) + self.trees: list = [] + self.items: list = [] + self.refusal = "" + self.notes: list = [] + self.caller = (os.environ.get("LANES_SESSION") or + os.environ.get("CLAUDE_CODE_SESSION_ID") or "") + self.procs = None + self.evidence = MergeEvidence(self.helper) + self.stamp = utc_stamp() + self.archive = "" + self.failed = 0 + self.deps: dict = {} + self.estate_root = "" + self.fetched: dict = {} + self.rescues: list = [] + self.dispositions: list = [] + + # ------------------------------------------------------------- reading + def gather(self) -> None: + if not self.helper.path: + self.refusal = ("no lanes-edit.sh beside this command or on PATH: which trees " + "are this lane's is #97's inventory, and nothing else answers it") + return + self.facts = self.helper.lane_facts(self.lane) + f = self.facts + if f.why: + self.refusal = f"a read failed, which is never an answer (Amendment 7(d)): {f.why}" + return + if f.managed == "owned": + self.refusal = (f"lane {self.lane} is owned by the managed ledger (owner " + f"{f.managed_owner or 'unnamed'}): its worktrees are that ledger's " + "(ruling 2026-10-04: \"managed ledger owns enrolled lanes; #97 owns legacy\")") + return + if f.managed == "unknown": + self.refusal = (f"whether the managed ledger owns lane {self.lane} is UNKNOWN; " + f"read it: lanes-edit.sh managed-projection {self.lane}") + return + self.estate_root = (self.opts.estate or os.environ.get("PROJECTS_ROOT", "") + or (os.path.dirname(f.dir.rstrip("/")) if f.dir else "")) + # THE GATE DECIDES BEFORE ANYTHING IS WRITTEN - the fetch included. A + # refused lane is still READ, so the table says what a sweep from the + # right place would do; it is never fetched or touched from here. + self._lane_gate() + self.procs = ProcScan() + self._discover() + if self.opts.fetch and not self.refusal: + self._fetch_all() + self.deps = dependents_map(estate_repos(self.estate_root)) if self.estate_root else {} + for tree in self.trees: + read_tree(tree) + self._other_lanes() + for tree in self.trees: + self._liveness(tree) + if tree.is_git and tree.exists: + tree.lean = leaned_on(tree.path, tree.common if tree.kind == "clone" else "", + self.deps) + if tree.branch and not tree.unborn and tree.branch not in PROTECTED_BRANCHES: + tree.merge = self.evidence.judge(tree.path, slug_of(tree.origin_url), + tree.branch, tree.head) + + def _roots(self) -> list: + roots = [] + if self.facts.dir: + roots.append(os.path.join(self.facts.dir, ".claude", "worktrees")) + roots.append(os.path.join(os.path.dirname(self.facts.dir.rstrip("/")), + ".lane-worktrees", self.lane)) + elif os.environ.get("PROJECTS_ROOT"): + roots.append(os.path.join(os.environ["PROJECTS_ROOT"], ".lane-worktrees", self.lane)) + return roots + + def _discover(self) -> None: + seen: dict = {} + lane_dir = real(self.facts.dir) if self.facts.dir else "" + for rec in self.facts.inventory: + path = rec.get("path") or "" + if not path or rec.get("schema") not in ("1",): + if rec.get("id"): + self.notes.append(f"inventory record {rec.get('id')} is unreadable or of " + f"schema {rec.get('schema') or '?'}; nothing of it is read " + "and it is never swept") + continue + if lane_dir and real(path) == lane_dir: + continue + t = Tree(path, "inventory") + t.sidecar = rec + t.checkout = rec.get("checkout") if rec.get("checkout") not in ("", "unknown") else "" + seen[t.real] = t + checkouts = [] + if self.facts.dir and os.path.isdir(self.facts.dir): + checkouts.append(self.facts.dir) + for t in list(seen.values()): + if t.checkout and os.path.isdir(t.checkout) and t.checkout not in checkouts: + checkouts.append(t.checkout) + regs_by_real: dict = {} + for co in checkouts: + regs = worktree_registrations(co) + for i, reg in enumerate(regs): + if i == 0 or reg.get("bare"): + continue # the main worktree is the checkout itself + regs_by_real[real(reg["path"])] = (co, reg) + for rp, (co, reg) in regs_by_real.items(): + t = seen.get(rp) + if t is None: + t = Tree(reg["path"], "foreign", "registered") + seen[t.real] = t + t.registered = True + t.checkout = t.checkout or co + t.locked = bool(reg.get("locked")) + for root in self._roots(): + try: + names = sorted(os.listdir(root)) + except OSError: + continue + for name in names: + full = os.path.join(root, name) + if not os.path.isdir(full) or os.path.islink(full): + continue + if real(full) in seen: + continue + if not os.path.lexists(os.path.join(full, ".git")): + continue + kind = "clone" if os.path.isdir(os.path.join(full, ".git")) else "lane-root" + seen[real(full)] = Tree(full, "foreign", kind) + self.trees = sorted(seen.values(), key=lambda t: (t.owner != "inventory", t.path)) + + def _other_lanes(self) -> None: + """A FOREIGN tree another lane's inventory names is THAT lane's.""" + parents = set() + if self.facts.root: + parents.add(os.path.dirname(self.facts.root.rstrip("/"))) + if self.facts.dir: + parents.add(os.path.join(os.path.dirname(self.facts.dir.rstrip("/")), ".lane-state")) + if os.environ.get("LANES_LANE_STATE_ROOT"): + parents.add(os.environ["LANES_LANE_STATE_ROOT"]) + claimed: dict = {} + for parent in parents: + try: + lanes = os.listdir(parent) + except OSError: + continue + for other in lanes: + if other == self.lane: + continue + tdir = os.path.join(parent, other, "trees") + try: + files = os.listdir(tdir) + except OSError: + continue + for fname in files: + try: + with open(os.path.join(tdir, fname), encoding="utf-8") as fh: + for line in fh: + if line.startswith("path: "): + claimed[real(line[6:].strip())] = other + break + except OSError: + continue + for tree in self.trees: + if tree.owner == "foreign" and tree.real in claimed: + tree.foreign_kind = "other-lane" + tree.other_lane = claimed[tree.real] + + def _fetch_all(self) -> None: + commons: dict = {} + for tree in self.trees: + probe = tree.path if tree.exists else tree.checkout + if not probe or not os.path.isdir(probe): + continue + common = git_out(probe, "rev-parse", "--git-common-dir") + if common is None: + continue + if not os.path.isabs(common): + common = os.path.normpath(os.path.join(probe, common)) + commons.setdefault(real(common), probe) + for common, probe in commons.items(): + if not git_out(probe, "config", "--get", "remote.origin.url"): + self.fetched[common] = "no origin remote" + continue + proc = git(probe, "fetch", "--prune", "--quiet", "origin", timeout=GIT_NET_TIMEOUT) + self.fetched[common] = "ok" if proc.returncode == 0 else ( + "fetch failed: " + _first_line(proc.stderr)) + + def _liveness(self, tree: Tree) -> None: + if not tree.exists: + return + for named in self.opts.live: + if named != "none" and (under(tree.path, named) or under(named, tree.path)): + tree.live = (True, "the coordinator's writer count names it (--live)") + return + try: + here = os.getcwd() + except OSError: + here = "" + if here and under(here, tree.path): + tree.live = (True, "this sweep was started from inside it") + return + if self.procs is not None and self.procs.available: + holders = self.procs.holders(tree.path) + if holders: + pid, comm, kind, where = holders[0] + tree.live = (True, f"process {pid} ({comm or '?'}) has its {kind} there") + return + elif self.procs is not None: + tree.live = (None, "liveness could not be read: " + self.procs.why) + return + writer = (tree.sidecar or {}).get("writer", "") + if writer and UUID_RE.match(writer) and writer != self.caller: + answer = self.helper.transcript_live(writer) + if answer == "live": + tree.live = (True, f"the session that recorded it ({writer[:8]}) is live") + return + if answer == "unknown": + tree.live = (None, f"whether session {writer[:8]} is live could not be read") + return + if tree.owner == "foreign" and tree.is_git: + quiet = self.conf["foreign_quiet_hours"] * 3600 + newest = _gitdir_activity(tree.path) + if newest and time.time() - newest < quiet: + tree.live = (True, f"FOREIGN and active {int((time.time() - newest) // 60)} min ago " + f"(foreign trees quiet for {self.conf['foreign_quiet_hours']} h only)") + + def _lane_gate(self) -> None: + f = self.facts + if f.binding == "elsewhere": + self.refusal = (f"lane {self.lane} is bound on {f.binding_at or 'another place'}, and " + "liveness is pronounced only from inside that binding (Amendment " + "18(b)): from here it is UNKNOWN, never dead. Sweep it from there, or " + "ask it to hand off: lane-start --request-handoff") + return + if f.binding == "unknown": + self.refusal = (f"where lane {self.lane} is bound could not be read, and a read that " + "failed is never an answer (Amendment 7(d))") + return + if f.holder == "unknown": + self.refusal = (f"whether a live session holds lane {self.lane} could not be read " + f"({f.holder_detail or 'the session records'}); that is not 'none'") + return + if f.holder == "live" and f.holder_uuid != self.caller: + self.refusal = (f"lane {self.lane} is held by live session {f.holder_uuid or '?'} " + f"({f.holder_detail}); sweep it from inside that session, or after " + "it hands off") + return + if self.opts.yes and f.holder == "live" and f.state == "SWAPPING": + self.refusal = (f"lane {self.lane} is SWAPPING in a live session: a handoff is in " + "flight and a sweep does not compete with it") + return + if self.opts.yes and f.holder == "live" and not self.opts.live: + self.refusal = (f"lane {self.lane} is held by THIS session, and only its coordinator " + "knows which writers are live (ListAgents; Amendment 17 Addendum 1 " + "(i)). State the count: --live for each live writer, or " + "--live none") + return + if f.verdict == "indeterminate": + self.refusal = (f"#97's reconciliation of lane {self.lane} is INDETERMINATE: " + + (f.verdict_why or "a read it rests on failed")) + return + if self.opts.yes and self.opts.foreign and not self.opts.word: + self.refusal = ("--include-foreign acts on trees #97's inventory does not give to " + "this lane, and that takes a word: --word \"\"") + return + + # ---------------------------------------------------------- deciding + def decide(self) -> None: + for tree in self.trees: + self._decide(tree) + if self.opts.branches: + self._branches() + if self.opts.scratch: + self._scratch() + if self.opts.caches: + self._caches() + if self.opts.sandboxes: + self.items.extend(sandbox_items(self.procs or ProcScan(), self.conf)) + if self.opts.links and self.estate_root: + self.items.extend(link_items(self.estate_root)) + + def _decide(self, t: Tree) -> None: + mine = t.owner == "inventory" or (self.opts.foreign and t.foreign_kind != "other-lane") + + def put(disp: str, why: str, retire: bool = True) -> None: + t.disposition, t.reason, t.retire = disp, why, retire + + if t.owner == "foreign" and not mine: + where = {"registered": "registered in the lane's checkout", + "lane-root": "under this lane's worktree root", + "clone": "a standalone CLONE under this lane's root (a lane creates " + "worktrees, never clones)", + "other-lane": f"lane {t.other_lane}'s inventory names it"}.get(t.foreign_kind, "") + put("foreign", f"not in lane {self.lane}'s inventory (#97); {where}", False) + if t.lean: + t.reason += "; LOAD-BEARING: " + _lean_text(t.lean) + return + if not t.exists: + if t.registered: + if t.locked: + put("keep", "its directory is gone and the registration is LOCKED; " + f"git -C {t.checkout} worktree unlock {t.path} first") + else: + put("prune", "the directory is gone and git still registers it") + else: + put("gone", "the directory is gone and nothing registers it; the inventory " + "record is history", False) + return + if t.live is not None: + if t.live[0]: + put("live", "a live writer owns it: " + t.live[1]) + else: + put("keep", t.live[1]) + return + if not t.is_git: + put("keep", "the path exists and is no checkout; it is left exactly as it is") + return + if t.read_error: + put("keep", "git could not be read through it: " + t.read_error) + return + t.guard = tree_guards(t) + if t.guard: + put("keep", t.guard) + return + if t.lean: + put("load-bearing", "LOAD-BEARING: " + _lean_text(t.lean) + + "; remedy: in each dependent `git repack -a -d` then drop its " + "objects/info/alternates, or re-point the local remote - then sweep again") + return + if self.opts.fetch: + state = self.fetched.get(real(t.common), "") + if state and state != "ok": + put("keep", f"origin could not be read ({state}), so what is published is unknown") + return + if not t.origin_url: + if self.opts.bundle: + put("bundle+remove", "no origin remote: a bundle is its only rescue") + else: + put("keep", "no origin remote, so nothing in it can be published; --bundle " + "archives it instead") + return + dirty = len(t.status_lines) + if dirty: + put("wip-rescue+remove", f"{dirty} dirty or untracked path(s): a WIP commit on " + "a rescue branch, pushed, then removed") + return + if t.unborn: + put("remove", "an unborn branch with nothing in it") + return + if t.detached: + if t.on_origin: + put("remove", "detached at a commit origin holds") + elif t.on_origin is None: + put("keep", "detached, and whether origin holds the commit could not be read") + else: + put("rescue+remove", "detached with commits of its own: a rescue branch, pushed, " + "then removed") + return + m = t.merge or {} + if m.get("state") == "merged": + put("remove+delete-branch", m["why"] + "; the local branch is deleted, and the " + "remote one where it is this lane's and holds nothing the PR did not") + return + if t.remote_tip and t.ahead == 0: + put("remove", "clean, and every commit is on origin/" + t.remote_branch + + (f" ({m.get('why')})" if m.get("state") in ("open", "closed", "moved") else "")) + return + if not t.remote_tip and t.on_origin: + put("remove", "clean, and its head is on origin") + return + if t.branch in PROTECTED_BRANCHES or (t.remote_tip and t.behind > 0): + why = ("a default branch is never pushed by a sweep" if t.branch in PROTECTED_BRANCHES + else f"origin/{t.remote_branch} has {t.behind} commit(s) it lacks (diverged)") + put("rescue+remove", f"{t.ahead or 'unpublished'} commit(s) not on origin and {why}: " + "a rescue branch, pushed, then removed") + return + put("push+remove", (f"{t.ahead} commit(s) not on origin/{t.remote_branch}" + if t.remote_tip else f"origin has no {t.remote_branch}") + + ": pushed as is, then removed") + + def _branches(self) -> None: + checkouts = [] + if self.facts.dir and os.path.isdir(self.facts.dir): + checkouts.append(self.facts.dir) + for t in self.trees: + if t.owner == "inventory" and t.checkout and os.path.isdir(t.checkout) \ + and real(t.checkout) not in [real(c) for c in checkouts]: + checkouts.append(t.checkout) + for co in checkouts: + self.items.extend(branch_items(co, self.lane, self.evidence)) + + def _scratch(self) -> None: + if not self.facts.dir: + return + root = os.path.join(os.path.dirname(self.facts.dir.rstrip("/")), ".lane-worktrees", + self.lane) + try: + names = sorted(os.listdir(root)) + except OSError: + return + for name in names: + full = os.path.join(root, name) + if not (name.endswith("-scratch") or name in ("briefs", "bin")): + continue + if os.path.islink(full) or not os.path.isdir(full): + continue + if os.path.lexists(os.path.join(full, ".git")): + continue # a checkout is a tree, and the table above has it + holders = self.procs.holders(full) if self.procs and self.procs.available else [] + lean = leaned_on(full, "", self.deps) + if lean: + it = Item("scratch", full, "load-bearing", "LOAD-BEARING: " + _lean_text(lean) + + "; it is never removed until they are re-pointed", True) + elif holders: + pid, comm, kind, _w = holders[0] + it = Item("scratch", full, "live", f"process {pid} ({comm}) has its {kind} there", + True) + elif self.procs is not None and not self.procs.available: + it = Item("scratch", full, "keep", "liveness could not be read: " + self.procs.why, + True) + else: + it = Item("scratch", full, "archive+remove", + "lane scratch: tar and sha256 under the sweeps directory, then removed", + True) + it.size = dir_size(full) + self.items.append(it) + + def _caches(self) -> None: + roots = [] + if self.facts.dir: + roots.append(os.path.join(os.path.dirname(self.facts.dir.rstrip("/")), + ".lane-worktrees", self.lane)) + for t in self.trees: + if t.owner == "inventory" and t.exists and t.is_git \ + and not any(under(t.path, r) for r in roots): + roots.append(t.path) + live_paths = [t.path for t in self.trees if t.disposition == "live"] + # A CACHE INSIDE SCRATCH THAT IS BEING ARCHIVED GOES WITH THE SCRATCH + # (and the archive leaves caches out): two acts on one path is one too many. + archived = [i.path for i in self.items if i.kind == "scratch" + and i.disposition == "archive+remove"] + for root in roots: + for path, size in find_caches(root): + if any(under(path, lp) for lp in live_paths + archived): + continue + if self.procs and self.procs.available and self.procs.holders(path): + continue + if tracked(path): + continue + it = Item("cache", path, "remove", "a dependency cache or bytecode: it holds no " + "work and is rebuilt", False) + it.size = size + self.items.append(it) + + # ------------------------------------------------------------ output + def retire_count(self) -> int: + return (sum(1 for t in self.trees if t.retire) + + sum(1 for i in self.items if i.retire)) + + def render(self) -> None: + f = self.facts + mode = "YES - performing the table" if self.opts.yes else \ + "DRY RUN - nothing is changed; --yes performs it" + say(f"{PROG} sweep {self.lane} - {mode}") + holder = f.holder + if f.holder == "live": + holder = f"live {f.holder_uuid[:8]}" + (" (this session)" if f.holder_uuid == self.caller else "") + say(f" lane: state {f.state} · holder {holder} · binding {f.binding}" + + (f" ({f.binding_at})" if f.binding_at else "") + (f" · verdict {f.verdict}" if f.verdict else "")) + say(f" checkout: {f.dir or '(no recorded directory)'} · inventory " + f"{len(f.inventory)} tree(s) · origin " + + ("fetched now" if self.opts.fetch else "as of the last fetch")) + ev = self.evidence + say(" merged means: the register's LANDED line for the branch's PR, or gh's MERGED" + + (f" (gh: {ev.gh_why})" if ev.gh_why else "") + + (f" (register: {ev.register_why})" if ev.register_why else "")) + if self.refusal: + say("") + say(f"REFUSED: {self.refusal}") + for line in self.notes: + say(f" note: {line}") + say("") + say("TREES") + if not self.trees: + say(" (none: no inventory tree, no registration, nothing under the lane's roots)") + for t in self.trees: + res = f" -> {t.result}" if t.result else "" + say(f" {t.disposition:<22} {t.path}") + say(f" {'':<22} {t.label()} · {t.reason}{res}") + kinds = (("branch", "BRANCHES (--branches)"), ("scratch", "SCRATCH (--include-scratch)"), + ("cache", "CACHES (--include-caches)"), ("sandbox", "SANDBOXES (--include-sandboxes)"), + ("link", "BROKEN LINKS (--links)")) + for kind, title in kinds: + rows = [i for i in self.items if i.kind == kind] + if not rows: + continue + say("") + say(title) + for i in rows: + size = f" · {human_size(i.size)}" if i.size else "" + res = f" -> {i.result}" if i.result else "" + say(f" {i.disposition:<22} {i.path}{size}") + say(f" {'':<22} {i.detail + ' · ' if i.detail else ''}{i.reason}{res}") + say("") + foreign = sum(1 for t in self.trees if t.disposition == "foreign") + live = sum(1 for t in self.trees if t.disposition == "live") + say(f"SUMMARY: {len(self.trees)} tree(s): {self.retire_count()} to retire · " + f"{live} live · {foreign} foreign" + + (f" · archive {tilde(self.archive)}" if self.archive else "")) + + def porcelain(self) -> None: + f = self.facts + say("\t".join(["lane", self.lane, f.state or "-", f.holder or "-", + f.holder_uuid or "-", f.binding or "-", f.verdict or "-"])) + if self.refusal: + say("refused\t" + self.refusal.replace("\t", " ")) + for t in self.trees: + say("\t".join(["tree", t.disposition, "retire" if t.retire else "-", t.path, + t.branch or ("(detached)" if t.detached else "-"), t.head or "-", + t.reason.replace("\t", " ") + (f" -> {t.result}" if t.result else "")])) + for i in self.items: + say("\t".join([i.kind, i.disposition, "retire" if i.retire else "-", i.path, + i.detail or "-", str(i.size), + i.reason.replace("\t", " ") + (f" -> {i.result}" if i.result else "")])) + say(f"summary\t{self.retire_count()}\t{len(self.trees)}\t" + f"{sum(1 for t in self.trees if t.disposition == 'live')}\t" + f"{sum(1 for t in self.trees if t.disposition == 'foreign')}") + + # -------------------------------------------------------------- acting + def act(self) -> None: + os.umask(0o077) + for t in self.trees: + if t.disposition in ("live", "foreign", "keep", "gone", "load-bearing", ""): + continue + try: + self._act_tree(t) + except _ActFailed as exc: + t.result = "FAILED: " + str(exc) + self.failed += 1 + except OSError as exc: + t.result = f"FAILED: {exc}" + self.failed += 1 + self._record(t) + for i in self.items: + try: + self._act_item(i) + except (_ActFailed, OSError) as exc: + i.result = "FAILED: " + str(exc) + self.failed += 1 + self._register_items() + if self.archive: + self._write_disposition() + + def _archive_dir(self) -> str: + if not self.archive: + path = os.path.join(sweeps_dir(), safe_ref_part(self.lane), self.stamp) + os.makedirs(path, mode=0o700, exist_ok=True) + self.archive = path + return self.archive + + def _rescue_name(self, t: Tree) -> str: + base = f"rescue/{safe_ref_part(self.lane)}/{t.slice}-{self.stamp}" + name, n = base, 1 + while git(t.path, "rev-parse", "-q", "--verify", f"refs/heads/{name}").returncode == 0: + n += 1 + name = f"{base}-{n}" + return name + + def _act_tree(self, t: Tree) -> None: + # LIVENESS IS ASKED AGAIN AT THE MOMENT OF THE ACT, not trusted from the + # read: a writer may have started in the tree since the table was made. + if t.exists and self.procs is not None: + fresh = ProcScan() + if fresh.available and fresh.holders(t.path): + pid, comm, kind, _w = fresh.holders(t.path)[0] + t.disposition = "live" + t.result = f"left: process {pid} ({comm}) arrived in it" + return + if t.disposition == "prune": + proc = git(t.checkout, "worktree", "remove", t.path) + if proc.returncode != 0: + proc = git(t.checkout, "worktree", "prune") + if proc.returncode != 0: + raise _ActFailed("git worktree prune: " + _first_line(proc.stderr)) + regs = [r["path"] for r in worktree_registrations(t.checkout)] + if t.path in regs: + raise _ActFailed("the registration is still there after the prune") + t.result = "pruned" + self._note(t, f"swept {t.path}: stale registration pruned in {t.checkout}") + return + guard = tree_guards(t) if t.exists else "" + if guard: + t.disposition = "keep" + t.result = "left: " + guard + return + # 1. RESCUE OR PUBLISH FIRST - nothing below removes what is not on + # origin or in a bundle. + disp = t.disposition + ref_to_bundle = "" + wip_tree = "" + if disp == "wip-rescue+remove": + # THE WORK IS COMMITTED BESIDE THE TREE, NOT INTO IT: a temporary + # index takes `git add -A`, and `commit-tree` + `update-ref` make the + # rescue branch. The tree's own branch, index and files are never + # moved, so a push that fails leaves it exactly as it was. + name = self._rescue_name(t) + wip_tree = snapshot_tree(t.path) + if not wip_tree: + raise _ActFailed("the tree's content could not be snapshotted (git add -A into " + "a temporary index failed); left in place") + subject = f"WIP({self.lane}): {t.slice} swept {self.stamp}; swept, not reviewed" + body = (f"Rescued by {PROG} sweep from {t.path} (branch " + f"{t.branch or 'detached'} at {short(t.head) if t.head else 'no commit'}). " + "The tree held uncommitted or untracked work when its lane was swept; " + "this commit is that work as it stood, unreviewed.") + commit = self._commit_tree(t.path, wip_tree, t.head, subject, body) + self._git_or_fail(t.path, "update-ref", f"refs/heads/{name}", commit, "") + self._push(t, name, name) + ref_to_bundle = name + t.rescue = f"origin/{name}" + elif disp == "rescue+remove": + name = self._rescue_name(t) + self._git_or_fail(t.path, "update-ref", f"refs/heads/{name}", t.head, "") + self._push(t, name, name) + ref_to_bundle = name + t.rescue = f"origin/{name}" + elif disp == "push+remove": + self._push(t, t.branch, t.remote_branch or t.branch) + ref_to_bundle = t.branch if self.opts.bundle else "" + t.rescue = f"origin/{t.remote_branch or t.branch}" + elif disp in ("remove", "remove+delete-branch"): + ref_to_bundle = (t.branch or "HEAD") if self.opts.bundle and t.head else "" + elif disp == "bundle+remove": + ref_to_bundle = t.branch or "HEAD" + # 2. THE BUNDLE - always for a rescue, and for everything under --bundle. + if ref_to_bundle: + t.bundle = self._bundle(t, ref_to_bundle) + # 3. IGNORED WORK no commit could carry. + extra = ignored_work(t) + if extra: + size = sum(dir_size(os.path.join(t.path, r)) if os.path.isdir(os.path.join(t.path, r)) + else os.lstat(os.path.join(t.path, r)).st_size for r in extra + if os.path.lexists(os.path.join(t.path, r))) + cap = self.conf["ignored_archive_mb"] * 1024 * 1024 + if size > cap: + raise _ActFailed(f"{human_size(size)} of ignored files no commit can carry " + f"(over ignored_archive_mb={self.conf['ignored_archive_mb']}): " + "move or delete them by hand, or raise the cap; left in place" + + (f"; its rescue is at {t.rescue}" if t.rescue else "")) + t.ignored_archive = self._tar(t.path, extra, + f"{t.slice}-ignored.tar.gz") + # 4. NOTHING MOVED, PROVED AGAIN AT THE MOMENT OF REMOVAL: the head is the + # one classified, and the content is clean - or, for a WIP rescue, is + # byte for byte the tree the rescue commit carries. + head_now = git_out(t.path, "rev-parse", "-q", "--verify", "HEAD") or "" + if head_now != t.head: + raise _ActFailed(f"HEAD moved from {short(t.head) or 'none'} to " + f"{short(head_now) or 'none'} under the sweep; left in place" + + (f"; its rescue is at {t.rescue}" if t.rescue else "")) + if wip_tree: + if snapshot_tree(t.path) != wip_tree: + raise _ActFailed("the tree changed after its rescue was taken; left in place; " + f"the rescue is at {t.rescue}") + else: + again = git(t.path, "status", "--porcelain=v1", "--ignore-submodules=none", + "--untracked-files=normal") + if again.returncode != 0 or again.stdout.strip(): + raise _ActFailed("the tree changed under the sweep (git status is not clean); " + "left in place" + + (f"; its rescue is at {t.rescue}" if t.rescue else "")) + checkout = t.checkout or os.path.dirname(t.common.rstrip("/")) + if t.kind == "worktree": + co = t.checkout if t.checkout and os.path.isdir(t.checkout) else t.path + proc = git(co, "worktree", "remove", "--force", t.path) + if proc.returncode != 0: + raise _ActFailed("git worktree remove: " + _first_line(proc.stderr)) + else: + rmtree(t.path) + if os.path.exists(t.path): + raise _ActFailed("the directory is still there after the removal") + t.result = "removed" + ((f"; pushed to {t.rescue}" if disp == "push+remove" else + f"; rescue {t.rescue}") if t.rescue else "") + # 5. THE BRANCHES OF A MERGED TREE. + if disp == "remove+delete-branch" and t.branch: + co = t.checkout if t.checkout and os.path.isdir(t.checkout) else checkout + proc = git(co, "branch", "-D", t.branch) + t.result += "; local branch deleted" if proc.returncode == 0 else ( + "; local branch NOT deleted: " + _first_line(proc.stderr)) + head = (t.merge or {}).get("head", "") + ours = t.lane_trailer.lower() == self.lane.lower() + if t.remote_tip and ours and head and (t.remote_tip == head or git( + co, "merge-base", "--is-ancestor", t.remote_tip, head).returncode == 0): + proc = git(co, "push", "--no-verify", "origin", "--delete", t.remote_branch, + timeout=GIT_NET_TIMEOUT) + t.result += "; remote branch deleted" if proc.returncode == 0 else ( + "; remote branch NOT deleted: " + _first_line(proc.stderr)) + elif t.remote_tip: + t.result += ("; remote branch kept (" + ("its Lane: trailer is " + + (t.lane_trailer or "absent") if not ours else + "it holds commits the PR head does not") + ")") + self._note(t, f"swept {t.path}: {disp} (was {t.label()})" + + ((f"; pushed to {t.rescue}" if disp == "push+remove" else + f"; rescue {t.rescue}") if t.rescue else "") + + (f"; bundle {tilde(t.bundle)}" if t.bundle else "") + + (f"; ignored files {tilde(t.ignored_archive)}" if t.ignored_archive else "") + + f"; archive {tilde(self._archive_dir())}") + + def _git_or_fail(self, path: str, *args: str) -> str: + proc = git(path, *args) + if proc.returncode != 0: + raise _ActFailed(f"git {' '.join(args)}: " + _first_line(proc.stderr)) + return proc.stdout + + def _commit_tree(self, path: str, tree: str, parent: str, subject: str, body: str) -> str: + """A commit of `tree` on `parent`, made without touching any branch, + index or file - unsigned, because a rescue must not wait on a key.""" + args = [] + if not git_out(path, "config", "--get", "user.email"): + args += ["-c", "user.email=lane-worktrees@localhost", "-c", "user.name=lane-worktrees"] + msg = f"{subject}\n\n{body}\n\nLane: {self.lane}\n" + argv = ["git", "-C", path, *args, "commit-tree", "--no-gpg-sign", tree] + if parent: + argv += ["-p", parent] + proc = run(argv + ["-F", "-"], env=git_env(), input_text=msg) + if proc.returncode != 0 or not proc.stdout.strip(): + raise _ActFailed("git commit-tree: " + _first_line(proc.stderr + proc.stdout)) + return proc.stdout.strip() + + def _push(self, t: Tree, local: str, remote: str) -> None: + proc = git(t.path, "push", "--no-verify", "--quiet", "origin", + f"refs/heads/{local}:refs/heads/{remote}", timeout=GIT_NET_TIMEOUT) + if proc.returncode != 0: + raise _ActFailed(f"git push origin {local}:{remote}: " + _first_line(proc.stderr)) + local_sha = git_out(t.path, "rev-parse", f"refs/heads/{local}") or "" + seen = git(t.path, "ls-remote", "origin", f"refs/heads/{remote}", timeout=GIT_NET_TIMEOUT) + got = seen.stdout.split()[0] if seen.returncode == 0 and seen.stdout.split() else "" + if not local_sha or got != local_sha: + raise _ActFailed(f"origin does not show refs/heads/{remote} at {short(local_sha)} " + f"after the push (it shows {short(got) or 'nothing'}); the tree is left") + # ONLY A RESCUE BRANCH PINS ITS ARCHIVE (`--expire`): a branch pushed as + # is is the lane's own, and its later merge and deletion is no loss. + if remote.startswith("rescue/"): + self.rescues.append((t.origin_url, remote, local_sha, t.checkout or "", t.path)) + + def _bundle(self, t: Tree, ref: str) -> str: + """A bundle of `ref` holding what origin's default branch lacks (its + prerequisites are on origin), or the whole history where that would be + empty or origin has no default branch here.""" + out = os.path.join(self._archive_dir(), f"{t.slice}.bundle") + n = 1 + while os.path.exists(out): + n += 1 + out = os.path.join(self._archive_dir(), f"{t.slice}-{n}.bundle") + base = git_out(t.path, "symbolic-ref", "-q", "refs/remotes/origin/HEAD") or "" + if not base and git(t.path, "rev-parse", "-q", "--verify", + "refs/remotes/origin/main").returncode == 0: + base = "refs/remotes/origin/main" + proc = git(t.path, "bundle", "create", out, ref, "--not", base) if base else None + if proc is None or proc.returncode != 0: + if os.path.exists(out): + os.unlink(out) + proc = git(t.path, "bundle", "create", out, ref) + if proc.returncode != 0: + raise _ActFailed("git bundle create: " + _first_line(proc.stderr)) + check = git(t.path, "bundle", "verify", out) + if check.returncode != 0: + raise _ActFailed("git bundle verify: " + _first_line(check.stderr)) + return out + + def _tar(self, base: str, rels: list, name: str) -> str: + out = os.path.join(self._archive_dir(), name) + n = 1 + while os.path.exists(out): + n += 1 + out = os.path.join(self._archive_dir(), name.replace(".tar.gz", f"-{n}.tar.gz")) + try: + with tarfile.open(out, "w:gz") as tf: + for rel in rels: + tf.add(os.path.join(base, rel), arcname=rel, recursive=True, + filter=_no_caches) + with tarfile.open(out, "r:gz") as tf: + if not tf.getmembers(): + raise _ActFailed(f"the archive {out} came back empty") + except (OSError, tarfile.TarError) as exc: + raise _ActFailed(f"the archive {out} could not be written: {exc}") + return out + + def _act_item(self, i: Item) -> None: + if i.kind == "scratch" and i.disposition == "archive+remove": + fresh = ProcScan() + if fresh.available and fresh.holders(i.path): + i.disposition, i.result = "live", "left: a process arrived in it" + return + parent, name = os.path.dirname(i.path), os.path.basename(i.path) + i.extra["archive"] = self._tar(parent, [name], f"scratch-{safe_ref_part(name)}.tar.gz") + rmtree(i.path) + i.result = "archived and removed" + elif i.kind == "cache" and i.disposition == "remove": + rmtree(i.path) + i.result = "removed" + elif i.kind == "sandbox" and i.disposition == "remove": + fresh = ProcScan() + fresh.load_environ() + if not fresh.available or fresh.holders(i.path): + i.disposition, i.result = "live", "left: a process arrived in it" + return + rmtree(i.path) + i.result = "removed" + elif i.kind == "branch" and i.disposition in ("delete", "delete+remote"): + proc = git(i.extra["checkout"], "branch", "-D", i.extra["branch"]) + if proc.returncode != 0: + raise _ActFailed("git branch -D: " + _first_line(proc.stderr)) + i.result = "local branch deleted" + if i.disposition == "delete+remote": + proc = git(i.extra["checkout"], "push", "--no-verify", "origin", "--delete", + i.extra["branch"], timeout=GIT_NET_TIMEOUT) + i.result += "; remote branch deleted" if proc.returncode == 0 else ( + "; remote branch NOT deleted: " + _first_line(proc.stderr)) + + def _register_items(self) -> None: + done = [i for i in self.items if i.result and not i.result.startswith(("FAILED", "left"))] + scratch = [i for i in done if i.kind == "scratch"] + branches = [i for i in done if i.kind == "branch"] + cleaned = [i for i in done if i.kind in ("cache", "sandbox")] + if scratch: + self._note(None, "swept scratch: " + "; ".join( + f"{os.path.basename(i.path)} archived to {tilde(i.extra.get('archive', ''))}" + for i in scratch)) + if branches: + self._note(None, "swept branches merged by PR: " + "; ".join( + f"{i.extra['branch']} (PR #{i.extra.get('pr')}{' + remote' if 'remote' in i.result else ''})" + for i in branches)) + if cleaned: + total = sum(i.size for i in cleaned) + self._note(None, f"swept {len(cleaned)} cache or sandbox director(ies) " + f"({human_size(total)}): they hold no work and are rebuilt") + + def _note(self, t, text: str) -> None: + if self.opts.foreign and self.opts.word and t is not None and t.owner == "foreign": + text += f"; foreign on the word '{self.opts.word}'" + proc = self.helper.note_line(self.lane, text) + line = "NOTED " + utc_now() if proc.returncode == 0 else \ + f"NOT WRITTEN (lanes-edit.sh log exited {proc.returncode}: {_first_line(proc.stderr)})" + if t is not None: + t.register = line + if proc.returncode != 0: + self.failed += 1 + note(f"the register line for {t.path if t else 'this sweep'} was NOT written " + f"(exit {proc.returncode}); write it by hand: LANES_LANE={self.lane} " + f"lanes-edit.sh log NOTED lane:{self.lane} \"{register_text(text)}\"") + self.dispositions.append((t.path if t else "-", text, line)) + + def _record(self, t: Tree) -> None: + if not self.archive and (t.rescue or t.bundle or t.result): + self._archive_dir() + if self.archive: + self._write_disposition() + + def _write_disposition(self) -> None: + d = self._archive_dir() + lines = [f"# Sweep of lane {self.lane} - {self.stamp}", "", + f"Run by `{PROG} sweep {self.lane} --yes` on " + f"{os.environ.get('LANES_WORKSTATION') or socket.gethostname()} as " + f"{getpass.getuser()}. Nothing here was deleted that was not first on origin " + "or in this directory.", ""] + if self.opts.word: + lines += [f"Foreign trees were included on the word: \"{self.opts.word}\"", ""] + lines += ["| tree | was | disposition | why | result | rescue | bundle | register |", + "|---|---|---|---|---|---|---|---|"] + + def cell(text: str) -> str: + return (text or "-").replace("|", "\\|") + + for t in self.trees: + if not t.result: + continue + lines.append("| " + " | ".join(cell(x) for x in ( + t.path, t.label(), t.disposition, t.reason, t.result, t.rescue, + os.path.basename(t.bundle) if t.bundle else "", t.register)) + " |") + rows = [i for i in self.items if i.result] + if rows: + lines += ["", "| item | kind | disposition | result | size |", "|---|---|---|---|---|"] + for i in rows: + lines.append("| " + " | ".join(cell(x) for x in ( + i.path, i.kind, i.disposition, i.result, human_size(i.size))) + " |") + lines += ["", "## Register lines", ""] + for path, text, line in self.dispositions: + lines.append(f"- `{path}`: {line} - {text}") + lines.append("") + with open(os.path.join(d, "DISPOSITION.md"), "w", encoding="utf-8") as fh: + fh.write("\n".join(lines)) + with open(os.path.join(d, "rescues.tsv"), "w", encoding="utf-8") as fh: + fh.write("# origin\tbranch\tsha\tcheckout\ttree\n") + for row in self.rescues: + fh.write("\t".join(row) + "\n") + write_manifest(d) + + +class _ActFailed(Exception): + pass + + +def _lean_text(lean: list) -> str: + return "; ".join(f"{dep} leans on it ({how})" for dep, how in lean[:5]) + ( + f"; and {len(lean) - 5} more" if len(lean) > 5 else "") + + +def _gitdir_activity(path: str) -> float: + gitdir = git_out(path, "rev-parse", "--absolute-git-dir") + newest = 0.0 + if not gitdir: + return newest + for name in ("index", "HEAD", os.path.join("logs", "HEAD")): + try: + newest = max(newest, os.stat(os.path.join(gitdir, name)).st_mtime) + except OSError: + pass + return newest + + +def _no_caches(info): + """A tar member filter: caches and bytecode hold no work, so an archive + never carries them.""" + parts = info.name.split("/") + if any(p in CACHE_NAMES or p in VENV_NAMES for p in parts): + return None + if info.name.endswith((".pyc", ".pyo")): + return None + return info + + +def write_manifest(d: str) -> None: + rows = [] + for base, _dirs, files in os.walk(d): + for name in files: + full = os.path.join(base, name) + rel = os.path.relpath(full, d) + if rel == "MANIFEST.sha256": + continue + rows.append(f"{sha256_of(full)} {rel}") + with open(os.path.join(d, "MANIFEST.sha256"), "w", encoding="utf-8") as fh: + fh.write("\n".join(sorted(rows, key=lambda r: r.split(" ", 1)[1])) + "\n") + + +def tracked(path: str) -> bool: + parent = os.path.dirname(path.rstrip("/")) + proc = git(parent, "ls-files", "-z", "--", os.path.basename(path.rstrip("/"))) + return proc.returncode == 0 and bool(proc.stdout.strip("\0")) + + +def find_caches(root: str) -> list: + out = [] + for base, dirs, _files in os.walk(root, followlinks=False): + keep = [] + for d in dirs: + full = os.path.join(base, d) + if d == ".git" or os.path.islink(full): + continue + is_venv = os.path.isfile(os.path.join(full, "pyvenv.cfg")) or ( + d in VENV_NAMES and (os.path.exists(os.path.join(full, "bin", "activate")) + or os.path.exists(os.path.join(full, "Scripts", "activate")))) + if d in CACHE_NAMES or is_venv: + out.append((full, dir_size(full))) + else: + keep.append(d) + dirs[:] = keep + return out + + +def branch_items(checkout: str, lane: str, evidence: MergeEvidence) -> list: + """`--branches`: local branches NO worktree has checked out. Merged by + PR evidence: deleted (and the remote where it is this lane's and holds + nothing the PR did not). Unmerged with a missing, diverged or behind + upstream: LISTED with tip, distance and age, never deleted. A branch an + open PR names is never touched; rescue branches are never touched.""" + out = [] + regs = worktree_registrations(checkout) + held = {r.get("branch", "")[len("refs/heads/"):] for r in regs if r.get("branch")} + fmt = "%(refname:short)\t%(objectname)\t%(upstream:short)\t%(committerdate:iso-strict)" + proc = git(checkout, "for-each-ref", f"--format={fmt}", "refs/heads/") + if proc.returncode != 0: + return [Item("branch", checkout, "keep", "the branches could not be read: " + + _first_line(proc.stderr), True)] + origin = git_out(checkout, "config", "--get", "remote.origin.url") or "" + slug = slug_of(origin) + default = (git_out(checkout, "symbolic-ref", "-q", "--short", "refs/remotes/origin/HEAD") + or "origin/main") + for line in proc.stdout.splitlines(): + cols = line.split("\t") + if len(cols) < 4: + continue + name, tip, upstream, date = cols + if name in held or name in PROTECTED_BRANCHES or name.startswith("rescue/"): + continue + msg = git_out(checkout, "log", "-1", "--format=%B", tip) or "" + trailers = re.findall(r"^Lane:\s*(\S+)\s*$", msg, re.MULTILINE) + owner = trailers[-1] if trailers else "" + ours = owner.lower() == lane.lower() + counts = git_out(checkout, "rev-list", "--left-right", "--count", f"{default}...{tip}") or "" + behind_main, ahead_main = (counts.split() + ["?", "?"])[:2] + age = _age_days(date) + detail = (f"{short(tip)} · {ahead_main} ahead/{behind_main} behind {default} · last " + f"commit {date[:10]} ({age} d) · owner {owner or 'unknown'}") + judged = evidence.judge(checkout, slug, name, tip) if slug else { + "state": "unknown", "why": "not a GitHub origin"} + it = None + if judged["state"] == "open": + it = Item("branch", name, "keep", judged["why"] + ": an open PR's branch is never touched", + False, detail) + elif judged["state"] == "merged": + remote_tip = git_out(checkout, "rev-parse", "-q", "--verify", + f"refs/remotes/origin/{name}") or "" + head = judged.get("head", "") + remote_ok = bool(remote_tip) and ours and (remote_tip == head or git( + checkout, "merge-base", "--is-ancestor", remote_tip, head).returncode == 0) + it = Item("branch", name, "delete+remote" if remote_ok else "delete", + judged["why"] + ("; the remote branch too (this lane's, nothing beyond the PR)" + if remote_ok else ""), ours, detail) + it.extra["pr"] = judged.get("pr") + else: + up_state = "missing" + if upstream: + up_tip = git_out(checkout, "rev-parse", "-q", "--verify", f"refs/remotes/{upstream}") \ + if not upstream.startswith("refs/") else None + if up_tip: + c = (git_out(checkout, "rev-list", "--left-right", "--count", + f"{up_tip}...{tip}") or "0 0").split() + b, a = int(c[0]), int(c[1]) + up_state = ("diverged" if a and b else "ahead" if a else + "behind" if b else "published") + else: + up_state = "missing" + if up_state in ("published", "behind"): + continue + it = Item("branch", name, "list", f"unmerged ({judged.get('why', '')}); upstream " + f"{upstream or 'none'} is {up_state}: a person disposes of it (push it to " + f"rescue/{safe_ref_part(lane)}/..., or a word) - never deleted", ours, detail) + it.extra.update(checkout=checkout, branch=name) + out.append(it) + return out + + +def _age_days(iso: str) -> int: + try: + when = _dt.datetime.fromisoformat(iso.replace("Z", "+00:00")) + return max(0, (_dt.datetime.now(_dt.timezone.utc) - when).days) + except ValueError: + return 0 + + +def sandbox_roots() -> list: + named = os.environ.get("LANE_WORKTREES_SANDBOX_ROOTS", "") + if named: + return [p for p in named.split(os.pathsep) if p] + roots = [] + for cand in ("/tmp", os.environ.get("TMPDIR", ""), tempfile.gettempdir()): + if cand and os.path.isdir(cand) and real(cand) not in [real(r) for r in roots]: + roots.append(cand) + return roots + + +def sandbox_items(procs: ProcScan, conf: dict) -> list: + """Killed-suite sandboxes: `tmp.*` and `pytest-of-$USER/pytest-*` whose + owning process is gone, and `tests/run.sh`'s own run roots. They hold no + work; one a live process stands in, holds open or names in its + environment is never touched, and neither is one younger than the + minimum age (`sandbox_min_age_minutes`).""" + out = [] + uid = os.getuid() + procs.load_environ() + min_age = conf["sandbox_min_age_minutes"] * 60 + user = getpass.getuser() + candidates = [] + for root in sandbox_roots(): + try: + names = os.listdir(root) + except OSError: + continue + for name in names: + full = os.path.join(root, name) + if name.startswith("tmp.") and os.path.isdir(full) and not os.path.islink(full): + candidates.append((full, "mktemp", 0)) + if name == f"pytest-of-{user}" and os.path.isdir(full): + for sub in os.listdir(full): + sp = os.path.join(full, sub) + if sub.startswith("pytest-") and os.path.isdir(sp) and not os.path.islink(sp): + candidates.append((sp, "pytest", _lock_pid(os.path.join(sp, ".lock")))) + runs = os.path.join(state_dir(), "tmp") + try: + for name in os.listdir(runs): + full = os.path.join(runs, name) + if os.path.isdir(full) and not os.path.islink(full): + m = re.search(r"-(\d+)$", name) + candidates.append((full, "run.sh", int(m.group(1)) if m else 0)) + except OSError: + pass + now = time.time() + for path, kind, pid in candidates: + try: + st = os.lstat(path) + except OSError: + continue + if st.st_uid != uid: + continue + if pid and pid_alive(pid): + continue + if procs.available and procs.holders(path): + continue + if not procs.available: + continue + if kind == "mktemp" and now - st.st_mtime < min_age: + continue + it = Item("sandbox", path, "remove", f"a {kind} sandbox whose owning process is gone" + + (f" (pid {pid})" if pid else ""), False) + it.size = dir_size(path) + out.append(it) + return out + + +def _lock_pid(path: str) -> int: + try: + with open(path, encoding="utf-8") as fh: + text = fh.read().strip() + return int(text) if text.isdigit() else 0 + except (OSError, ValueError): + return 0 + + +def link_items(root: str) -> list: + """`--links`: every symlink under the estate and every worktree `.git` + gitdir pointer, checked to resolve; the broken ones, with target and age.""" + out = [] + prune = set(CACHE_NAMES) | set(VENV_NAMES) | {".git"} + now = time.time() + for base, dirs, files in os.walk(root, followlinks=False): + for name in list(dirs) + files: + full = os.path.join(base, name) + try: + st = os.lstat(full) + except OSError: + continue + if stat.S_ISLNK(st.st_mode): + if not os.path.exists(full): + try: + target = os.readlink(full) + except OSError: + target = "?" + age = int((now - st.st_mtime) // 86400) + out.append(Item("link", full, "broken", f"symlink to {target} that does not " + f"resolve ({age} d old)", False)) + elif name == ".git" and stat.S_ISREG(st.st_mode): + try: + with open(full, encoding="utf-8") as fh: + first = fh.readline().strip() + except OSError: + continue + if first.startswith("gitdir:"): + target = first[7:].strip() + resolved = target if os.path.isabs(target) else os.path.join(base, target) + if not os.path.isdir(resolved): + age = int((now - st.st_mtime) // 86400) + out.append(Item("link", full, "broken", f"gitdir pointer to {target} " + f"that does not resolve ({age} d old)", False)) + dirs[:] = [d for d in dirs if d not in prune and not os.path.islink(os.path.join(base, d))] + return out + + +# ================================================================== expire + +def expire(opts) -> int: + conf = load_conf() + root = sweeps_dir() + keep_days = conf["retention_days"] + now = _dt.datetime.now(_dt.timezone.utc) + rows = [] + try: + lanes = sorted(os.listdir(root)) + except OSError: + lanes = [] + for lane in lanes: + ldir = os.path.join(root, lane) + if not os.path.isdir(ldir): + continue + for stamp in sorted(os.listdir(ldir)): + adir = os.path.join(ldir, stamp) + if not os.path.isdir(adir): + continue + m = STAMP_RE.match(stamp) + if m: + when = _dt.datetime.strptime(m.group(1), "%Y%m%dT%H%M%SZ").replace( + tzinfo=_dt.timezone.utc) + else: + when = _dt.datetime.fromtimestamp(os.stat(adir).st_mtime, _dt.timezone.utc) + age = (now - when).days + if age < keep_days: + rows.append((adir, age, "keep", f"{age} d old; retention {keep_days} d")) + continue + missing = [] + for origin, branch, sha in _rescues_of(adir): + proc = run(["git", "ls-remote", origin, f"refs/heads/{branch}"], env=git_env(), + timeout=GIT_NET_TIMEOUT) + got = proc.stdout.split()[0] if proc.returncode == 0 and proc.stdout.split() else "" + if proc.returncode != 0: + missing.append(f"{branch} (origin could not be read)") + elif not got: + missing.append(f"{branch} (gone from origin)") + if missing: + rows.append((adir, age, "keep", "a rescue branch is not on origin: " + + "; ".join(missing) + " - this archive is the only copy")) + else: + rows.append((adir, age, "expire", f"{age} d old and every rescue branch is on origin")) + failed = 0 + for i, (adir, age, disp, why) in enumerate(rows): + if disp == "expire" and opts.yes: + try: + rmtree(adir) + rows[i] = (adir, age, "expired", why) + os.makedirs(root, exist_ok=True) + with open(os.path.join(root, "EXPIRED.log"), "a", encoding="utf-8") as fh: + fh.write(f"{utc_now()}\t{adir}\t{why}\n") + except OSError as exc: + rows[i] = (adir, age, "FAILED", str(exc)) + failed += 1 + if opts.porcelain: + for adir, age, disp, why in rows: + say("\t".join(["archive", disp, adir, str(age), why])) + else: + say(f"{PROG} sweep --expire - " + ("YES" if opts.yes else "DRY RUN") + + f" - retention {keep_days} d ({conf_path()})") + if not rows: + say(" (no archives under " + tilde(root) + ")") + for adir, age, disp, why in rows: + say(f" {disp:<8} {tilde(adir)} · {why}") + if failed: + return 1 + if not opts.yes and any(r[2] == "expire" for r in rows): + return 3 + return 0 + + +def _rescues_of(adir: str) -> list: + out = [] + try: + with open(os.path.join(adir, "rescues.tsv"), encoding="utf-8") as fh: + for line in fh: + if line.startswith("#") or not line.strip(): + continue + cols = line.rstrip("\n").split("\t") + if len(cols) >= 3 and cols[0] and cols[1]: + out.append((cols[0], cols[1], cols[2])) + except OSError: + pass + return out + + +# ==================================================================== main + +class _Parser(argparse.ArgumentParser): + def error(self, message): + raise Usage(message) + + +def parse(argv: list): + parser = _Parser(prog=PROG, add_help=False) + parser.add_argument("verb", nargs="?") + parser.add_argument("lane", nargs="?") + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--dry-run", action="store_true") + mode.add_argument("--yes", action="store_true") + parser.add_argument("--porcelain", action="store_true") + parser.add_argument("--bundle", action="store_true") + parser.add_argument("--branches", action="store_true") + parser.add_argument("--links", action="store_true") + parser.add_argument("--include-scratch", action="store_true") + parser.add_argument("--include-caches", action="store_true") + parser.add_argument("--include-sandboxes", action="store_true") + parser.add_argument("--include-foreign", action="store_true") + parser.add_argument("--word") + parser.add_argument("--live", action="append") + parser.add_argument("--fetch", action="store_true") + parser.add_argument("--estate") + parser.add_argument("--expire", action="store_true") + parser.add_argument("-h", "--help", action="store_true") + parser.add_argument("--version", action="store_true") + return parser.parse_args(argv) + + +def main(argv: list) -> int: + try: + ns = parse(argv) + except Usage as exc: + note(f"usage: {exc} (see {PROG} --help)") + return 64 + if ns.help: + sys.stdout.write(__doc__) + return 0 + if ns.version: + say(f"{PROG} {VERSION}") + return 0 + if ns.verb != "sweep": + note("usage: lane-worktrees sweep [--dry-run | --yes] ... (see --help)") + return 64 + if ns.expire: + if ns.lane: + note("usage: sweep --expire takes no lane: it reads every lane's archives") + return 64 + return expire(ns) + if not ns.lane: + note("usage: lane-worktrees sweep [--dry-run | --yes] ... (see --help)") + return 64 + if not re.match(r"^[A-Za-z0-9][A-Za-z0-9._-]*$", ns.lane): + note(f"usage: '{ns.lane}' is not a lane name") + return 64 + if ns.word is not None and not ns.include_foreign: + note("usage: --word is the word --include-foreign acts on, and nothing else takes it") + return 64 + opts = Options(ns) + sweep = Sweep(ns.lane, opts) + sweep.gather() + if sweep.refusal and not sweep.trees: + sweep.decide() + (sweep.porcelain if opts.porcelain else sweep.render)() + if not opts.porcelain: + note("REFUSED: " + sweep.refusal) + return 2 + sweep.decide() + if opts.yes and not sweep.refusal: + sweep.act() + (sweep.porcelain if opts.porcelain else sweep.render)() + if sweep.refusal: + return 2 + if opts.yes: + return 1 if sweep.failed else 0 + return 3 if sweep.retire_count() else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 8b49339a05bb3501ecd540dff55381df8fb40e1b Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 20:24:11 +0000 Subject: [PATCH 02/29] Test the sweep against every row of #162's disposition table tests/test_lane_worktrees.py builds a lane's estate under tmp_path: a bare origin reached through a url.insteadOf for a GitHub-shaped URL, the lane's checkout and both worktree roots, and fakes for lanes-edit.sh, gh and tmux. One lane holds a tree in every state, and the dry run is held to the table row for row and to a whole-estate snapshot that does not move. The same lane under --yes is held to every act: pushes, rescue branches seen on origin, the WIP commit's subject and parent, the bundles, the ignored-file archive without bytecode, a manifest that verifies, and one register line per tree. Also: a refused push leaves the dirty tree byte for byte; a live writer (a process's cwd, a tmux pane) is never touched; this session's --yes needs its writer count; bound elsewhere, held elsewhere and unreadable are refused with 2 and change nothing; merged by register and by gh against a branch on main by ancestry alone; no gh, nothing merged; --include-foreign takes a word and never another lane's tree; --branches; scratch, caches and sandboxes; --links; a LOAD-BEARING clone; --expire at 89/90/120 days; the porcelain exit codes; and two cases on the REAL lanes-edit.sh, for the inventory, the register line and a lane bound on another host. Part of #162. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- tests/test_lane_worktrees.py | 953 +++++++++++++++++++++++++++++++++++ 1 file changed, 953 insertions(+) create mode 100644 tests/test_lane_worktrees.py diff --git a/tests/test_lane_worktrees.py b/tests/test_lane_worktrees.py new file mode 100644 index 0000000..43d39bc --- /dev/null +++ b/tests/test_lane_worktrees.py @@ -0,0 +1,953 @@ +# SPDX-License-Identifier: Apache-2.0 +"""`lane-worktrees sweep` - one act that archives, rescues and removes a lane's +abandoned worktrees (opensoft/openRepoTools#162). + +What these tests hold it to, row for row with the issue's disposition table: + + * a tree a LIVE writer stands in is never touched; + * clean and MERGED (by the register's LANDED line or `gh`, never by + ancestry alone) is removed with its branches; + * clean and pushed is removed and its branch stays; + * unpushed commits are pushed as is - or, where origin diverged, to a + rescue branch - and only then removed; + * dirty or untracked work becomes a WIP commit on a rescue branch, pushed, + then removed; detached with commits of its own is rescued the same way; + * a registration whose directory is gone is pruned; + * a FOREIGN tree is reported and left; + * the rescue is ON ORIGIN before anything is removed - a refused push leaves + the tree exactly as it was; + * the dry run changes nothing at all; + * `--yes` is refused for a lane bound elsewhere (Amendment 18(b)); + * `--branches`, `--links`, alternates, `--expire` and the porcelain exit + codes `lane-end`'s gate (#163) reads. + +NOTHING REAL IS SWEPT. Every estate is built under `tmp_path`: bare +repositories stand in for origin (reached through a `url..insteadOf` for +a GitHub-shaped URL, so the code under test reads a real `owner/repo` slug), +`gh`, `tmux` and `lanes-edit.sh` are fakes on `PATH`, and the sandbox sweep is +pointed at a root of its own. One case runs the REAL `lanes-edit.sh` against a +fixture workspace, so the inventory, the reconciliation and the register line +are the helper's own. +""" + +from __future__ import annotations + +import datetime as _dt +import hashlib +import json +import os +import shutil +import signal +import subprocess +import sys +import tarfile +import time +from pathlib import Path + +import pytest + +from conftest import REPO, WINDOWS_SKIP + +pytestmark = [WINDOWS_SKIP, pytest.mark.skipif( + shutil.which("git") is None or shutil.which("bash") is None, + reason="the sweep drives git, and the lane helper is bash")] + +LW = REPO / "lane-worktrees" +LANES_EDIT = REPO / "lanes-edit.sh" +LANE = "repoA-1" +ME = "11111111-1111-4111-8111-111111111111" +OTHER = "22222222-2222-4222-8222-222222222222" +SLUG = "opensoft/repoA" +GH_URL = f"git@github.com:{SLUG}.git" +US = "\x1f" + +FAKE_HELPER = r'''#!/usr/bin/env python3 +import json, os, sys +spec = json.load(open(os.environ["FAKE_HELPER_SPEC"])) +args = sys.argv[1:] +with open(os.environ["FAKE_HELPER_LOG"], "a") as fh: + fh.write(json.dumps({"argv": args, "lane": os.environ.get("LANES_LANE", ""), + "no_fetch": os.environ.get("LANES_NO_FETCH", "")}) + "\n") +ans = spec.get(args[0] if args else "", {"rc": 2, "err": "unknown subcommand\n"}) +sys.stdout.write(ans.get("out", "")) +sys.stderr.write(ans.get("err", "")) +sys.exit(ans.get("rc", 0)) +''' + +FAKE_GH = r'''#!/usr/bin/env python3 +import os, sys +if sys.argv[1:3] == ["pr", "list"]: + path = os.environ.get("FAKE_GH_PRS", "") + sys.stdout.write(open(path).read() if path and os.path.exists(path) else "[]") + sys.exit(0) +sys.exit(1) +''' + +FAKE_TMUX = """#!/bin/sh +if [ -n "${FAKE_TMUX_PANES:-}" ] && [ -f "$FAKE_TMUX_PANES" ]; then cat "$FAKE_TMUX_PANES"; exit 0; fi +exit 1 +""" + + +def _clean_path() -> str: + """PATH without any directory holding an installed lane command.""" + keep = [] + for part in os.environ.get("PATH", "").split(os.pathsep): + if part and not any(os.path.exists(os.path.join(part, n)) + for n in ("lanes-edit.sh", "lane-worktrees", "lanes-index")): + keep.append(part) + return os.pathsep.join(keep) + + +class Estate: + """One lane's estate under `tmp_path`: a bare origin, the lane's + checkout, its two worktree roots, the fakes, and the environment the + sweep is run in.""" + + def __init__(self, root: Path): + self.root = root + self.home = root / "home" + self.projects = root / "projects" + self.remotes = root / "remotes" + self.fakebin = root / "fakebin" + self.state = root / "state" + self.config = root / "config" + self.sandboxes = root / "sandboxes" + for d in (self.home, self.projects, self.remotes, self.fakebin, self.state, + self.config, self.sandboxes, root / "tmp"): + d.mkdir(parents=True, exist_ok=True) + self.origin = self.remotes / "repoA.git" + self.checkout = self.projects / "repoA" + self.claude_root = self.checkout / ".claude" / "worktrees" + self.lane_root = self.projects / ".lane-worktrees" / LANE + self.spec_file = root / "helper-spec.json" + self.helper_log = root / "helper.log" + self.prs_file = root / "prs.json" + for name, text in (("lanes-edit.sh", FAKE_HELPER), ("gh", FAKE_GH), ("tmux", FAKE_TMUX)): + (self.fakebin / name).write_text(text) + (self.fakebin / name).chmod(0o755) + (self.home / ".gitconfig").write_text( + f'[url "{self.origin}"]\n\tinsteadOf = {GH_URL}\n' + "[user]\n\tname = sweep tests\n\temail = sweep@example.invalid\n" + "[init]\n\tdefaultBranch = main\n" + '[protocol "file"]\n\tallow = always\n' + "[advice]\n\tdetachedHead = false\n") + env = {k: v for k, v in os.environ.items() + if not k.startswith(("LANES_", "CLAUDE_", "WORKBENCHES_", "TMUX", "XDG_", + "PROJECTS_", "GH_", "GIT_", "FAKE_", "LANE_WORKTREES_", + "AGENT_PROTOCOL_ROOT"))} + env.update( + HOME=str(self.home), TMPDIR=str(root / "tmp"), + XDG_STATE_HOME=str(self.state), XDG_CONFIG_HOME=str(self.config), + PROJECTS_ROOT=str(self.projects), LANES_EDIT=str(self.fakebin / "lanes-edit.sh"), + FAKE_HELPER_SPEC=str(self.spec_file), FAKE_HELPER_LOG=str(self.helper_log), + FAKE_GH_PRS=str(self.prs_file), CLAUDE_CODE_SESSION_ID=ME, + LANE_WORKTREES_SANDBOX_ROOTS=str(self.sandboxes), LANES_WORKSTATION="Eagle", + GIT_CONFIG_NOSYSTEM="1", + PATH=str(self.fakebin) + os.pathsep + _clean_path()) + self.env = env + self.inventory: list = [] + self.prs: list = [] + self.holder = "none" + self.binding = ("here", "Eagle/none") + self.state_word = "RUNNING" + self.verdict = "ungraceful-stop" + self.workspace = "" + self.log_rc = 0 + self.extra_spec: dict = {} + self._seed() + + # ------------------------------------------------------------ git + def git(self, *args, cwd=None, check=True, env=None) -> str: + child = dict(self.env) + child.update(env or {}) + proc = subprocess.run(["git", *[str(a) for a in args]], cwd=str(cwd or self.checkout), + capture_output=True, text=True, env=child, check=False) + if check: + assert proc.returncode == 0, f"git {' '.join(map(str, args))}: {proc.stderr}" + return proc.stdout.strip() + + def _seed(self) -> None: + self.git("init", "-q", "--bare", "-b", "main", self.origin, cwd=self.root) + self.git("clone", "-q", GH_URL, self.checkout, cwd=self.root) + (self.checkout / "README.md").write_text("repoA\n") + (self.checkout / ".gitignore").write_text(".env\n__pycache__/\n*.pyc\n") + self.git("add", "-A") + self.commit(self.checkout, "seed") + self.git("push", "-q", "origin", "main") + + def commit(self, cwd: Path, message: str, files: dict | None = None, + lane: str | None = LANE) -> str: + for rel, text in (files or {}).items(): + path = Path(cwd) / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + self.git("add", "-A", cwd=cwd) + body = f"{message}\n\nLane: {lane}\n" if lane else message + self.git("commit", "-q", "--allow-empty", "-m", body, cwd=cwd) + return self.git("rev-parse", "HEAD", cwd=cwd) + + def worktree(self, name: str, branch: str | None = None, where: str = "lane", + start: str = "origin/main", record: bool = True) -> Path: + root = self.lane_root if where == "lane" else self.claude_root + root.mkdir(parents=True, exist_ok=True) + path = root / name + if branch: + self.git("worktree", "add", "-q", "-b", branch, path, start) + else: + self.git("worktree", "add", "-q", "--detach", path, start) + if record: + self.record(path) + return path + + def record(self, path: Path, branch: str = "", head: str = "") -> None: + """An inventory row, as `lanes-edit.sh lane-trees` prints one.""" + if path.is_dir(): + branch = self.git("symbolic-ref", "-q", "--short", "HEAD", cwd=path, check=False) or "detached" + head = self.git("rev-parse", "HEAD", cwd=path) + row = [f"c{len(self.inventory)}-{path.name}", str(path), branch or "unknown", + head or "unknown", "none", "0", "0", ME, "2026-10-05T00:00:00Z", + str(self.checkout), "1", "op-1", "1"] + self.inventory.append(US.join(row)) + + # ------------------------------------------------------------ the fakes + def write_spec(self) -> None: + holder = {"none": "HOLDER" + US + "none", + "me": f"HOLDER{US}live{US}{ME} sess:1.0 {LANE} 4242 here", + "other": f"HOLDER{US}live{US}{OTHER} sess:2.0 {LANE} 4343 here", + "unknown": f"HOLDER{US}unknown{US}the records could not be read"}[self.holder] + reconcile = "\n".join([ + f"ROOT{US}{self.root}/lane-state/{LANE}", + f"STATE{US}{self.state_word}{US}generation 1{US}operation op-1{US}owner {ME}{US}updated now", + holder, + f"BINDING{US}{self.binding[0]}{US}{self.binding[1]}", + f"TREES{US}{len(self.inventory)} inventoried", + f"VERDICT{US}{self.verdict}{US}the fixture says so", ""]) + spec = { + "lane-reconcile": {"rc": 0, "out": reconcile}, + "lane-dir": {"rc": 0, "out": f"{self.checkout}\n"}, + "lane-trees": ({"rc": 0, "out": "\n".join(self.inventory) + "\n"} + if self.inventory else {"rc": 8}), + "managed-projection": {"rc": 8}, + "workspace-root": ({"rc": 0, "out": self.workspace + "\n"} if self.workspace + else {"rc": 1, "err": "no workspace\n"}), + "transcript-holders": {"rc": 8}, + "log": {"rc": self.log_rc, "err": "" if self.log_rc == 0 else "refused\n"}, + } + spec.update(self.extra_spec) + self.spec_file.write_text(json.dumps(spec)) + self.prs_file.write_text(json.dumps(self.prs)) + + def pr(self, number: int, branch: str, state: str, head: str) -> None: + self.prs.append({"number": number, "headRefName": branch, "state": state, + "headRefOid": head, "isCrossRepository": False, + "mergedAt": "2026-10-01T00:00:00Z" if state == "MERGED" else None}) + + # ------------------------------------------------------------ running + def sweep(self, *args, env: dict | None = None, timeout: int = 180, + cwd: Path | None = None) -> subprocess.CompletedProcess: + self.write_spec() + child = dict(self.env) + for key, value in (env or {}).items(): + if value is None: + child.pop(key, None) + else: + child[key] = value + return subprocess.run([sys.executable, str(LW), "sweep", *[str(a) for a in args]], + capture_output=True, text=True, env=child, timeout=timeout, + cwd=str(cwd or self.root), stdin=subprocess.DEVNULL) + + def notes(self) -> list: + if not self.helper_log.exists(): + return [] + rows = [json.loads(line) for line in self.helper_log.read_text().splitlines() if line] + return [r for r in rows if r["argv"][:1] == ["log"]] + + def remote_heads(self) -> dict: + out = self.git("ls-remote", "--heads", self.origin, cwd=self.root) + heads = {} + for line in out.splitlines(): + sha, ref = line.split("\t") + heads[ref[len("refs/heads/"):]] = sha + return heads + + def archives(self) -> list: + base = self.state / "openRepoTools" / "sweeps" / LANE + return sorted(base.iterdir()) if base.is_dir() else [] + + +def rows_of(out: str) -> dict: + """The porcelain's tree rows, by path: (disposition, retire, reason).""" + rows = {} + for line in out.splitlines(): + cols = line.split("\t") + if cols[0] == "tree": + rows[cols[3]] = (cols[1], cols[2], cols[6] if len(cols) > 6 else "") + return rows + + +def items_of(out: str, kind: str) -> dict: + rows = {} + for line in out.splitlines(): + cols = line.split("\t") + if cols[0] == kind: + rows[cols[3]] = (cols[1], cols[2], cols[4], cols[6] if len(cols) > 6 else "") + return rows + + +def snapshot(root: Path, skip: tuple = ()) -> dict: + """Every path under `root` with its kind, size, mode, mtime and content + digest - what a run that CHANGES NOTHING leaves exactly as it found.""" + out = {} + for base, dirs, files in os.walk(root): + for name in dirs + files: + full = os.path.join(base, name) + rel = os.path.relpath(full, root) + if any(rel == s or rel.startswith(s + os.sep) for s in skip): + continue + st = os.lstat(full) + digest = "" + if os.path.islink(full): + digest = "->" + os.readlink(full) + elif os.path.isfile(full): + with open(full, "rb") as fh: + digest = hashlib.sha256(fh.read()).hexdigest() + out[rel] = (st.st_mode, st.st_size if not os.path.isdir(full) else 0, + st.st_mtime_ns if not os.path.isdir(full) else 0, digest) + return out + + +@pytest.fixture +def estate(tmp_path): + return Estate(tmp_path) + + +def build_table(e: Estate) -> dict: + """A lane whose inventory holds a tree in every state of the table.""" + t = {} + # clean, squash-MERGED: gh says MERGED and its head is this tip; the + # squash commit on main is NOT a descendant of it. + t["merged"] = e.worktree("merged", "feat/merged") + tip = e.commit(t["merged"], "merged work", {"m.txt": "m\n"}) + e.git("push", "-q", "-u", "origin", "feat/merged", cwd=t["merged"]) + e.git("checkout", "-q", "main") + e.git("merge", "-q", "--squash", "feat/merged") + e.commit(e.checkout, "squash of feat/merged (#11)") + e.git("push", "-q", "origin", "main") + e.pr(11, "feat/merged", "MERGED", tip) + # clean, pushed, unmerged (an open PR) + t["pushed"] = e.worktree("pushed", "feat/pushed") + tip = e.commit(t["pushed"], "pushed work", {"p.txt": "p\n"}) + e.git("push", "-q", "-u", "origin", "feat/pushed", cwd=t["pushed"]) + e.pr(12, "feat/pushed", "OPEN", tip) + # clean, one commit beyond its pushed upstream + t["unpushed"] = e.worktree("unpushed", "feat/unpushed") + e.commit(t["unpushed"], "first", {"u.txt": "1\n"}) + e.git("push", "-q", "-u", "origin", "feat/unpushed", cwd=t["unpushed"]) + e.commit(t["unpushed"], "second", {"u.txt": "2\n"}) + # clean, never pushed at all + t["fresh"] = e.worktree("fresh", "feat/fresh", where="claude") + e.commit(t["fresh"], "fresh work", {"f.txt": "f\n"}) + # clean, DIVERGED from its remote branch + t["diverged"] = e.worktree("diverged", "feat/diverged") + e.commit(t["diverged"], "d1", {"d.txt": "1\n"}) + e.commit(t["diverged"], "d2", {"d.txt": "2\n"}) + e.git("push", "-q", "-u", "origin", "feat/diverged", cwd=t["diverged"]) + e.git("reset", "-q", "--hard", "HEAD~1", cwd=t["diverged"]) + e.commit(t["diverged"], "d3", {"d.txt": "3\n"}) + # dirty: a modified file, an untracked one, an ignored secret, a cache + t["dirty"] = e.worktree("dirty", "feat/dirty") + e.commit(t["dirty"], "base", {"w.txt": "base\n"}) + e.git("push", "-q", "-u", "origin", "feat/dirty", cwd=t["dirty"]) + (t["dirty"] / "w.txt").write_text("changed\n") + (t["dirty"] / "new.txt").write_text("untracked\n") + (t["dirty"] / ".env").write_text("SECRET=1\n") + (t["dirty"] / "__pycache__").mkdir() + (t["dirty"] / "__pycache__" / "x.cpython-312.pyc").write_bytes(b"\0bytecode") + # detached at a commit origin holds + t["det-on"] = e.worktree("det-on") + # detached with a commit of its own + t["det-own"] = e.worktree("det-own") + e.commit(t["det-own"], "orphaned work", {"o.txt": "o\n"}) + # a registration whose directory is gone + t["stale"] = e.worktree("stale", "feat/stale") + shutil.rmtree(t["stale"]) + # a FOREIGN tree: git registers it, no inventory names it + t["foreign"] = e.worktree("foreign", "feat/foreign", where="claude", record=False) + # an inventory record whose path never existed + t["gone"] = e.lane_root / "gone" + e.record(t["gone"], "feat/gone", "0" * 40) + # a locked registration + t["locked"] = e.worktree("locked", "feat/locked") + e.git("worktree", "lock", t["locked"]) + # a clone nested inside a tree: its work is no commit of the tree's + t["nested"] = e.worktree("nested", "feat/nested") + e.git("push", "-q", "-u", "origin", "feat/nested", cwd=t["nested"]) + e.git("init", "-q", t["nested"] / "inner", cwd=e.root) + return t + + +# =================================================================== the table + +def test_every_disposition_row_in_the_dry_run_and_it_changes_nothing(estate): + t = build_table(estate) + before = snapshot(estate.root, skip=("helper.log", "helper-spec.json", "prs.json")) + proc = estate.sweep(LANE, "--porcelain") + after = snapshot(estate.root, skip=("helper.log", "helper-spec.json", "prs.json")) + assert proc.returncode == 3, proc.stdout + proc.stderr + rows = rows_of(proc.stdout) + want = { + "merged": "remove+delete-branch", "pushed": "remove", "unpushed": "push+remove", + "fresh": "push+remove", "diverged": "rescue+remove", "dirty": "wip-rescue+remove", + "det-on": "remove", "det-own": "rescue+remove", "stale": "prune", + "foreign": "foreign", "gone": "gone", "locked": "keep", "nested": "keep", + } + for name, disposition in want.items(): + assert str(t[name]) in rows, f"{name} is not in the table:\n{proc.stdout}" + assert rows[str(t[name])][0] == disposition, (name, rows[str(t[name])]) + assert rows[str(t["foreign"])][1] == "-", "a FOREIGN tree is not the lane's to retire" + assert rows[str(t["gone"])][1] == "-" + assert rows[str(t["merged"])][1] == "retire" + assert "PR #11 is merged (gh MERGED)" in rows[str(t["merged"])][2] + assert "LOCKED" in rows[str(t["locked"])][2] + assert "nested" in rows[str(t["nested"])][2] + # ZERO CHANGE: not a ref, not an index, not a file, not a sweeps directory. + assert before == after, sorted(set(before.items()) ^ set(after.items()))[:10] + assert not (estate.state / "openRepoTools").exists() + assert estate.notes() == [], "a dry run writes no register line" + + +def test_yes_performs_the_table_and_records_every_act(estate): + t = build_table(estate) + heads_before = estate.remote_heads() + tips = {n: estate.git("rev-parse", "HEAD", cwd=p) for n, p in t.items() + if n not in ("stale", "gone") and p.is_dir()} + proc = estate.sweep(LANE, "--yes", "--porcelain") + assert proc.returncode == 0, proc.stdout + proc.stderr + heads = estate.remote_heads() + for name in ("merged", "pushed", "unpushed", "fresh", "diverged", "dirty", "det-on", "det-own"): + assert not t[name].exists(), f"{name} was not removed:\n{proc.stdout}" + # MERGED: both branches gone (its Lane: trailer is this lane's). + assert "feat/merged" not in heads + assert estate.git("branch", "--list", "feat/merged") == "" + # PUSHED: the branch stays, here and on origin. + assert heads["feat/pushed"] == heads_before["feat/pushed"] + assert estate.git("branch", "--list", "feat/pushed").strip() == "feat/pushed" + # UNPUSHED and FRESH: pushed as is. + assert heads["feat/unpushed"] == tips["unpushed"] + assert heads["feat/fresh"] == tips["fresh"] + rescues = {b: s for b, s in heads.items() if b.startswith(f"rescue/{LANE}/")} + # DIVERGED: its tip went to a rescue branch; the remote branch is untouched. + assert heads["feat/diverged"] == heads_before["feat/diverged"] + assert [b for b, s in rescues.items() if b.startswith(f"rescue/{LANE}/diverged-") + and s == tips["diverged"]], rescues + # DETACHED with its own commit: rescued too. + assert [b for b, s in rescues.items() if b.startswith(f"rescue/{LANE}/det-own-") + and s == tips["det-own"]], rescues + # DIRTY: a WIP commit on a rescue branch, carrying the work, named for it. + wip = [b for b in rescues if b.startswith(f"rescue/{LANE}/dirty-")] + assert len(wip) == 1, rescues + subject = estate.git("log", "-1", "--format=%s", rescues[wip[0]], cwd=estate.origin) + assert LANE in subject and "dirty" in subject and "swept, not reviewed" in subject + stamp = wip[0].rsplit("-", 1)[1] + assert stamp in subject + assert estate.git("show", f"{rescues[wip[0]]}:w.txt", cwd=estate.origin) == "changed" + assert estate.git("show", f"{rescues[wip[0]]}:new.txt", cwd=estate.origin) == "untracked" + parent = estate.git("rev-parse", f"{rescues[wip[0]]}^", cwd=estate.origin) + assert parent == tips["dirty"], "the WIP commit sits on the tree's own head" + # STALE: pruned. + assert str(t["stale"]) not in estate.git("worktree", "list", "--porcelain") + # LEFT ALONE: foreign, locked, nested. + for name in ("foreign", "locked", "nested"): + assert t[name].is_dir(), f"{name} was touched" + # THE ARCHIVE: one directory, a manifest that verifies, the disposition, + # the rescues, a bundle per rescue, the ignored secret and no bytecode. + (archive,) = estate.archives() + files = {p.name for p in archive.iterdir()} + assert {"MANIFEST.sha256", "DISPOSITION.md", "rescues.tsv"} <= files + for line in (archive / "MANIFEST.sha256").read_text().splitlines(): + digest, name = line.split(" ", 1) + assert hashlib.sha256((archive / name).read_bytes()).hexdigest() == digest, name + assert any(n.startswith("dirty") and n.endswith(".bundle") for n in files), files + assert any(n.startswith("diverged") and n.endswith(".bundle") for n in files), files + with tarfile.open(archive / "dirty-ignored.tar.gz") as tf: + names = tf.getnames() + assert ".env" in names and not any("__pycache__" in n for n in names), names + disposition = (archive / "DISPOSITION.md").read_text() + for name in ("merged", "pushed", "dirty", "stale"): + assert str(t[name]) in disposition + # ONE REGISTER LINE PER TREE ACTED ON, and no line breaks the grammar. + notes = estate.notes() + acted = ("merged", "pushed", "unpushed", "fresh", "diverged", "dirty", "det-on", + "det-own", "stale") + assert len(notes) == len(acted), [n["argv"] for n in notes] + for n in notes: + assert n["argv"][:3] == ["log", "NOTED", f"lane:{LANE}"] and n["lane"] == LANE + text = n["argv"][3] + assert " — " not in text and ", " not in text and "managed" not in text.lower() + for name in acted: + assert any(str(t[name]) in n["argv"][3] for n in notes), name + + +def test_the_rescue_is_on_origin_before_the_tree_is_removed(estate): + """A refused push leaves the tree EXACTLY as it was: same branch, same + index, same files - the WIP commit is made beside it, never into it.""" + tree = estate.worktree("dirty", "feat/dirty") + estate.commit(tree, "base", {"w.txt": "base\n"}) + estate.git("push", "-q", "-u", "origin", "feat/dirty", cwd=tree) + (tree / "w.txt").write_text("precious\n") + (tree / "new.txt").write_text("also precious\n") + hook = estate.origin / "hooks" / "pre-receive" + hook.write_text("#!/bin/sh\nwhile read old new ref; do case $ref in " + "refs/heads/rescue/*) echo 'no rescues here' >&2; exit 1;; esac; done\n") + hook.chmod(0o755) + before = snapshot(tree) + status_before = estate.git("status", "--porcelain", cwd=tree) + proc = estate.sweep(LANE, "--yes", "--porcelain") + assert proc.returncode == 1, proc.stdout + proc.stderr + assert rows_of(proc.stdout)[str(tree)][2].count("FAILED") == 1 + assert tree.is_dir() and (tree / "w.txt").read_text() == "precious\n" + assert estate.git("status", "--porcelain", cwd=tree) == status_before + assert estate.git("symbolic-ref", "--short", "HEAD", cwd=tree) == "feat/dirty" + assert snapshot(tree) == before + assert not [b for b in estate.remote_heads() if b.startswith("rescue/")] + assert estate.notes() == [], "no register line names a removal that did not happen" + + +def test_a_live_writer_is_never_touched(estate): + tree = estate.worktree("live", "feat/live") + estate.commit(tree, "w", {"l.txt": "l\n"}) + estate.git("push", "-q", "-u", "origin", "feat/live", cwd=tree) + pane_tree = estate.worktree("pane", "feat/pane") + estate.git("push", "-q", "-u", "origin", "feat/pane", cwd=pane_tree) + panes = estate.root / "panes" + panes.write_text(f"0\tsess:1.0\t{pane_tree}\n") + sleeper = subprocess.Popen(["sleep", "300"], cwd=str(tree)) + try: + proc = estate.sweep(LANE, "--yes", "--porcelain", + env={"FAKE_TMUX_PANES": str(panes)}) + finally: + sleeper.kill() + sleeper.wait() + rows = rows_of(proc.stdout) + assert rows[str(tree)][0] == "live", rows + assert f"process {sleeper.pid}" in rows[str(tree)][2] + assert rows[str(pane_tree)][0] == "live" and "tmux pane" in rows[str(pane_tree)][2] + assert tree.is_dir() and pane_tree.is_dir() + + +def test_the_coordinator_states_its_writer_count_before_yes(estate): + """THIS session holds the lane: only its ListAgents count says which + writers are live, so `--yes` wants it (`--live ` or `--live none`).""" + tree = estate.worktree("w1", "feat/w1") + estate.git("push", "-q", "-u", "origin", "feat/w1", cwd=tree) + busy = estate.worktree("w2", "feat/w2") + estate.git("push", "-q", "-u", "origin", "feat/w2", cwd=busy) + estate.holder = "me" + estate.state_word, estate.verdict = "RUNNING", "running" + refused = estate.sweep(LANE, "--yes") + assert refused.returncode == 2 and "--live" in refused.stdout, refused.stdout + assert tree.is_dir() and busy.is_dir() + proc = estate.sweep(LANE, "--yes", "--live", busy, "--porcelain") + assert proc.returncode == 0, proc.stdout + proc.stderr + rows = rows_of(proc.stdout) + assert rows[str(busy)][0] == "live" and "--live" in rows[str(busy)][2] + assert busy.is_dir() and not tree.exists() + + +@pytest.mark.parametrize("holder,binding,word", [ + ("none", ("elsewhere", "raven/none"), "bound on raven/none"), + ("none", ("unknown", ""), "could not be read"), + ("other", ("here", "Eagle/none"), f"held by live session {OTHER}"), + ("unknown", ("here", "Eagle/none"), "could not be read"), +]) +def test_yes_is_refused_for_a_lane_live_elsewhere_or_unread(estate, holder, binding, word): + """Amendment 18(b): from outside the binding a lane is UNKNOWN, never + dead - so nothing is acted on, and the porcelain's answer is 2.""" + tree = estate.worktree("w", "feat/w") + estate.commit(tree, "w", {"w.txt": "w\n"}) + estate.holder, estate.binding = holder, binding + before = snapshot(estate.root, skip=("helper.log", "helper-spec.json", "prs.json")) + dry = estate.sweep(LANE, "--dry-run", "--porcelain") + assert dry.returncode == 2, dry.stdout + dry.stderr + assert dry.stdout.splitlines()[1].startswith("refused\t") and word in dry.stdout + yes = estate.sweep(LANE, "--yes", "--live", "none") + assert yes.returncode == 2 and word in yes.stdout + assert tree.is_dir() + assert snapshot(estate.root, skip=("helper.log", "helper-spec.json", "prs.json")) == before + assert estate.notes() == [] + + +def test_a_managed_lane_is_the_ledgers_and_is_refused(estate): + estate.worktree("w", "feat/w") + estate.extra_spec = {"lane-reconcile": {"rc": 0, "out": ( + f"MANAGED{US}ledger-1\nVERDICT{US}managed-owned{US}the ledger's\n")}} + proc = estate.sweep(LANE, "--porcelain") + assert proc.returncode == 2 and "managed ledger" in proc.stdout + + +def test_merged_is_the_register_or_gh_never_ancestry(estate, tmp_path): + """SQUASH MERGES HIDE COMPLETION BY ANCESTRY (cause 13): a squash-merged + branch is merged by its PR, and a branch whose tip IS on main but which no + PR names is NOT merged - its worktree goes, its branch stays.""" + # the register says #21 LANDED while gh still reads it OPEN (a stale read) + wip_origin = tmp_path / "wip.git" + wip = tmp_path / "wip" + estate.git("init", "-q", "--bare", "-b", "main", wip_origin, cwd=tmp_path) + estate.git("clone", "-q", wip_origin, wip, cwd=tmp_path) + (wip / "lanes" / "log").mkdir(parents=True) + (wip / "lanes" / "log" / f"{LANE}.md").write_text( + f"LANDED — lane {LANE}, session {ME}@Eagle, 2026-10-05T00:00:00Z, {SLUG}#21 → main abc1234\n") + (wip / "lanes" / "LANES.md").write_text("# register\n") + estate.git("add", "-A", cwd=wip) + estate.git("commit", "-q", "-m", "seed", cwd=wip) + estate.git("push", "-q", "origin", "main", cwd=wip) + estate.workspace = str(wip) + by_register = estate.worktree("by-register", "feat/reg") + tip = estate.commit(by_register, "r", {"r.txt": "r\n"}) + estate.git("push", "-q", "-u", "origin", "feat/reg", cwd=by_register) + estate.pr(21, "feat/reg", "OPEN", tip) + # on main by ancestry, and no PR at all + ancestor = estate.worktree("ancestor", "feat/ancestor") + estate.commit(ancestor, "a", {"a.txt": "a\n"}) + estate.git("push", "-q", "-u", "origin", "feat/ancestor", cwd=ancestor) + estate.git("merge", "-q", "--ff-only", "feat/ancestor") + estate.git("push", "-q", "origin", "main") + assert "feat/ancestor" in estate.git("branch", "--merged", "main") + proc = estate.sweep(LANE, "--porcelain") + rows = rows_of(proc.stdout) + assert rows[str(by_register)][0] == "remove+delete-branch" + assert "register LANDED" in rows[str(by_register)][2] + assert rows[str(ancestor)][0] == "remove", rows[str(ancestor)] + yes = estate.sweep(LANE, "--yes") + assert yes.returncode == 0, yes.stdout + yes.stderr + assert estate.git("branch", "--list", "feat/ancestor").strip() == "feat/ancestor" + assert estate.git("branch", "--list", "feat/reg") == "" + + +def test_without_gh_nothing_is_called_merged(estate): + tree = estate.worktree("m", "feat/m") + tip = estate.commit(tree, "m", {"m.txt": "m\n"}) + estate.git("push", "-q", "-u", "origin", "feat/m", cwd=tree) + estate.pr(5, "feat/m", "MERGED", tip) + proc = estate.sweep(LANE, "--porcelain", env={"LANES_NO_GITHUB": "1"}) + assert rows_of(proc.stdout)[str(tree)][0] == "remove" + assert "LANES_NO_GITHUB=1" in estate.sweep(LANE, env={"LANES_NO_GITHUB": "1"}).stdout + + +def test_include_foreign_takes_a_word_and_another_lanes_tree_is_never_taken(estate): + mine = estate.worktree("foreign-a", "feat/fa", where="claude", record=False) + estate.git("push", "-q", "-u", "origin", "feat/fa", cwd=mine) + theirs = estate.worktree("foreign-b", "feat/fb", where="claude", record=False) + estate.git("push", "-q", "-u", "origin", "feat/fb", cwd=theirs) + other_trees = estate.root / "lane-state" / "repoB-2" / "trees" + other_trees.mkdir(parents=True) + (other_trees / "x.yaml").write_text(f"schema: 1\npath: {theirs}\n") + conf = estate.root / "sweep.conf" + conf.write_text("foreign_quiet_hours=0\n") + env = {"LANE_WORKTREES_CONF": str(conf)} + refused = estate.sweep(LANE, "--yes", "--include-foreign", env=env) + assert refused.returncode == 2 and "--word" in refused.stdout + proc = estate.sweep(LANE, "--yes", "--include-foreign", "--word", "sweep them", "--porcelain", + env=env) + assert proc.returncode == 0, proc.stdout + proc.stderr + rows = rows_of(proc.stdout) + assert not mine.exists() + assert rows[str(theirs)][0] == "foreign" and "repoB-2" in rows[str(theirs)][2] + assert theirs.is_dir() + assert any("sweep them" in n["argv"][3] for n in estate.notes()) + + +# ================================================================ --branches + +def test_branches_deletes_the_merged_lists_the_unpublished_and_spares_an_open_pr(estate): + def branch(name, push=True, lane=LANE): + estate.git("checkout", "-q", "-b", name, "origin/main") + tip = estate.commit(estate.checkout, name, {name.replace("/", "_"): "x\n"}, lane=lane) + if push: + estate.git("push", "-q", "-u", "origin", name) + estate.git("checkout", "-q", "main") + return tip + + merged = branch("feat/b-merged") + estate.pr(31, "feat/b-merged", "MERGED", merged) + theirs = branch("feat/b-theirs", lane="repoB-2") + estate.pr(32, "feat/b-theirs", "MERGED", theirs) + opened = branch("feat/b-open", push=False) + estate.pr(33, "feat/b-open", "OPEN", opened) + unpublished = branch("feat/b-local", push=False) + estate.git("checkout", "-q", "-b", "rescue/repoA-1/old-20260101T000000Z", "origin/main") + estate.git("checkout", "-q", "main") + proc = estate.sweep(LANE, "--branches", "--porcelain") + items = items_of(proc.stdout, "branch") + assert items["feat/b-merged"][0] == "delete+remote", items + assert items["feat/b-theirs"][0] == "delete" and items["feat/b-theirs"][1] == "-" + assert items["feat/b-open"][0] == "keep" and "open" in items["feat/b-open"][3] + assert items["feat/b-local"][0] == "list" + detail = items["feat/b-local"][2] + assert unpublished[:12] in detail and "1 ahead/0 behind" in detail and "owner repoA-1" in detail + assert not any(k.startswith("rescue/") for k in items) + yes = estate.sweep(LANE, "--branches", "--yes") + assert yes.returncode == 0, yes.stdout + yes.stderr + heads = estate.remote_heads() + local = estate.git("branch", "--format=%(refname:short)").splitlines() + assert "feat/b-merged" not in local and "feat/b-merged" not in heads + assert "feat/b-theirs" not in local and "feat/b-theirs" in heads, "not ours: remote kept" + assert "feat/b-open" in local and "feat/b-local" in local + assert "rescue/repoA-1/old-20260101T000000Z" in local + assert any("branches merged by PR" in n["argv"][3] for n in estate.notes()) + + +# ========================================== scratch, caches, sandboxes, links + +def test_scratch_is_archived_then_removed_and_caches_are_removed(estate): + scratch = estate.lane_root / "x-scratch" + (scratch / "notes").mkdir(parents=True) + (scratch / "notes" / "a.md").write_text("findings\n") + (scratch / "__pycache__").mkdir() + (scratch / "__pycache__" / "z.pyc").write_bytes(b"\0") + briefs = estate.lane_root / "briefs" + briefs.mkdir(parents=True) + (briefs / "b.md").write_text("brief\n") + # a tree the sweep KEEPS (its registration is locked), with caches in it + kept = estate.worktree("kept", "feat/kept") + estate.commit(kept, "k", {"k.txt": "k\n", ".gitignore": "__pycache__/\nenv/\n"}) + estate.git("push", "-q", "-u", "origin", "feat/kept", cwd=kept) + (kept / "__pycache__").mkdir() + (kept / "__pycache__" / "y.pyc").write_bytes(b"\0") + venv = kept / "env" + (venv / "bin").mkdir(parents=True) + (venv / "pyvenv.cfg").write_text("home = /usr\n") + estate.git("worktree", "lock", kept) + # a tree a LIVE writer stands in: its caches are its writer's + busy = estate.worktree("busy", "feat/busy") + estate.git("push", "-q", "-u", "origin", "feat/busy", cwd=busy) + (busy / "__pycache__").mkdir() + sleeper = subprocess.Popen(["sleep", "300"], cwd=str(busy)) + try: + proc = estate.sweep(LANE, "--include-scratch", "--include-caches", "--yes", "--porcelain") + finally: + sleeper.kill() + sleeper.wait() + assert proc.returncode == 0, proc.stdout + proc.stderr + scratch_rows = items_of(proc.stdout, "scratch") + assert scratch_rows[str(scratch)][0] == "archive+remove" + assert not scratch.exists() and not briefs.exists() + (archive,) = estate.archives() + with tarfile.open(archive / "scratch-x-scratch.tar.gz") as tf: + names = tf.getnames() + assert "x-scratch/notes/a.md" in names and not any("__pycache__" in n for n in names) + caches = items_of(proc.stdout, "cache") + assert str(kept / "__pycache__") in caches and str(venv) in caches + assert not (kept / "__pycache__").exists() and not venv.exists() + assert (kept / "k.txt").exists(), "a cache sweep removes caches and nothing else" + assert (busy / "__pycache__").is_dir(), "a live writer's caches are left" + assert str(scratch / "__pycache__") not in caches, "it went with its scratch" + + +def test_killed_suite_sandboxes_whose_owner_is_gone_are_removed(estate): + user = os.environ.get("USER") or __import__("getpass").getuser() + base = estate.sandboxes + old = time.time() - 7200 + dead = base / "tmp.deadbeef01" + young = base / "tmp.youngone02" + held = base / "tmp.heldopen03" + for d in (dead, young, held): + d.mkdir() + (d / "f").write_text("x") + for d in (dead, held): + os.utime(d, (old, old)) + pyroot = base / f"pytest-of-{user}" + gone_run = pyroot / "pytest-7" + live_run = pyroot / "pytest-8" + for d in (gone_run, live_run): + d.mkdir(parents=True) + (gone_run / ".lock").write_text("999999\n") + (live_run / ".lock").write_text(f"{os.getpid()}\n") + sleeper = subprocess.Popen(["sleep", "300"], cwd=str(held)) + try: + proc = estate.sweep(LANE, "--include-sandboxes", "--yes", "--porcelain") + finally: + sleeper.kill() + sleeper.wait() + assert proc.returncode == 0, proc.stdout + proc.stderr + assert not dead.exists() and not gone_run.exists() + assert young.is_dir(), "younger than the minimum age" + assert held.is_dir(), "a live process stands in it" + assert live_run.is_dir(), "its .lock names a live pid" + + +def test_links_lists_every_broken_symlink_and_gitdir_pointer(estate): + estate.git("push", "-q", "origin", "main") + (estate.projects / "dangling").symlink_to(estate.projects / "nowhere") + (estate.projects / "fine").symlink_to(estate.checkout) + tree = estate.worktree("moved", "feat/moved") + shutil.move(str(estate.checkout / ".git" / "worktrees" / "moved"), + str(estate.root / "elsewhere")) + proc = estate.sweep(LANE, "--links", "--porcelain") + links = items_of(proc.stdout, "link") + assert str(estate.projects / "dangling") in links + assert "nowhere" in links[str(estate.projects / "dangling")][3] + assert str(tree / ".git") in links and "gitdir pointer" in links[str(tree / ".git")][3] + assert str(estate.projects / "fine") not in links + + +def test_a_clone_something_leans_on_is_load_bearing_and_never_removed(estate): + """CLONES AS SHARED GIT STORES (cause 14): a clone another repository + reads objects from, or names as a remote, is never removed.""" + store = estate.lane_root / "store" + estate.git("clone", "-q", GH_URL, store, cwd=estate.root) + dependent = estate.projects / "dependent" + estate.git("clone", "-q", "--shared", store, dependent, cwd=estate.root) + assert (dependent / ".git" / "objects" / "info" / "alternates").exists() + conf = estate.root / "sweep.conf" + conf.write_text("foreign_quiet_hours=0\n") + proc = estate.sweep(LANE, "--include-foreign", "--word", "go", "--yes", "--porcelain", + env={"LANE_WORKTREES_CONF": str(conf)}) + rows = rows_of(proc.stdout) + assert rows[str(store)][0] == "load-bearing", rows + assert str(dependent) in rows[str(store)][2] and "repack" in rows[str(store)][2] + assert store.is_dir() + foreign = estate.sweep(LANE, "--porcelain") + assert "LOAD-BEARING" in rows_of(foreign.stdout)[str(store)][2] + + +# =================================================================== --expire + +def _archive(estate, days: int, branch: str | None) -> Path: + when = _dt.datetime.now(_dt.timezone.utc) - _dt.timedelta(days=days, hours=1) + d = estate.state / "openRepoTools" / "sweeps" / LANE / when.strftime("%Y%m%dT%H%M%SZ") + d.mkdir(parents=True) + (d / "DISPOSITION.md").write_text("# x\n") + rows = "# origin\tbranch\tsha\tcheckout\ttree\n" + if branch: + rows += f"{GH_URL}\t{branch}\t{'0' * 40}\t-\t-\n" + (d / "rescues.tsv").write_text(rows) + return d + + +def test_expire_keeps_young_archives_and_any_whose_rescue_left_origin(estate): + estate.git("push", "-q", "origin", "main:refs/heads/rescue/repoA-1/kept-1") + young = _archive(estate, 89, "rescue/repoA-1/kept-1") + due = _archive(estate, 90, "rescue/repoA-1/kept-1") + orphan = _archive(estate, 120, "rescue/repoA-1/gone-1") + plain = _archive(estate, 91, None) + dry = estate.sweep("--expire", "--porcelain") + assert dry.returncode == 3, dry.stdout + dry.stderr + rows = {line.split("\t")[2]: line.split("\t")[1] for line in dry.stdout.splitlines()} + assert rows[str(young)] == "keep" and rows[str(due)] == "expire" + assert rows[str(orphan)] == "keep" and rows[str(plain)] == "expire" + assert young.is_dir() and due.is_dir() + yes = estate.sweep("--expire", "--yes") + assert yes.returncode == 0, yes.stdout + yes.stderr + assert young.is_dir() and orphan.is_dir() + assert not due.exists() and not plain.exists() + log = (estate.state / "openRepoTools" / "sweeps" / "EXPIRED.log").read_text() + assert str(due) in log and str(orphan) not in log + conf = estate.root / "sweep.conf" + conf.write_text("retention_days=200\n") + later = estate.sweep("--expire", "--porcelain", env={"LANE_WORKTREES_CONF": str(conf)}) + assert later.returncode == 0 and "\texpire\t" not in later.stdout + + +# ======================================================= the porcelain contract + +def test_the_porcelain_exit_codes_lane_end_reads(estate): + """#163's gate: 0 nothing to retire, 3 something to retire, 2 refused.""" + nothing = estate.sweep(LANE, "--dry-run", "--porcelain") + assert nothing.returncode == 0, nothing.stdout + nothing.stderr + last = nothing.stdout.splitlines()[-1].split("\t") + assert last[:2] == ["summary", "0"] + estate.worktree("w", "feat/w") + something = estate.sweep(LANE, "--dry-run", "--porcelain") + assert something.returncode == 3 + assert something.stdout.splitlines()[0].split("\t")[:2] == ["lane", LANE] + assert something.stdout.splitlines()[-1].split("\t")[:2] == ["summary", "1"] + estate.extra_spec = {"lane-reconcile": {"rc": 1, "err": "the log could not be read\n"}} + refused = estate.sweep(LANE, "--dry-run", "--porcelain") + assert refused.returncode == 2 and "refused\t" in refused.stdout + + +def test_usage_errors_are_64(estate): + for argv in (["--bogus"], [], ["--word", "x"]): + proc = estate.sweep(*argv) if argv != [] else estate.sweep() + assert proc.returncode == 64, (argv, proc.stdout, proc.stderr) + proc = subprocess.run([sys.executable, str(LW), "--help"], capture_output=True, text=True) + assert proc.returncode == 0 and "sweep " in proc.stdout + + +# ========================================== the real helper, end to end (#97) + +UUID_R = "33333333-3333-4333-8333-333333333333" + + +def _real_estate(tmp_path: Path, host: str) -> tuple: + """The fake estate, with the REAL `lanes-edit.sh` and a workspace.""" + e = Estate(tmp_path) + wip_origin = tmp_path / "wip.git" + wip = e.projects / "wip" + e.git("init", "-q", "--bare", "-b", "main", wip_origin, cwd=tmp_path) + e.git("clone", "-q", wip_origin, wip, cwd=tmp_path) + (wip / "lanes" / "log").mkdir(parents=True) + (wip / "lanes" / "LANES.md").write_text("\n".join([ + "# LANES.md", "", + "| lane | session id | workstation / env / user | started (UTC) | objects owned | handoff path | state |", + "|---|---|---|---|---|---|---|", + f"| `{LANE}` | harness `{UUID_R}` | Eagle / test / brett | 2026-10-05T00:00Z | none | handoffs/{LANE}.md | LIVE · 2026-10-05T00:00:00Z · working |", + ""])) + (wip / "lanes" / "log" / f"{LANE}.md").write_text( + f"STARTED — lane {LANE}, session {UUID_R}@Eagle, 2026-10-05T00:00:00Z, " + f"lane:{LANE} → home {SLUG}; dir {e.checkout}; host {host}; container none; os linux\n") + e.git("add", "-A", cwd=wip) + e.git("commit", "-q", "-m", "seed", cwd=wip) + e.git("push", "-q", "origin", "main", cwd=wip) + agents = e.home / ".agents" + agents.mkdir() + (agents / "workspace.yaml").write_text(f"repository: {wip_origin}\npath: {wip}\n") + e.env.update(AGENT_PROTOCOL_ROOT=str(agents), LANES_EDIT=str(LANES_EDIT), + LANES_HOST="eagle", LANES_OS="linux", LANES_CONTAINER="none", + LANES_IN_CONTAINER="0", LANES_NO_GITHUB="1", LANES_INDEX="off", + LANES_LANE_STATE_ROOT=str(tmp_path / "lane-state"), + CLAUDE_CONFIG_DIR=str(e.home / ".claude"), LANES_SESSION=UUID_R) + e.env.pop("CLAUDE_CODE_SESSION_ID", None) + return e, wip + + +def _real_tree(e: Estate, name: str) -> Path: + path = e.worktree(name, f"feat/{name}", record=False) + e.commit(path, name, {f"{name}.txt": "x\n"}) + e.git("push", "-q", "-u", "origin", f"feat/{name}", cwd=path) + proc = subprocess.run(["bash", str(LANES_EDIT), "set-lane-tree", LANE, str(path), + "--checkout", str(e.checkout)], capture_output=True, text=True, + env=e.env, check=False) + assert proc.returncode == 0, proc.stderr + return path + + +def test_the_real_helper_inventory_and_register_line(tmp_path): + """The inventory is #97's own (`set-lane-tree`), the reconciliation is + the helper's own, and the register line lands in the lane's own log.""" + e, wip = _real_estate(tmp_path, host="eagle") + mine = _real_tree(e, "mine") + foreign = e.worktree("other", "feat/other", where="claude", record=False) + dry = e.sweep(LANE, "--porcelain") + rows = rows_of(dry.stdout) + assert rows[str(mine)][0] == "remove", dry.stdout + dry.stderr + assert rows[str(foreign)][0] == "foreign" + yes = e.sweep(LANE, "--yes") + assert yes.returncode == 0, yes.stdout + yes.stderr + assert not mine.exists() and foreign.is_dir() + e.git("pull", "-q", "origin", "main", cwd=wip) + log = (wip / "lanes" / "log" / f"{LANE}.md").read_text() + noted = [ln for ln in log.splitlines() if ln.startswith("NOTED")] + assert len(noted) == 1 and str(mine) in noted[0] and "remove" in noted[0], log + + +def test_the_real_helper_refuses_a_lane_bound_on_another_host(tmp_path): + e, _wip = _real_estate(tmp_path, host="raven") + mine = _real_tree(e, "mine") + dry = e.sweep(LANE, "--porcelain") + assert dry.returncode == 2, dry.stdout + dry.stderr + yes = e.sweep(LANE, "--yes", "--live", "none") + assert yes.returncode == 2 and "raven" in yes.stdout + assert mine.is_dir() From 7d130ea00f782c90076aaa24149d0454e7e002a6 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 20:24:12 +0000 Subject: [PATCH 03/29] Place lane-worktrees as the seventeenth installed file `openRepoTools --install` places `lane-worktrees` at the end of INSTALLABLES, so every index a test takes still names the file it did. The file count moves 16 -> 17 and the artifact count 30 -> 31 wherever they are stated, with the count history extended rather than restated; the receipt sentence now says 31 artifacts, 29 rows. --help names the new command. The hygiene suite compiles `lane-worktrees` beside `lanes-index`: the two shipped commands that are Python and have no .py suffix for the glob to find. Part of #162. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- README.md | 16 +++--- openRepoTools | 78 ++++++++++++++++------------ tests/test_install_skill_and_hook.py | 28 +++++----- tests/test_openrepotools_command.py | 28 +++++----- tests/test_repo_hygiene.py | 14 ++--- 6 files changed, 93 insertions(+), 73 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 37cd1ea..89fad47 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,7 +3,7 @@ Three estate commands, `park`, `resume` and `status`; the lane tooling `lanes-edit.sh`, `lane-start`, `lane-end` and `link-estates`, which came here with their history under lane-collision-protocol Amendment 9; and the -`openRepoTools` that places all sixteen files and creates the workspace they read. +`openRepoTools` that places all seventeen files and creates the workspace they read. **The verbs add no mechanics.** They find the estate and run its own `make park` / `make resume`, which run the Speckit git extension's scripts — one implementation, ruled 2026-09-09 diff --git a/README.md b/README.md index be1d5a0..3cce271 100644 --- a/README.md +++ b/README.md @@ -237,14 +237,14 @@ gh api repos/opensoft/openRepoTools/contents/openRepoTools \ -H 'Accept: application/vnd.github.raw' | bash -s -- --install ``` -It places SIXTEEN files into `~/.local/bin` — `openRepoTools`, `park`, `resume`, +It places SEVENTEEN files into `~/.local/bin` — `openRepoTools`, `park`, `resume`, `status`, `lane`, `lanes`, `lane-handoff`, `lane-rename`, `lanes-edit.sh`, `lane-start`, `lane-end`, `link-estates`, the alias table `repos.tsv`, -`claude-current`, `claude-restart-check` and `lanes-index` — 755, idempotently: a +`claude-current`, `claude-restart-check`, `lanes-index` and `lane-worktrees` — 755, idempotently: a second run prints `already installed … (unchanged)` per file, one whose bytes have drifted prints `updated at`, and one whose bytes were right and whose MODE was not prints `(mode restored to 755)`: the mode is stamped on every artifact on every -run, whether or not the bytes moved. ALL SIXTEEN ARE IN HAND BEFORE ANY IS +run, whether or not the bytes moved. ALL SEVENTEEN ARE IN HAND BEFORE ANY IS PLACED, so a fetch that failed replaces nothing and names the file it could not get. A mode stamp that FAILS is a refusal naming the file it could not mode (exit 2), never the shell's own 1: every `chmod` this command performs goes @@ -259,7 +259,7 @@ THAT EXISTS, so a bin directory that is not there yet is refused for the parent that would not take it, and a create needs a directory's search bit as well as its write bit. A refusal creates none of those directories either. `cp` follows a symlink, and an install through one leaves the command uninstalled and writes -these bytes into whatever it points at. Then a `16 of 16 placed in ` line, +these bytes into whatever it points at. Then a `17 of 17 placed in ` line, and the `export PATH=…` line if that directory is not on your `PATH`. It also places **fourteen things that are not files in that directory**: THREE @@ -288,7 +288,7 @@ changes no byte of them. An entry that runs string — a second writer of one of these hooks — a file it cannot parse, or a `hooks` that is not an object → it **refuses, prints the exact block, and places nothing at all**, because both merges are computed -with the sixteen files in hand before either is placed. An installer that +with the seventeen files in hand before either is placed. An installer that repairs a file it does not understand is how you lose a setting you meant. A SECOND WRITER REFUSES WHETHER OR NOT OUR OWN ENTRY IS BESIDE IT: the read asks for a rival BEFORE it asks whether ours is already there, so a file carrying @@ -299,7 +299,7 @@ own `UserPromptSubmit` hooks are left exactly where they are, which is why that arm keys on the VERB and not on the word anywhere in a path. It never writes a profile's own `settings.json`: the launcher owns that one. -Thirty artifacts, and the count is the invariant. It was sixteen until A11 +Thirty-one artifacts, and the count is the invariant. It was sixteen until A11 Addendum 4 ruling 9 gave `--install` a command-file list and `commands/swap.md` in it, at the same pair of paths a skill takes — because `opensoft/workBenches#74` deletes the launcher's copy and `/swap` would otherwise be installed by nobody; @@ -313,7 +313,7 @@ respawned with a new session whose first prompt is that handoff's top block. Twenty-six until **Amendment 16** (ratified the same day) put `lane-rename` on `PATH`: a lane is renamed by one word, in one commit — the row, the object log, the handoff and `lanes/aliases.tsv` — and its old name resolves for ever -afterwards, in every reader that takes a lane name. Twenty-seven, with THIRTEEN files and a `13 of 13 placed` line, until opensoft/workBenches#119 put `claude-current` and `claude-restart-check` on `PATH` (Brett Heap, 2026-09-29, verbatim *"this work is really for openRepoTools repo"*): the Claude Code a launch starts, and the RESTART NEEDED line the status line prints. Twenty-nine, with FIFTEEN files and a `15 of 15 placed` line, until opensoft/openRepoTools#160 put `lanes-index` on `PATH` (lane-collision-protocol Amendment 14, Brett Heap, 2026-10-05, verbatim *"ratify 48"*): the derived index of the lane register, which nothing that acts reads. +afterwards, in every reader that takes a lane name. Twenty-seven, with THIRTEEN files and a `13 of 13 placed` line, until opensoft/workBenches#119 put `claude-current` and `claude-restart-check` on `PATH` (Brett Heap, 2026-09-29, verbatim *"this work is really for openRepoTools repo"*): the Claude Code a launch starts, and the RESTART NEEDED line the status line prints. Twenty-nine, with FIFTEEN files and a `15 of 15 placed` line, until opensoft/openRepoTools#160 put `lanes-index` on `PATH` (lane-collision-protocol Amendment 14, Brett Heap, 2026-10-05, verbatim *"ratify 48"*): the derived index of the lane register, which nothing that acts reads. Thirty, with SIXTEEN files and a `16 of 16 placed` line, until opensoft/openRepoTools#162 put `lane-worktrees` on `PATH` (Brett Heap, 2026-10-05, verbatim *"build 162"*): the sweep that retires a lane's abandoned worktrees, rescue first. **And it writes down what it placed.** A RECEIPT (#57) — one row of ` ` per regular file placed, in @@ -335,7 +335,7 @@ where `raw.githubusercontent.com` is blocked. |---|---|---| | `$OPENREPOTOOLS_REPO` | `opensoft/openRepoTools` | the `owner/name` to fetch from — a fork or a mirror, named once | | `$OPENREPOTOOLS_REF` | `main` | the ref to fetch it at | -| `$OPENREPOTOOLS_BIN_DIR` | `~/.local/bin` | where `--install` puts the sixteen, and where `lane-start` looks for `claude-current` | +| `$OPENREPOTOOLS_BIN_DIR` | `~/.local/bin` | where `--install` puts the seventeen, and where `lane-start` looks for `claude-current` | | `$OPENREPOTOOLS_DATA_DIR` | `${XDG_DATA_HOME:-~/.local/share}/openRepoTools` | where `--install` writes the receipt of what it placed | | `$AGENT_PROTOCOL_ROOT` | `~/.agents` | where `workspace.yaml` lives — the one pointer to your data | | `$CLAUDE_PROFILES_HOME` | `~/.claude-profiles` | the profiles root `--install` places the shared skills under | diff --git a/openRepoTools b/openRepoTools index 3cb5293..052fc27 100755 --- a/openRepoTools +++ b/openRepoTools @@ -25,7 +25,7 @@ # change (Amendment 9(b), and R-A9-1). The identity that survives is narrower # and truer. # -# `--install` PLACES SIXTEEN FILES, not four: this file, `park`, `resume`, +# `--install` PLACES SEVENTEEN FILES, not four: this file, `park`, `resume`, # `status` (openRepoShape #82; the fourth under Brett Heap's RULING of # 2026-09-10 in this repository, "lets go with a fourth file"), `lane` and # `lanes` (lane-collision-protocol Amendment 18 Addendum 1 and Amendment 11's @@ -41,15 +41,18 @@ # goes — `claude-current` and `claude-restart-check` # (opensoft/workBenches#119, Brett Heap's home ruling of 2026-09-29: the Claude # Code a launch starts, and the RESTART NEEDED line a running session shows) — -# and `lanes-index` (opensoft/openRepoTools#160, lane-collision-protocol +# `lanes-index` (opensoft/openRepoTools#160, lane-collision-protocol # Amendment 14, ratified by Brett Heap 2026-10-05: the DERIVED INDEX of the lane -# register, which nothing that acts reads). It also places THREE skills, +# register, which nothing that acts reads) — and `lane-worktrees` +# (opensoft/openRepoTools#162, on Brett Heap's word of 2026-10-05, "build 162": +# the sweep that retires a lane's abandoned worktrees, rescue first). It also +# places THREE skills, # `/handoff`, `/lane-swap` and `/restart`, at two paths each, THREE command # files, `/handoff`, `/ctx` and `/swap`, at that same pair of paths (A11 # Addendum 4 ruling 9; Amendment 17(a) and (f)), and merges TWO hook entries into # `~/.claude/settings.json` — the `SessionStart` one (Amendment 9(b), A8 # Addendum 2 R-A8-5) and, since lane-collision-protocol Amendment 12 adoption -# act 3, the `UserPromptSubmit` NAME GUARD beside it: THIRTY +# act 3, the `UserPromptSubmit` NAME GUARD beside it: THIRTY-ONE # artifacts, and the count is the invariant. # # One install line for the whole toolset, so a second engineer gets the estate @@ -77,7 +80,7 @@ die() { printf '\nREFUSED: %s\n' "$1" >&2; exit "${2:-2}"; } usage() { cat <<'USAGE' -openRepoTools --install install (or update) the sixteen estate and +openRepoTools --install install (or update) the seventeen estate and lane commands into ~/.local/bin, plus the /handoff, /lane-swap and /restart skills, the /handoff, /ctx and /swap command files, the @@ -88,10 +91,11 @@ openRepoTools wip init create your workspace repository, clone it, openRepoTools --help | --version `openRepoShape` is the standard's front door and scaffolds product -repositories; this command scaffolds none. `--install` places sixteen files +repositories; this command scaffolds none. `--install` places seventeen files beside each other — this command, the two verbs, the read-only view, the four lane words, the four lane helpers, the alias table they read, the two Claude -Code launch commands and the derived index of the lane register: +Code launch commands, the derived index of the lane register and the worktree +sweep: park [] commit, push and RECORD an estate's open features, so another workstation can take @@ -143,6 +147,10 @@ Code launch commands and the derived index of the lane register: (Amendment 14): written behind the sources, read only by lanes --index and history --index + lane-worktrees sweep retire a lane's abandoned worktrees: a dry + run by default; --yes pushes or rescues + first and removes only what is on origin + or in a bundle, never a live writer's `--install` also places THREE SKILLS at two paths each — `~/.claude-profiles/shared/skills//SKILL.md` and @@ -254,7 +262,7 @@ fetch_from_repo() { return 1 } -# THE SIXTEEN FILES `--install` PLACES. One list, so the installer, the fetch +# THE SEVENTEEN FILES `--install` PLACES. One list, so the installer, the fetch # fallback and the tests cannot disagree about what a complete install is. # `openRepoTools` is first because it is the one a person types to get the # others. @@ -310,7 +318,12 @@ fetch_from_repo() { # `lanes-edit.sh history --index` read; nothing that acts reads it. It is the one # command in the list that is Python rather than bash, and it is placed at the # END like the two before it, for the same reason. -INSTALLABLES=(openRepoTools park resume status lane lanes lane-handoff lane-rename lanes-edit.sh lane-start lane-end link-estates repos.tsv claude-current claude-restart-check lanes-index) +# SEVENTEEN SINCE opensoft/openRepoTools#162 (Brett Heap's word of 2026-10-05, +# "build 162"): `lane-worktrees sweep ` retires a lane's abandoned +# worktrees, branches, scratch and killed-suite sandboxes - a dry run unless +# `--yes`, and nothing removed that is not first on origin or in a bundle. It is +# Python like `lanes-index`, and at the END for the same reason. +INSTALLABLES=(openRepoTools park resume status lane lanes lane-handoff lane-rename lanes-edit.sh lane-start lane-end link-estates repos.tsv claude-current claude-restart-check lanes-index lane-worktrees) # ------------------------------------------ THE RECEIPT OF WHAT IT PLACED # @@ -365,9 +378,9 @@ INSTALLABLES=(openRepoTools park resume status lane lanes lane-handoff lane-rena # it went stale the next time a list moved (Copilot on #103, `openRepoTools:325`, # which found twenty-six where there were twenty-seven): `ARTIFACTS - HOOK_ENTRIES` # in `tests/test_openrepotools_command.py`, which is `len(INSTALLED) + 2 * -# len(SKILL_NAMES) + 2 * len(COMMAND_NAMES)`. Today that is 30 artifacts, 28 of -# them files, 28 rows — 16 + 6 + 6, since opensoft/openRepoTools#160 put -# `lanes-index` in `INSTALLABLES` — and it moves with `INSTALLABLES`, `SKILLS` +# len(SKILL_NAMES) + 2 * len(COMMAND_NAMES)`. Today that is 31 artifacts, 29 of +# them files, 29 rows — 17 + 6 + 6, since opensoft/openRepoTools#162 put +# `lane-worktrees` in `INSTALLABLES` — and it moves with `INSTALLABLES`, `SKILLS` # and `COMMANDS`, and with nothing else. # # A RECEIPT THAT CANNOT BE WRITTEN IS A NOTE AND NEVER A REFUSAL (#57's own @@ -982,7 +995,7 @@ mode_is() { # install_commands() { local dir="${OPENREPOTOOLS_BIN_DIR:-$HOME/.local/bin}" local name target source verb placed=0 - # ALL THIRTY IN HAND BEFORE ANY IS PLACED. The sixteen files, the + # ALL THIRTY-ONE IN HAND BEFORE ANY IS PLACED. The seventeen files, the # three skills, the three command files, and the TWO artifacts that are not # whole files — the two hook entries merged into one `settings.json` — # are all computed here, so a fetch that failed replaces nothing and a merge @@ -1081,7 +1094,7 @@ install_commands() { # ========================================================================= -# THE FOURTEEN ARTIFACTS THAT ARE NOT ONE OF THE SIXTEEN FILES +# THE FOURTEEN ARTIFACTS THAT ARE NOT ONE OF THE SEVENTEEN FILES # ========================================================================= # # lane-collision-protocol Amendment 9(b), inheriting A8 Addendum 2's R-A8-5 @@ -1099,7 +1112,7 @@ install_commands() { # run and preserves hook keys. This command never writes a profile's # settings.json at all. One path, one writer, in both halves. # -# THIRTY ARTIFACTS, AND THE COUNT IS THE INVARIANT: sixteen files in the bin +# THIRTY-ONE ARTIFACTS, AND THE COUNT IS THE INVARIANT: seventeen files in the bin # directory, three skills in the shared skills directory, their three bare-run # copies, THREE COMMAND FILES at those same two paths, and TWO merged entries in # `~/.claude/settings.json`. It was twelve before Amendment 11 added a SECOND @@ -1110,8 +1123,9 @@ install_commands() { # before Amendment 17 added `lane-handoff`, the `handoff` skill and the # `/handoff` and `/ctx` command files, twenty-six before Amendment 16 added # `lane-rename`, twenty-seven before opensoft/workBenches#119 added -# `claude-current` and `claude-restart-check`, and twenty-nine before -# opensoft/openRepoTools#160 added `lanes-index`; the count moves +# `claude-current` and `claude-restart-check`, twenty-nine before +# opensoft/openRepoTools#160 added `lanes-index`, and thirty before +# opensoft/openRepoTools#162 added `lane-worktrees`; the count moves # with the lists rather than being restated from memory. # # WHY A COMMAND FILE IS HERE AT ALL, WHEN ADOPTION ACT 6 SAYS THE LAUNCHER @@ -1211,7 +1225,7 @@ BLOCK # EVERY SKILL'S BYTES, by `--install`'s own rule: from the checkout beside this # file when run from one, else fetched at this same ref. ALL OF THEM IN HAND -# BEFORE ANY IS PLACED, which is the same all-or-nothing rule the sixteen files +# BEFORE ANY IS PLACED, which is the same all-or-nothing rule the seventeen files # already have — a new `/restart` beside a `handoff` that could not be fetched # is a half-install that reads like a whole one. collect_skills() { @@ -1277,8 +1291,8 @@ HOOK_PLAN="" # "absent" | "present" | "nofile" GUARD_PLAN="" # the same three, for Amendment 12's UserPromptSubmit entry HOOK_MERGED="" # a staged whole file, ready to move into place -# THE DESTINATIONS, PROVED WRITABLE BEFORE ANY OF THE SIXTEEN IS PLACED — every -# directory the artifacts that are not one of the sixteen files go into, derived +# THE DESTINATIONS, PROVED WRITABLE BEFORE ANY OF THE SEVENTEEN IS PLACED — every +# directory the artifacts that are not one of the seventeen files go into, derived # from the two lists and NOT counted in this sentence. It said "THE THREE # DESTINATIONS" while the loop below built seven of them, which is the same # defect `collect_commands` names above: a number restated beside a list that @@ -1321,7 +1335,7 @@ plan_skill_targets() { # `~/.claude/skills/restart/SKILL.md` was followed and WRITTEN THROUGH: the # skill stays uninstalled, the target stays a link, and the bytes land # wherever it points. That is the same failure R-A9-12 was ruled on for the - # sixteen in the bin directory, one directory along, and the ruling says + # seventeen in the bin directory, one directory along, and the ruling says # nothing about which path it applies to. # # `unplaceable_kind` is asked rather than re-tested, because two copies of @@ -1361,7 +1375,7 @@ $bad rm -f -- $paths" } -# THE SIXTEEN TARGETS, PROVED PLACEABLE BEFORE ANY OF THE THIRTY IS PLACED +# THE SEVENTEEN TARGETS, PROVED PLACEABLE BEFORE ANY OF THE THIRTY-ONE IS PLACED # (A9 Addendum 4, R-A9-12, ruled on F5 of the #24 review). # # `cp` FOLLOWS A SYMLINK. The loop below tests `[ -e "$target" ]`, compares @@ -1386,7 +1400,7 @@ $bad # `rm` that clears them — rather than leaving a person to remember a step. # WHAT IS AT A PATH, WHERE IT IS NOT A REGULAR FILE. # ONE implementation of R-A9-12's test, for every path this command writes: the -# sixteen in the bin directory, the six `SKILL.md`s (F-X17), the six command +# seventeen in the bin directory, the six `SKILL.md`s (F-X17), the six command # files, and `~/.claude/settings.json` (#44 round 1, `openRepoTools:878`). # Prints what is there and returns 0; returns 1 — saying nothing — where the # path is absent or is a regular file. @@ -1421,9 +1435,9 @@ path_kind() { # # This answered "a regular file" with `return 1` and asked nothing else, so # every existing file counted as one `--install` may place over — and # `plan_skill_targets` let a READ-ONLY `SKILL.md` or command file through. -# `install_commands` has already copied the sixteen PATH files by the time +# `install_commands` has already copied the seventeen PATH files by the time # `place_skill_and_hook` reaches the `cp`, so an ordinary permission failure -# there left a host with sixteen of the thirty artifacts placed, on the one +# there left a host with seventeen of the thirty-one artifacts placed, on the one # path in this file that promises all or none. A mode is not an exotic state # either: `chmod 444` on a file a person did not want overwritten is what a # person does. @@ -1596,7 +1610,7 @@ $(guard_block_text)" # WHAT IS AT THAT PATH, BEFORE A BYTE OF IT IS READ (#44 round 1, # `openRepoTools:878`). This file carries the LAST TWO artifacts — the - # twenty-ninth and thirtieth — since Amendment 12 adoption act 3 put the + # thirtieth and thirty-first — since Amendment 12 adoption act 3 put the # name guard beside the `SessionStart` entry, and it is the only path no # walk asked about; the # `present` arm now CHMODs it, and `chmod` @@ -1796,7 +1810,7 @@ $(guard_block_text)" } # The fourteen artifacts that are not bin-directory files, in thirteen writes -# (the two hook entries go into settings.json in one), AFTER the sixteen files +# (the two hook entries go into settings.json in one), AFTER the seventeen files # and the merge are all in hand. place_skill_and_hook() { local shared_dir bare_dir settings target verb hook_tmp name @@ -1808,7 +1822,7 @@ place_skill_and_hook() { # EVERY FAILURE HERE IS A `die`, never the shell's own 1. `--install`'s # codes are this toolset's — 0 done, 1 findings printed, 2 a refusal — and # an unguarded `mkdir` or `cp` under `set -e` exits 1, which reads as a - # report of findings from a command that placed sixteen of thirty artifacts. + # report of findings from a command that placed seventeen of thirty-one artifacts. for target in "$shared_dir/SKILL.md" "$bare_dir/SKILL.md"; do mkdir -p -- "$(dirname -- "$target")" || die "could not create $(dirname -- "$target") for the $name skill. @@ -1833,7 +1847,7 @@ place_skill_and_hook() { the skills and the hook are not." fi # 644: a SKILL.md is a document a session READS, not a command. The - # sixteen in the bin directory are stamped 755 for the reason that loop + # seventeen in the bin directory are stamped 755 for the reason that loop # gives — "a copy that is not executable is not a command" — and # that reason does not reach this file. Amendment 9(b) describes # `--install` as stamping 755 "on everything it places" while @@ -1855,7 +1869,7 @@ place_skill_and_hook() { # reason a skill has two. # # 644 for the same reason a `SKILL.md` is: a command file is a document a - # session READS. The 755 argument belongs to the sixteen in the bin + # session READS. The 755 argument belongs to the seventeen in the bin # directory, where a copy that is not executable is not a command. for name in "${COMMANDS[@]}"; do for target in "$(skills_home)/shared/commands/$name.md" "$(claude_home)/commands/$name.md"; do @@ -1899,8 +1913,8 @@ place_skill_and_hook() { # round 1, the suppressed comment at `openRepoTools:880`): a # `settings.json` this euid can write but does not own takes `jq` and # every check `plan_hook_merge` puts to it, then refuses the mode - # stamp, with the sixteen commands, the skills and the command files — - # twenty-eight of the thirty artifacts — already placed. The guard that + # stamp, with the seventeen commands, the skills and the command files — + # twenty-nine of the thirty-one artifacts — already placed. The guard that # would catch it first is the same `[ -O ]` `unplaceable_kind`'s own # comment declines, for the same reason: no test in this suite can # make a second account to own this file, and the check would refuse a diff --git a/tests/test_install_skill_and_hook.py b/tests/test_install_skill_and_hook.py index 99cfd5b..fa366bc 100644 --- a/tests/test_install_skill_and_hook.py +++ b/tests/test_install_skill_and_hook.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -"""`--install`'s artifacts that are not one of the SIXTEEN files: the three +"""`--install`'s artifacts that are not one of the SEVENTEEN files: the three skills at two paths each, the three command files at two more each, and the TWO merged hook entries. @@ -17,7 +17,7 @@ on the command string. So the tests below are about the four answers that string can have — present, absent, differing, unreadable — asked of EACH entry, and about the one rule that makes a wrong answer survivable: both merges are -computed with the sixteen files in hand, BEFORE any of them is placed, so a +computed with the seventeen files in hand, BEFORE any of them is placed, so a refusal costs a whole install rather than half of one — and, since the pair, a settings file never passes through a state carrying one entry of the two. @@ -550,7 +550,7 @@ def test_a_differing_session_start_entry_refuses_and_places_nothing(tmp_path): twice. AND THE COST IS A WHOLE INSTALL, NOT HALF OF ONE: both merges are computed - with the sixteen files in hand, before any of them is placed, so the bin + with the seventeen files in hand, before any of them is placed, so the bin directory is untouched. That is the same all-or-nothing rule `--install` already had, extended to the two artifacts that are not whole files. """ @@ -663,16 +663,16 @@ def test_a_destination_that_cannot_be_written_refuses_before_anything_is_placed( Amendment 9(b) computes the merge in hand "so a merge that cannot be computed refuses having placed nothing". A filesystem offers no transaction - across thirty artifacts, so nothing can make the last fourteen atomic - with the first sixteen — but the failure that actually happens is not an + across thirty-one artifacts, so nothing can make the last fourteen atomic + with the first seventeen — but the failure that actually happens is not an exotic one, it is a directory that is not this installer's to write, and that question can be asked in the planning phase where the refusal still costs nothing. - Without the check the run places sixteen files and some of the remaining + Without the check the run places seventeen files and some of the remaining artifacts, then dies — leaving a host with commands installed, neither hook entry, and an installer that reports the same "already - installed (unchanged)" for the sixteen on every re-run while never reaching + installed (unchanged)" for the seventeen on every re-run while never reaching the one that failed. """ if which == "shared skills": @@ -687,13 +687,13 @@ def test_a_destination_that_cannot_be_written_refuses_before_anything_is_placed( assert result.returncode == 2, result.stdout + result.stderr assert "NOTHING was installed" in result.stderr assert not bin_dir.exists() or not any(bin_dir.iterdir()), ( - "the sixteen files were placed against a destination that was never " + "the seventeen files were placed against a destination that was never " "going to take the other fourteen") finally: blocked.chmod(0o700) -# --- the sixteen targets, and what they are (R-A9-12) ---------------------- +# --- the seventeen targets, and what they are (R-A9-12) ---------------------- @NEEDS_JQ @NOT_ROOT @@ -714,7 +714,7 @@ def test_a_leaf_destination_that_exists_unwritable_refuses_before_anything_is_pl The bin directory is the assertion that tells the two apart: the refusal here is a PLANNING one and nothing is placed, where the mutant places all - sixteen files and dies on the `cp` into this same directory, which is the + seventeen files and dies on the `cp` into this same directory, which is the half-install the planning phase exists to prevent. """ blocked = tmp_path / ".claude-profiles" / "shared" / "skills" / SKILL_DIR_NAME @@ -727,7 +727,7 @@ def test_a_leaf_destination_that_exists_unwritable_refuses_before_anything_is_pl assert "is not writable" in result.stderr, result.stderr assert "NOTHING was installed" in result.stderr assert not bin_dir.exists() or not any(bin_dir.iterdir()), ( - "the sixteen files were placed against a leaf directory that was " + "the seventeen files were placed against a leaf directory that was " "never going to take the skill") finally: blocked.chmod(0o700) @@ -796,7 +796,7 @@ def test_a_symlinked_target_is_refused_and_nothing_is_written_through_it(tmp_pat def test_a_directory_where_a_command_goes_is_refused_the_same_way(tmp_path): """THE RULE IS `A REGULAR FILE`, not `not a symlink`. A directory at `$BIN/park` is the same refusal for the same reason — `cp` cannot place a - file over it, and finding that out after fifteen of the sixteen are placed is + file over it, and finding that out after sixteen of the seventeen are placed is the half-install the planning phase exists to prevent.""" bin_dir = tmp_path / ".local" / "bin" (bin_dir / "park").mkdir(parents=True) @@ -1057,7 +1057,7 @@ def test_a_symlinked_skill_target_is_refused_and_nothing_written_through_it( tmp_path, shared, name): """R-A9-12 IS ABOUT WHAT `cp` DOES, NOT ABOUT WHICH DIRECTORY (F-X17). - `plan_install_targets` refuses a symlink for every one of the sixteen files + `plan_install_targets` refuses a symlink for every one of the seventeen files in the bin directory. `plan_skill_targets` proved only the DIRECTORIES writable, and `place_skill_and_hook` then reached each `SKILL.md` with `[ -e ]`, `cmp -s` and `cp` — none of which can tell a regular file from a @@ -1091,7 +1091,7 @@ def test_a_symlinked_skill_target_is_refused_and_nothing_written_through_it( f"the refusal must print the exact `rm` that clears it:\n{result.stderr}") assert far.read_bytes() == before, f"--install wrote through the link into {far}" assert target.is_symlink(), f"{target} is no longer the link it was" - # AND IT REFUSED IN THE PLANNING PHASE: the sixteen commands never arrived + # AND IT REFUSED IN THE PLANNING PHASE: the seventeen commands never arrived # either, which is what makes `NOTHING was installed` true rather than # nearly true. assert not bin_dir.exists() or not any(bin_dir.iterdir()), ( diff --git a/tests/test_openrepotools_command.py b/tests/test_openrepotools_command.py index 640c6c3..380836a 100644 --- a/tests/test_openrepotools_command.py +++ b/tests/test_openrepotools_command.py @@ -65,11 +65,14 @@ #: FIFTEEN. `lanes-index` joined it for opensoft/openRepoTools#160 #: (lane-collision-protocol Amendment 14, ratified 2026-10-05), at the END for #: the same reason, and it is the one command in the list that is Python rather -#: than bash: the list is SIXTEEN. +#: than bash, which made the list SIXTEEN. `lane-worktrees` joined it for +#: opensoft/openRepoTools#162 (Brett Heap's word of 2026-10-05, "build 162"), +#: at the END for the same reason, and Python like `lanes-index`: the list is +#: SEVENTEEN. INSTALLED = ("openRepoTools", "park", "resume", "status", "lane", "lanes", "lane-handoff", "lane-rename", "lanes-edit.sh", "lane-start", "lane-end", "link-estates", "repos.tsv", "claude-current", - "claude-restart-check", "lanes-index") + "claude-restart-check", "lanes-index", "lane-worktrees") #: The skills `--install` also places, at two paths each, and the paths they are #: fetched from when there is no checkout to copy them out of (Amendment 9(b), @@ -91,11 +94,11 @@ COMMAND_NAMES = ("handoff", "ctx", "swap") COMMAND_PATHS = tuple(f"commands/{n}.md" for n in COMMAND_NAMES) -#: Everything a stdin install has to fetch: the sixteen files, the three skills +#: Everything a stdin install has to fetch: the seventeen files, the three skills #: and the three command files. FETCHED = INSTALLED + SKILL_PATHS + COMMAND_PATHS -#: THIRTY ARTIFACTS, AND THE COUNT IS THE INVARIANT: sixteen files in the +#: THIRTY-ONE ARTIFACTS, AND THE COUNT IS THE INVARIANT: seventeen files in the #: bin directory, three skills in the shared skills directory, their three #: bare-run copies, three command files at that same pair of destinations, and #: TWO merged entries in `~/.claude/settings.json`. Derived from the three @@ -114,7 +117,7 @@ + HOOK_ENTRIES) USAGE_LINES = ( - "openRepoTools --install install (or update) the sixteen estate and", + "openRepoTools --install install (or update) the seventeen estate and", "openRepoTools wip init create your workspace repository, clone it,", "openRepoTools --help | --version", ) @@ -127,7 +130,7 @@ WINDOWS_SKIP] #: `--install` HARD-REQUIRES `jq` SINCE lane-collision-protocol AMENDMENT 9(b): -#: two of its thirty artifacts are merged entries inside a JSON file somebody +#: two of its thirty-one artifacts are merged entries inside a JSON file somebody #: else owns, and the clause has it refuse naming `jq` rather than rewriting #: that file by hand. So a run of `--install` on a host without `jq` is a #: REFUSAL BY DESIGN, and a test that asserts a successful placement there is @@ -225,7 +228,7 @@ def test_help_prints_every_usage_line(): def test_help_names_every_command_it_places_and_the_standards_front_door(): - """`--install` places sixteen files, and fifteen of them are commands this one + """`--install` places seventeen files, and sixteen of them are commands this one knows nothing about — so `--help` has to say what they are and where the rest is written down. A command a person has on PATH and cannot find written down is a command they will not use. @@ -243,7 +246,8 @@ def test_help_names_every_command_it_places_and_the_standards_front_door(): "lane-start ", "lane-end ", "lanes-edit.sh ", "link-estates", "repos.tsv", "claude-current [--porcelain]", "claude-restart-check", - "lanes-index sync|reconcile|status"): + "lanes-index sync|reconcile|status", + "lane-worktrees sweep "): assert line in result.stdout, line assert "`openRepoShape` is the standard's front door" in result.stdout assert "this command scaffolds none" in result.stdout @@ -416,7 +420,7 @@ def test_installing_twice_changes_nothing(tmp_path): assert second.returncode == 0, second.stderr for name in INSTALLED: assert f"{name}: already installed at" in second.stdout, name - # THIRTY, not sixteen: the six skill copies, the six command-file copies + # THIRTY-ONE, not seventeen: the six skill copies, the six command-file copies # and BOTH hook entries each report `unchanged` too, and the count is the # invariant Amendment 9(b) names — derived from the three lists, never # restated, so a new skill or command moves it. It was eighteen until @@ -429,7 +433,7 @@ def test_installing_twice_changes_nothing(tmp_path): @NEEDS_JQ def test_install_replaces_a_copy_that_has_drifted(tmp_path, name): """Per file, and only the one that drifted: an install that rewrote all - sixteen every time would have nothing to say about which one was stale.""" + seventeen every time would have nothing to say about which one was stale.""" assert run_cmd("--install", home=tmp_path).returncode == 0 target = tmp_path / ".local" / "bin" / name target.write_text(target.read_text(encoding="utf-8") + "# drift\n", @@ -853,8 +857,8 @@ def test_install_writes_a_receipt_of_every_file_it_placed(tmp_path): command files at both of their destinations. The two hook entries are entries inside `~/.claude/settings.json` and not files this command placed, and the receipt carries no row for itself either. Today that is - 30 artifacts and 28 rows (16 + 6 + 6, since opensoft/openRepoTools#160 put - `lanes-index` in the list); the number moves with those three lists and + 31 artifacts and 29 rows (17 + 6 + 6, since opensoft/openRepoTools#162 put + `lane-worktrees` in the list); the number moves with those three lists and with nothing else, which is why no assertion below spells it. """ result = run_cmd("--install", home=tmp_path) diff --git a/tests/test_repo_hygiene.py b/tests/test_repo_hygiene.py index 82775c2..52cb284 100644 --- a/tests/test_repo_hygiene.py +++ b/tests/test_repo_hygiene.py @@ -2696,7 +2696,7 @@ def test_every_shipped_bash_file_is_tracked_with_lf(): def test_the_python_files_compile(): - """The suite is the Python here, bar one shipped command named below, and a + """The suite is the Python here, bar two shipped commands named below, and a file that does not parse is a file that fails as a collection ERROR rather than as a test — which reads as the suite being broken instead of as one file being wrong. @@ -2704,15 +2704,17 @@ def test_the_python_files_compile(): `compile()` rather than `py_compile`: nothing is written anywhere, so this runs in a read-only checkout and leaves no `__pycache__` behind. - opensoft/openRepoTools#160: `lanes-index` is the one shipped command that - is Python, and it has no `.py` suffix for the glob below to find, so its + opensoft/openRepoTools#160: `lanes-index` is a shipped command that is + Python, and it has no `.py` suffix for the glob below to find, so its text is compiled the same way — a syntax error there is a command that - fails on every workstation's first sync. + fails on every workstation's first sync. opensoft/openRepoTools#162 put + `lane-worktrees` beside it, for the same reason. """ for path in sorted((REPO / "tests").glob("*.py")): compile(path.read_text(encoding="utf-8"), str(path), "exec") - command = REPO / "lanes-index" - compile(command.read_text(encoding="utf-8"), str(command), "exec") + for name in ("lanes-index", "lane-worktrees"): + command = REPO / name + compile(command.read_text(encoding="utf-8"), str(command), "exec") # --- the macOS job RUNS this bash, it does not only parse it (R-A9-11) ------ From 4f2acfb6b56e3e81e20627379d4490af389d36fc Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 20:24:12 +0000 Subject: [PATCH 04/29] Document the sweep in the lanes manual "Retiring a lane's worktrees": the rule, which trees are the lane's and who may act, the disposition table as implemented in the order it is decided, the other rows (--branches, scratch, caches, sandboxes, links, bundles), the sweeps directory and its manifest, the one register line per tree, retention and sweep.conf, the porcelain exit contract for lane-end's gate, and the two protocol lines the act assumes, proposed for the amendment that ratifies it. Part of #162. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- docs/README-lanes.md | 167 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) diff --git a/docs/README-lanes.md b/docs/README-lanes.md index 6285268..6beff9c 100644 --- a/docs/README-lanes.md +++ b/docs/README-lanes.md @@ -3301,6 +3301,173 @@ reconcile** (the rebuild equals a synced index row for row), **replay**, says `read: sources` and gives the source read's answer). The parser is held to `LOG_AWK`'s output line for line. No test creates a real database. +## Retiring a lane's worktrees — `lane-worktrees sweep` (#162) + +**On Brett Heap's word of 2026-10-05 ("build 162")**, tracked on +[opensoft/openRepoTools#162](https://github.com/opensoft/openRepoTools/issues/162). +A swap that does not finish gracefully — a usage limit, a killed pane, a crashed +harness — leaves its writers' trees where they stood, and the protocol's +worktree-safety rule (a swap never commits, pushes, stashes, resets or cleans) +is right to leave them. This is the act that retires them afterwards, in one +place, under one rule: **nothing is deleted that is not first on origin or in a +bundle under the sweeps directory, and a tree a live writer owns is never +touched.** + +```sh +lane-worktrees sweep # the DRY RUN: the table, nothing changed +lane-worktrees sweep --yes [--live |--live none] +lane-worktrees sweep --dry-run --porcelain # lane-end's gate (#163): 0 / 3 / 2 +lane-worktrees sweep --expire [--yes] # archives past retention +``` + +**Which trees are the lane's** is #97's inventory (`lanes-edit.sh lane-trees`) +and the disk — **never the derived index** (Amendment 14 clause (b)). The sweep +also reads every registration of the lane's checkout and every checkout under +its two roots (`/.claude/worktrees/*`, `.lane-worktrees//*`). +A tree the inventory does not name is **FOREIGN**: reported and left, unless +`--include-foreign` and a `--word ""` (recorded in every register line +it causes). A tree ANOTHER lane's inventory names is that lane's and is never +taken from here, word or no word. The lane's own checkout is never a candidate. + +**Who may act** is #97's reconciliation (`lanes-edit.sh lane-reconcile`), read +before anything is written — the fetch included: + +| the lane | dry run | `--yes` | +|---|---|---| +| bound on another host or container (Amendment 18(b): UNKNOWN, never dead) | the table, exit **2** | refused, exit **2** | +| its binding, holder or reconciliation could not be read | the table, exit **2** | refused, exit **2** | +| held by ANOTHER live session | the table, exit **2** | refused, exit **2** | +| managed-owned (ruling 2026-10-04) | refused, exit **2** | refused, exit **2** | +| held by THIS session | the table | only with the coordinator's writer count: `--live ` per live writer (its ListAgents count, Amendment 17 Addendum 1 (i)), or `--live none` | +| `SWAPPING` in a live session | the table | refused: a handoff is in flight | +| no live holder, bound here, free or gone | the table | performed | + +### The table, as implemented + +Each tree is classified in this order; the first row that matches decides. + +| tree | disposition | what `--yes` does | +|---|---|---| +| not in the lane's inventory | `foreign` | nothing (`--include-foreign --word …` applies the rows below) | +| a process stands in it or holds a file open there (`/proc`, else `lsof`), a tmux pane's current path is in it, it is named by `--live`, the sweep was started inside it, or the session that recorded it is live | `live` | nothing, ever — asked again at the moment of the act | +| liveness could not be read | `keep` | nothing | +| its directory is gone and git still registers it | `prune` | `git worktree remove ` (the one registration; `git worktree prune` only if that is refused) | +| its directory is gone and nothing registers it | `gone` | nothing; the inventory record is history | +| a registration someone LOCKED; a submodule with uncommitted work or a commit no remote holds; a repository nested inside it; a CLONE with a branch or a stash origin lacks | `keep` | nothing, and the line says which | +| something leans on it — another repository's `objects/info/alternates`, or a remote whose URL is its path | `load-bearing` | nothing; the line names every dependent and the remedy (`git repack -a -d`, then drop the alternates or re-point the remote) | +| no `origin` | `keep` (`bundle+remove` with `--bundle`) | a bundle is its only rescue | +| dirty or untracked work | `wip-rescue+remove` | `git add -A` into a COPY of its index, `commit-tree` on its head, `rescue//-` pushed and seen on origin, a bundle, then removed | +| an unborn branch, clean | `remove` | removed | +| detached at a commit origin holds | `remove` | removed | +| detached with commits of its own | `rescue+remove` | `rescue//-` pushed, a bundle, then removed | +| MERGED — its PR LANDED in the register, or `gh` says MERGED, and that PR's head holds this tip | `remove+delete-branch` | removed; local branch deleted; the remote branch too where its `Lane:` trailer is this lane's and it holds nothing the PR did not | +| clean, every commit on its own remote branch (an open PR is untouched) | `remove` | removed; the branch stays | +| clean, its head on some origin branch | `remove` | removed; the branch stays | +| unpublished commits on `main`/`master`, or on a branch origin has DIVERGED from | `rescue+remove` | the tip to `rescue//-`, then removed; the branch itself is never force-pushed | +| unpublished commits otherwise | `push+remove` | pushed AS IS under its OWN name — never its upstream's, which for a branch made from `origin/main` is `main` — then removed | + +**"Merged" is never `git branch --merged` alone**: a squash merge leaves no +ancestry, and a branch whose tip IS on `main` with no pull request naming it is +listed as unmerged and its branch kept. The branch's pull request comes from one +`gh pr list --state all` per repository; with `LANES_NO_GITHUB=1` or no `gh` +nothing is called merged. The register's `LANDED` lines are read from +`origin/` of the workspace repository. + +**At the moment of removal** the head is re-read, and the tree must still be +clean — or, after a WIP rescue, must still be byte for byte the tree the rescue +commit carries. A push that origin refuses leaves the tree exactly as it was: +the WIP commit is made BESIDE the tree, never into it, so its branch, index and +files never move. + +**Ignored files that are not caches or build output** (an `.env`, a local +config) are archived to `-ignored.tar.gz` before a tree goes: a commit +cannot carry them, and the rule is that nothing goes that is not first saved. +Build output (`target`, `build`, `dist`, `.tox`, `.next`, `*.egg-info`, …) is +generated and is not archived; more than `ignored_archive_mb` of the rest leaves +the tree in place for a person. + +### The other rows + +| flag | what | disposition | +|---|---|---| +| `--branches` | local branches no worktree holds, in the lane's checkout | merged by PR evidence: `delete` (`delete+remote` where the `Lane:` trailer is this lane's and origin's tip is inside the PR); an open PR's branch: `keep`; unmerged with a missing, diverged or unpushed upstream: `list` with its tip, distance from `origin/main`, last commit date and owner — never deleted; `main`, `master` and `rescue/*`: never touched | +| `--include-scratch` | `.lane-worktrees//*-scratch`, `briefs/`, `bin/` that are no checkout | `archive+remove`: tar (caches left out) and sha256, then removed | +| `--include-caches` | `__pycache__`, `.pytest_cache`, `.mypy_cache`, `.ruff_cache`, `node_modules`, any directory with `pyvenv.cfg` (and `venv/`, `.venv/` with an activate script), under `.lane-worktrees//**` and the lane's inventory trees | `remove`, without archiving — never one git tracks, one in a live tree, or one inside scratch being archived | +| `--include-sandboxes` | `tmp.*` and `pytest-of-$USER/pytest-*` in `/tmp` and `$TMPDIR` (or `LANE_WORKTREES_SANDBOX_ROOTS`), owned by this account | `remove` where the owning process is gone: a pytest `.lock` naming a dead pid, and no live process standing in it, holding it open or naming it in its environment; a `tmp.*` younger than `sandbox_min_age_minutes` is left | +| `--links` | every symlink under the estate and every worktree `.git` gitdir pointer | `broken`, listed with target and age; nothing changes | +| `--bundle` | every tree acted on | a `git bundle` beside the archive (always, for a rescue) | + +### The sweeps directory, the register line, retention + +`${XDG_STATE_HOME:-$HOME/.local/state}/openRepoTools/sweeps///`, +mode 0700, created by the first act of a `--yes` and never by a dry run: + +* `DISPOSITION.md` — every tree and item acted on: what it was, why, what was + done, where its rescue is, and the register line that records it; +* `rescues.tsv` — `origin`, rescue branch and sha for each rescue, which + `--expire` reads; +* the bundles, `-ignored.tar.gz` and `scratch-.tar.gz`; +* `MANIFEST.sha256` — `sha256sum` format, every file above. + +**One register `NOTED` line per tree acted on** (`lanes-edit.sh log NOTED +lane:`), written after the act so it says what happened, naming the +disposition, the rescue and the archive; one more each for scratch, branches, +and caches with sandboxes. A line the register refuses is printed whole for a +person to write by hand, and the exit is 1. The line is the pointer that +outlives the archive. + +**Retention: 90 days** (`sweep.conf`). `sweep --expire` lists archives older than +that, and with `--yes` removes those whose every rescue branch is still on +origin (`git ls-remote`); an archive whose rescue branch is gone from origin is +the only copy and is NEVER expired. What expired is appended to +`sweeps/EXPIRED.log`. + +`${XDG_CONFIG_HOME:-$HOME/.config}/openRepoTools/sweep.conf` (or +`$LANE_WORKTREES_CONF`), `key=value` lines: + +| key | default | what | +|---|---|---| +| `retention_days` | 90 | archive age before `--expire` may remove it | +| `aging_days` | 14 | a rescue branch or dirty inventory tree older than this is "awaiting disposition" in the report | +| `sandbox_min_age_minutes` | 60 | a `tmp.*` younger than this is never a killed suite's | +| `foreign_quiet_hours` | 24 | a FOREIGN tree active within this is treated as live, even with `--include-foreign` | +| `ignored_report_mb` | 50 | the report's threshold for an ignored directory | +| `ignored_archive_mb` | 200 | ignored files (not caches, not build output) a tree may carry into its archive; more, and the tree is left for a person | + +### The exit contract (`lane-end`'s gate, #163) + +`lane-worktrees sweep --dry-run --porcelain` prints one tab-separated +row per tree and item, between a `lane` row and a `summary` row: + +```text +lane +refused (only when refused) +tree + (branch, scratch, cache, sandbox, link) +summary +``` + +and exits **0** (nothing to retire), **3** (something to retire — a tree of the +lane's on disk, live or not, a stale registration, an unpublished or merged +branch of the lane's, included scratch), or **2** (refused: bound elsewhere, +held by another session, managed, or a read failed). FOREIGN trees, links, +caches and sandboxes are never the lane's to retire. `--yes` exits 0 when every +act completed and 1 when one failed part-way (the rest go on; each failure is in +the table and in `DISPOSITION.md`); usage is 64. + +### Two protocol lines this act assumes (proposed for the amendment that ratifies it) + +1. **A lane creates worktrees, never clones.** Every tree a lane works in is a + `git worktree` of the estate's checkout, recorded in its inventory (#97). A + standalone clone under a lane root is FOREIGN to every sweep, and one that is + a shared store or a local remote is LOAD-BEARING. A lane adds no local remote + and no alternates. +2. **Scratch lives in the session scratchpad or the lane's state directory, + never beside repositories.** `.lane-worktrees//*-scratch`, `briefs/` + and `bin/` are grandfathered into the archive-then-remove row; new scratch + goes to `${XDG_STATE_HOME:-$HOME/.local/state}/openRepoTools/lanes//scratch/` + or the harness scratchpad. + ## Hand edits After **any** hand edit made with an allowed tool (python read/write, `sed -i From 7e58b987c701949e71fa5b31f1b49ff51251963e Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 20:31:29 +0000 Subject: [PATCH 05/29] Walk the estate for --links with scandir `os.walk` lstat()s every entry, and the estate on Eagle is tens of thousands of directories: a read-only `--links` dry run took 245 s. A scandir walk reads each entry's kind from the directory read itself, so only a symlink or a `.git` file costs a call of its own. Measured on the same estate: 16.7 s, with the same 403 broken links found (99 of them worktree gitdir pointers). Part of #162. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lane-worktrees | 82 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 30 deletions(-) diff --git a/lane-worktrees b/lane-worktrees index dda13b4..211f5f9 100755 --- a/lane-worktrees +++ b/lane-worktrees @@ -2327,41 +2327,63 @@ def _lock_pid(path: str) -> int: def link_items(root: str) -> list: """`--links`: every symlink under the estate and every worktree `.git` - gitdir pointer, checked to resolve; the broken ones, with target and age.""" + gitdir pointer, checked to resolve; the broken ones, with target and age. + + A `scandir` walk: an entry's kind comes from the directory read itself, so + only a symlink or a `.git` FILE costs a call of its own - the estate is tens + of thousands of directories, and `os.walk`'s `lstat` of each was minutes.""" out = [] - prune = set(CACHE_NAMES) | set(VENV_NAMES) | {".git"} + prune = set(CACHE_NAMES) | set(VENV_NAMES) | {".git", ".tox", ".nox"} now = time.time() - for base, dirs, files in os.walk(root, followlinks=False): - for name in list(dirs) + files: - full = os.path.join(base, name) + stack = [root] + while stack: + d = stack.pop() + try: + entries = list(os.scandir(d)) + except OSError: + continue + for e in entries: try: - st = os.lstat(full) + if e.is_symlink(): + if not os.path.exists(e.path): + try: + target = os.readlink(e.path) + age = int((now - os.lstat(e.path).st_mtime) // 86400) + except OSError: + target, age = "?", 0 + out.append(Item("link", e.path, "broken", f"symlink to {target} that " + f"does not resolve ({age} d old)", False)) + continue + if e.name == ".git": + if e.is_file(follow_symlinks=False): + _gitdir_pointer(e.path, d, now, out) + continue + if e.name in prune: + continue + if e.is_dir(follow_symlinks=False): + stack.append(e.path) except OSError: continue - if stat.S_ISLNK(st.st_mode): - if not os.path.exists(full): - try: - target = os.readlink(full) - except OSError: - target = "?" - age = int((now - st.st_mtime) // 86400) - out.append(Item("link", full, "broken", f"symlink to {target} that does not " - f"resolve ({age} d old)", False)) - elif name == ".git" and stat.S_ISREG(st.st_mode): - try: - with open(full, encoding="utf-8") as fh: - first = fh.readline().strip() - except OSError: - continue - if first.startswith("gitdir:"): - target = first[7:].strip() - resolved = target if os.path.isabs(target) else os.path.join(base, target) - if not os.path.isdir(resolved): - age = int((now - st.st_mtime) // 86400) - out.append(Item("link", full, "broken", f"gitdir pointer to {target} " - f"that does not resolve ({age} d old)", False)) - dirs[:] = [d for d in dirs if d not in prune and not os.path.islink(os.path.join(base, d))] - return out + return sorted(out, key=lambda i: i.path) + + +def _gitdir_pointer(path: str, base: str, now: float, out: list) -> None: + try: + with open(path, encoding="utf-8") as fh: + first = fh.readline().strip() + except OSError: + return + if not first.startswith("gitdir:"): + return + target = first[7:].strip() + resolved = target if os.path.isabs(target) else os.path.join(base, target) + if not os.path.isdir(resolved): + try: + age = int((now - os.lstat(path).st_mtime) // 86400) + except OSError: + age = 0 + out.append(Item("link", path, "broken", f"gitdir pointer to {target} that does " + f"not resolve ({age} d old)", False)) # ================================================================== expire From 8d2b71bfea6f6d8b453b02f6bc5ee1eb884edd2d Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:08:27 +0000 Subject: [PATCH 06/29] Keep a test run's bytecode, cache and temp out of the worktree A suite run left tests/__pycache__, .pytest_cache and, when killed, its sandboxes in whichever worktree ran it; the estate cleanup found those were 968 of 1,000 removals (#162, causes 3 and 4). tests/run.sh now sends bytecode to ${XDG_CACHE_HOME:-~/.cache}/openRepoTools/pycache, turns pytest's cache off, and roots --basetemp and TMPDIR under one run root in ${XDG_STATE_HOME:-~/.local/state}/openRepoTools/tmp/-/ that an EXIT trap removes however the run ends. The suite now runs in the background and is waited for: bash runs a trap only once its foreground child returns, so a TERM used to wait out the whole suite. The INT and TERM handlers stop the suite, then exit, so the EXIT trap removes the run root and, on the mkdir path, the lock. The lock is computed before TMPDIR moves, so it stays the workstation's. The repository venv is used when it has pytest, first on PATH so the command line still reads `python3 -m pytest`. tests/test_run_wrapper.py proves it against a stub suite: the relocation, the run root gone after a pass, a failure, a TERM and an INT, under flock and under the mkdir lock, and the venv. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 19 +++- tests/run.sh | 88 +++++++++++++++- tests/test_run_wrapper.py | 209 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 308 insertions(+), 8 deletions(-) create mode 100644 tests/test_run_wrapper.py diff --git a/AGENTS.md b/AGENTS.md index 89fad47..2ee003b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -257,9 +257,22 @@ The run itself is minutes of bash and hundreds of `git` processes, and several lanes build in sibling worktrees of one checkout: the wrapper waits for any live run, takes `${TMPDIR:-/tmp}/openrepotools-pytest.lock` (`flock` where there is one, a `mkdir` lock on macOS, which has none), waits again -inside it, then runs `python3 -m pytest tests -q "$@"`. Every lane on one -workstation must name the SAME lock file or there is no lock, which is the -whole reason the path is written here as well as in the file. +inside it, then runs `python3 -m pytest tests -q -p no:cacheprovider +--basetemp=/basetemp "$@"`. Every lane on one workstation must name +the SAME lock file or there is no lock, which is the whole reason the path is +written here as well as in the file. + +**A run leaves nothing beside the code** (opensoft/openRepoTools#162): bytecode +goes to `${XDG_CACHE_HOME:-~/.cache}/openRepoTools/pycache` +(`PYTHONPYCACHEPREFIX`), pytest's cache is off, and every temporary directory — +`--basetemp` and the `TMPDIR` the suite's own `mktemp`s use — is under one run +root, `${XDG_STATE_HOME:-~/.local/state}/openRepoTools/tmp/-/`, which +the wrapper's EXIT trap removes however the run ends (a TERM or an INT stops the +suite first). The lock is computed BEFORE `TMPDIR` moves, so it is still the +workstation's. Where the repository's venv +(`${XDG_CACHE_HOME:-~/.cache}/openRepoTools/venvs/openRepoTools`) has `pytest`, +the suite runs from it, first on `PATH`, so the command line still reads +`python3 -m pytest`. Two measured defects on 2026-09-14 (opensoft/openRepoTools#51), both of them inside a guard that had been copied into four briefs: diff --git a/tests/run.sh b/tests/run.sh index f3f1835..be19e7f 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -42,6 +42,29 @@ # atomicity POSIX guarantees; the holder's pid is left inside it so a lock a # killed run left behind is taken over rather than waited on for ever. # +# NOTHING NEW LANDS BESIDE THE CODE (opensoft/openRepoTools#162, causes 3 and +# 4). A suite run used to leave `tests/__pycache__`, `.pytest_cache` and a +# killed run's sandboxes behind in whichever worktree ran it, and a lane's +# leftovers were 968 of 1,000 removals in the estate's cleanup. So: +# +# * BYTECODE goes to `${XDG_CACHE_HOME:-$HOME/.cache}/openRepoTools/pycache` +# (`PYTHONPYCACHEPREFIX`), never into a checkout; +# * pytest's cache is off (`-p no:cacheprovider`); +# * EVERY TEMPORARY DIRECTORY of the run - pytest's `--basetemp` and the +# `TMPDIR` the suite's own `mktemp`s use - is under ONE run root, +# `${XDG_STATE_HOME:-$HOME/.local/state}/openRepoTools/tmp/-/`, +# which an EXIT trap removes however the run ends. A SIGKILL is the one +# end no trap sees; the pid in the root's name is what lets +# `lane-worktrees sweep --include-sandboxes` tell its owner is gone. +# * the suite runs from the repository's own virtual environment where one +# is there and has pytest (`${XDG_CACHE_HOME:-$HOME/.cache}/openRepoTools/ +# venvs/openRepoTools`, which `openRepoTools --install` names), put FIRST +# on PATH so the command line still reads `python3 -m pytest` - the shape +# every other lane's poll counts. +# +# THE LOCK IS COMPUTED BEFORE `TMPDIR` IS MOVED: it is the workstation's, and +# a run that locked a file under its own run root would lock nothing. +# # bash 3.2 (Apple's stock `/bin/bash`) parses this file in CI, like every other # bash file here: no `${x,,}`, no `mapfile`, no `declare -A`, no `local -n`. @@ -79,13 +102,62 @@ wait_for_pytest() { # — block while any other suite is running return 0 } +# THE RUN ROOT AND ITS CLEANUP. `cleanup` is the EXIT trap and removes the run +# root and, on the `mkdir` path, the lock it holds; the INT and TERM handlers +# stop the suite and EXIT, so the EXIT trap runs - a handler that only cleaned +# up and returned would release the lock while the suite carried on. +RUN_ROOT="" +LOCKDIR_HELD="" +SUITE_PID="" +cleanup() { + [ -n "$RUN_ROOT" ] && rm -rf -- "$RUN_ROOT" + [ -n "$LOCKDIR_HELD" ] && rm -rf -- "$LOCKDIR_HELD" + return 0 +} +on_signal() { # + if [ -n "$SUITE_PID" ]; then + kill -TERM "$SUITE_PID" 2>/dev/null || : + wait "$SUITE_PID" 2>/dev/null || : + fi + exit "$1" +} + +make_run_root() { + run_base="${XDG_STATE_HOME:-$HOME/.local/state}/openRepoTools/tmp" + RUN_ROOT="$run_base/$(date -u +%Y%m%dT%H%M%SZ)-$$" + mkdir -p -- "$RUN_ROOT/tmp" "$RUN_ROOT/basetemp" || { + printf '%s: cannot create the run root %s\n' "$prog" "$RUN_ROOT" >&2 + exit 1 + } + chmod 700 "$RUN_ROOT" 2>/dev/null || : + export TMPDIR="$RUN_ROOT/tmp" + export PYTHONPYCACHEPREFIX="${XDG_CACHE_HOME:-$HOME/.cache}/openRepoTools/pycache" + venv="${XDG_CACHE_HOME:-$HOME/.cache}/openRepoTools/venvs/openRepoTools" + if [ -x "$venv/bin/python3" ] && "$venv/bin/python3" -c 'import pytest' >/dev/null 2>&1; then + PATH="$venv/bin:$PATH"; export PATH + printf '%s: the suite runs from the repository venv %s\n' "$prog" "$venv" >&2 + fi +} + # `${1+"$@"}` AND NEVER A BARE `"$@"`: under `set -u`, BASH 3.2 — Apple's stock # shell, and the one this file's `mkdir` lock exists for — treats `"$@"` with no # positional parameters as an unbound variable and exits. `"${*:-}"` is the same # rule for the line that echoes them. +# +# THE SUITE RUNS IN THE BACKGROUND AND IS WAITED FOR, because bash runs a trap +# only once its FOREGROUND child returns: a TERM sent to this wrapper while +# pytest ran in the foreground waited out the whole suite. `wait` is +# interrupted by a trapped signal; the handler then stops the suite itself. +# `<&0` keeps the suite's stdin this shell's: a background command's default +# stdin, with job control off, is /dev/null. run_suite() { - printf '%s: python3 -m pytest tests -q %s\n' "$prog" "${*:-}" >&2 - python3 -m pytest tests -q ${1+"$@"} + printf '%s: python3 -m pytest tests -q -p no:cacheprovider %s\n' "$prog" "${*:-}" >&2 + python3 -m pytest tests -q -p no:cacheprovider --basetemp="$RUN_ROOT/basetemp" ${1+"$@"} <&0 & + SUITE_PID=$! + wait "$SUITE_PID" + rs_rc=$? + SUITE_PID="" + return "$rs_rc" } wait_for_pytest "before the lock" @@ -95,7 +167,11 @@ if command -v flock >/dev/null 2>&1; then # by its exit — including a kill, which no `rm` in a trap would survive. exec 9> "$LOCK" || { printf '%s: cannot open the lock file %s\n' "$prog" "$LOCK" >&2; exit 1; } flock 9 || { printf '%s: could not take the lock %s\n' "$prog" "$LOCK" >&2; exit 1; } + trap cleanup EXIT + trap 'on_signal 130' INT + trap 'on_signal 143' TERM wait_for_pytest "inside the lock" + make_run_root run_suite ${1+"$@"} exit $? fi @@ -162,9 +238,11 @@ printf '%s\n' "$$" > "$LOCKDIR/pid" 2>/dev/null || : # THE SIGNAL HANDLERS EXIT, and that is the whole point of writing them out # separately: a handler that only cleans up and RETURNS releases the lock while # this suite carries on running, which is the collision the lock exists for. -trap 'rm -rf -- "$LOCKDIR"' EXIT -trap 'rm -rf -- "$LOCKDIR"; exit 130' INT -trap 'rm -rf -- "$LOCKDIR"; exit 143' TERM +LOCKDIR_HELD="$LOCKDIR" +trap cleanup EXIT +trap 'on_signal 130' INT +trap 'on_signal 143' TERM wait_for_pytest "inside the lock" +make_run_root run_suite ${1+"$@"} exit $? diff --git a/tests/test_run_wrapper.py b/tests/test_run_wrapper.py new file mode 100644 index 0000000..a17506f --- /dev/null +++ b/tests/test_run_wrapper.py @@ -0,0 +1,209 @@ +# SPDX-License-Identifier: Apache-2.0 +"""`tests/run.sh` leaves nothing beside the code (opensoft/openRepoTools#162). + +The wrapper's lock is `test_repo_hygiene.py`'s to pin; what is held here is +what #162 added around it: + + * bytecode goes to `${XDG_CACHE_HOME:-$HOME/.cache}/openRepoTools/pycache` + and pytest's cache is off, so no `__pycache__` or `.pytest_cache` lands in + a worktree; + * every temporary directory of a run is under ONE run root, + `${XDG_STATE_HOME:-$HOME/.local/state}/openRepoTools/tmp/-/`, + and the run root is gone when the run ends - however it ends: a pass, a + failure, a TERM or an INT in the middle of the suite; + * the lock stays where every lane names it, computed BEFORE `TMPDIR` moves; + * the `mkdir` lock (macOS has no `flock`) is released the same way; + * the repository's virtual environment is used when it has pytest, with the + command line still reading `python3 -m pytest`. + +THE SUITE ITSELF IS A STUB. A fake `python3` records how it was called and +sleeps or exits on cue, and a fake `pgrep` reports no other run - the real one +would see THIS suite's own `python3 -m pytest` and wait for ever. +""" + +from __future__ import annotations + +import os +import shutil +import signal +import subprocess +import time +from pathlib import Path + +import pytest + +from conftest import REPO, WINDOWS_SKIP + +pytestmark = [WINDOWS_SKIP, pytest.mark.skipif(shutil.which("bash") is None, + reason="the wrapper is bash")] + +WRAPPER = REPO / "tests" / "run.sh" + +FAKE_SUITE = """#!/bin/sh +# Records its own argv, the two environment values the wrapper sets, its pid +# and where it was found, then sleeps or exits on cue. +{ + printf 'argv0 %s\\n' "$0" + for a in "$@"; do printf 'arg %s\\n' "$a"; done + printf 'TMPDIR %s\\n' "$TMPDIR" + printf 'PYTHONPYCACHEPREFIX %s\\n' "$PYTHONPYCACHEPREFIX" + printf 'pid %s\\n' "$$" +} > "$FAKE_SUITE_LOG" +: > "$TMPDIR/the-suite-wrote-here" +if [ -n "${FAKE_SUITE_SLEEP:-}" ]; then + : > "$FAKE_SUITE_LOG.started" + exec sleep "$FAKE_SUITE_SLEEP" +fi +exit "${FAKE_SUITE_RC:-0}" +""" + +FAKE_PGREP = "#!/bin/sh\nexit 1\n" + + +class Box: + def __init__(self, root: Path, flock: bool = True): + self.root = root + self.fakebin = root / "fakebin" + self.fakebin.mkdir() + self.lockdir = root / "locktmp" + self.lockdir.mkdir() + self.state = root / "state" + self.cache = root / "cache" + self.log = root / "suite.log" + for name, text in (("python3", FAKE_SUITE), ("pgrep", FAKE_PGREP)): + (self.fakebin / name).write_text(text) + (self.fakebin / name).chmod(0o755) + if flock: + path = f"{self.fakebin}{os.pathsep}{os.environ.get('PATH', '')}" + else: + # NO `flock` ON THIS PATH: the wrapper takes its `mkdir` lock, the + # one macOS uses. Every other tool it calls is linked in by name. + for tool in ("dirname", "awk", "mkdir", "cat", "mv", "rm", "sleep", "date", + "chmod"): + found = shutil.which(tool) + assert found, tool + (self.fakebin / tool).symlink_to(found) + path = str(self.fakebin) + self.env = {k: v for k, v in os.environ.items() + if not k.startswith(("XDG_", "PYTHON", "FAKE_"))} + self.env.update(PATH=path, TMPDIR=str(self.lockdir), XDG_STATE_HOME=str(self.state), + XDG_CACHE_HOME=str(self.cache), HOME=str(root / "home"), + FAKE_SUITE_LOG=str(self.log)) + + def popen(self, *args, **env) -> subprocess.Popen: + """The wrapper, its output to FILES: a pipe is held open by every + process that inherited it, and `communicate` waits for all of them.""" + child = dict(self.env) + child.update(env) + self.out = open(self.root / "wrapper.out", "w") + self.err = open(self.root / "wrapper.err", "w") + return subprocess.Popen([shutil.which("bash") or "bash", str(WRAPPER), *args], + env=child, stdout=self.out, stderr=self.err, + stdin=subprocess.DEVNULL) + + def finish(self, proc: subprocess.Popen, timeout: float = 60) -> str: + proc.wait(timeout=timeout) + self.out.close() + self.err.close() + return (self.root / "wrapper.err").read_text() + + def recorded(self) -> dict: + out: dict = {"arg": []} + for line in self.log.read_text().splitlines(): + key, _, value = line.partition(" ") + if key == "arg": + out["arg"].append(value) + else: + out[key] = value + return out + + def runs(self) -> list: + base = self.state / "openRepoTools" / "tmp" + return sorted(base.iterdir()) if base.is_dir() else [] + + def wait_started(self, timeout: float = 30) -> None: + deadline = time.time() + timeout + marker = Path(str(self.log) + ".started") + while time.time() < deadline: + if marker.exists(): + return + time.sleep(0.1) + raise AssertionError("the stub suite never started") + + +def test_a_run_relocates_bytecode_caches_and_temp_and_leaves_none_of_it(tmp_path): + box = Box(tmp_path) + proc = box.popen("-k", "lane_worktrees", FAKE_SUITE_RC="5") + err = box.finish(proc) + assert proc.returncode == 5, err + rec = box.recorded() + run_root = Path(rec["TMPDIR"]).parent + assert run_root.parent == box.state / "openRepoTools" / "tmp" + assert run_root.name.endswith(f"-{proc.pid}"), run_root.name + assert rec["PYTHONPYCACHEPREFIX"] == str(box.cache / "openRepoTools" / "pycache") + assert rec["arg"][:5] == ["-m", "pytest", "tests", "-q", "-p"], rec["arg"] + assert "no:cacheprovider" in rec["arg"] + assert f"--basetemp={run_root}/basetemp" in rec["arg"] + assert rec["arg"][-2:] == ["-k", "lane_worktrees"] + assert not run_root.exists(), "the run root outlived the run" + assert box.runs() == [] + assert (box.lockdir / "openrepotools-pytest.lock").exists(), ( + "the lock is the workstation's, under the caller's TMPDIR, not the run's") + + +@pytest.mark.parametrize("sig,rc", [(signal.SIGTERM, 143), (signal.SIGINT, 130)]) +@pytest.mark.parametrize("flock", [True, False], ids=["flock", "mkdir-lock"]) +def test_a_killed_run_removes_its_run_root_and_stops_the_suite(tmp_path, sig, rc, flock): + """THE TRAP, PROVED: the wrapper is signalled while the suite is running. + It stops the suite, removes the run root (and, on the `mkdir` path, the + lock), and exits with the signal's code - not after the suite would have + finished, which is what a foreground child would have made it do.""" + if flock and shutil.which("flock") is None: + pytest.skip("this host has no flock; the mkdir-lock case covers it") + box = Box(tmp_path, flock=flock) + proc = box.popen(FAKE_SUITE_SLEEP="120") + try: + box.wait_started() + rec = box.recorded() + run_root = Path(rec["TMPDIR"]).parent + assert (run_root / "tmp" / "the-suite-wrote-here").exists() + if not flock: + assert (box.lockdir / "openrepotools-pytest.lock.d").is_dir() + started = time.time() + proc.send_signal(sig) + box.finish(proc, timeout=30) + finally: + if proc.poll() is None: + proc.kill() + proc.wait() + assert proc.returncode == rc + assert time.time() - started < 20, "the trap waited for the suite" + assert not run_root.exists(), "the run root outlived a killed run" + suite_pid = int(rec["pid"]) + deadline = time.time() + 10 + while time.time() < deadline: + try: + os.kill(suite_pid, 0) + except ProcessLookupError: + break + time.sleep(0.1) + else: + raise AssertionError("the suite outlived its wrapper") + if not flock: + assert not (box.lockdir / "openrepotools-pytest.lock.d").exists(), "the lock was kept" + + +def test_the_repository_venv_is_used_when_it_has_pytest(tmp_path): + box = Box(tmp_path) + venv_bin = box.cache / "openRepoTools" / "venvs" / "openRepoTools" / "bin" + venv_bin.mkdir(parents=True) + stub = venv_bin / "python3" + stub.write_text("#!/bin/sh\ncase \"$1\" in -c) exit 0 ;; esac\nexec " + f"{box.fakebin / 'python3'} \"$@\"\n") + stub.chmod(0o755) + proc = box.popen() + err = box.finish(proc) + assert proc.returncode == 0, err + assert "runs from the repository venv" in err + assert box.recorded()["arg"][:2] == ["-m", "pytest"] + assert box.runs() == [] From c4d97dbebba831ca4dba3f266446aba7a1fd6cd8 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:08:32 +0000 Subject: [PATCH 07/29] Name the repository's virtual environment on --install One virtual environment per repository, outside the estate, is the convention the sweep's --include-caches row assumes (#162, cause 3): a venv/ inside a worktree is a leftover to remove. --install now names this repository's, ${XDG_CACHE_HOME:-~/.cache}/openRepoTools/venvs/ openRepoTools, and says whether it is there; when it is not, it prints the two commands that make it. It never makes one: that needs pip and the network, and an install from a checkout needs neither. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- README.md | 7 ++++++- openRepoTools | 18 ++++++++++++++++++ tests/test_openrepotools_command.py | 21 +++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3cce271..415c85d 100644 --- a/README.md +++ b/README.md @@ -260,7 +260,12 @@ that would not take it, and a create needs a directory's search bit as well as its write bit. A refusal creates none of those directories either. `cp` follows a symlink, and an install through one leaves the command uninstalled and writes these bytes into whatever it points at. Then a `17 of 17 placed in ` line, -and the `export PATH=…` line if that directory is not on your `PATH`. +and the `export PATH=…` line if that directory is not on your `PATH`. Last, it +NAMES this repository's virtual environment — one per repository, outside the +estate, at `${XDG_CACHE_HOME:-~/.cache}/openRepoTools/venvs/openRepoTools/` +(opensoft/openRepoTools#162) — and says whether it is there; it never creates +one, since that needs `pip` and the network, and prints the two commands that +do. `tests/run.sh` runs the suite from it once it is there and has `pytest`. It also places **fourteen things that are not files in that directory**: THREE SKILLS, `/handoff`, `/lane-swap` and `/restart`, each at diff --git a/openRepoTools b/openRepoTools index 052fc27..cb0fbc0 100755 --- a/openRepoTools +++ b/openRepoTools @@ -1090,6 +1090,24 @@ install_commands() { say " export PATH=\"$dir:\$PATH\"" ;; esac + venv_note +} + +# ONE VIRTUAL ENVIRONMENT PER REPOSITORY, OUTSIDE THE ESTATE +# (opensoft/openRepoTools#162, cause 3: "968 of 1,000 removals were dependency +# caches"). `${XDG_CACHE_HOME:-$HOME/.cache}/openRepoTools/venvs//` is +# the convention `docs/README-lanes.md` states, and this repository's own is +# NAMED here with its state - and NOT CREATED: making one needs pip and the +# network, and an install from a checkout needs neither. `tests/run.sh` runs +# the suite from it once it is there and has pytest. +venv_note() { + local venv="${XDG_CACHE_HOME:-$HOME/.cache}/openRepoTools/venvs/openRepoTools" + if [ -x "$venv/bin/python3" ]; then + say "openRepoTools: this repository's virtual environment is $venv (present; tests/run.sh runs from it when it has pytest)" + else + say "openRepoTools: this repository's virtual environment is $venv (not created; one per repository, outside the estate):" + say " python3 -m venv $venv && $venv/bin/python3 -m pip install pytest" + fi } diff --git a/tests/test_openrepotools_command.py b/tests/test_openrepotools_command.py index 380836a..250da45 100644 --- a/tests/test_openrepotools_command.py +++ b/tests/test_openrepotools_command.py @@ -1882,3 +1882,24 @@ def test_a_fetching_install_follows_the_ref_it_was_given(offline_github, assert result.returncode == 0, result.stderr + result.stdout for name in INSTALLED: assert (bin_dir / name).is_file(), name + + +def test_install_names_the_repository_venv_and_whether_it_is_there(tmp_path): + """One virtual environment per repository, outside the estate (#162): + `--install` names this repository's, says whether it is there, and - when + it is not - prints the two commands that make it. It never makes one: that + needs pip and the network, and an install from a checkout needs neither.""" + cache = tmp_path / "cache" + venv = cache / "openRepoTools" / "venvs" / "openRepoTools" + first = run_cmd("--install", home=tmp_path, env={"XDG_CACHE_HOME": str(cache)}) + assert first.returncode == 0, first.stderr + assert (f"openRepoTools: this repository's virtual environment is {venv} " + "(not created; one per repository, outside the estate):") in first.stdout + assert f"python3 -m venv {venv} && {venv}/bin/python3 -m pip install pytest" in first.stdout + assert not venv.exists(), "the install made a venv" + (venv / "bin").mkdir(parents=True) + (venv / "bin" / "python3").write_text("#!/bin/sh\nexit 0\n") + (venv / "bin" / "python3").chmod(0o755) + second = run_cmd("--install", home=tmp_path, env={"XDG_CACHE_HOME": str(cache)}) + assert second.returncode == 0, second.stderr + assert f"virtual environment is {venv} (present;" in second.stdout From c01ec4361cc05fdf35b4010ef66262d20a70829d Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:08:50 +0000 Subject: [PATCH 08/29] Seed the workspace .gitignore with bytecode and cache rules Cause 19 of the cleanup analysis: a handoff's attachments were committed with a whole virtualenv's __pycache__ inside them, and brett-wip carries 703 bytecode paths in its history. `wip init` now adds __pycache__/, *.pyc, *.pyo, .pytest_cache/, .mypy_cache/, .ruff_cache/, node_modules/, .venv/ and venv/ to the seeded .gitignore, in wip_substitute - the one function both step-7 paths go through - so the bytes a seed writes and the bytes a re-run compares are the same. A line the template already carries is not repeated, so a template that adopts them upstream seeds them once. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- openRepoTools | 40 +++++++++++++++++++++++ tests/test_wip_init_command.py | 59 ++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/openRepoTools b/openRepoTools index cb0fbc0..de1f7dd 100755 --- a/openRepoTools +++ b/openRepoTools @@ -2306,9 +2306,49 @@ wip_substitute() { never a blank, and never a literal {{…}} written into your repository. Nothing was created." fi + # THE WORKSPACE'S `.gitignore` CARRIES THE BYTECODE AND CACHE RULES + # (opensoft/openRepoTools#162, cause 19: generated bytecode committed into + # handoff attachments). Added HERE, in the one function both step 7 paths + # go through, so the bytes a seed writes and the bytes a re-run compares + # an earlier run's leftover against are the same bytes. A line the template + # already carries is not repeated. + case "${file##*/}" in + tpl-.gitignore) wip_hygiene_lines "$file.subst" >>"$file.subst" || + die "could not add the bytecode and cache rules to the template file $file" ;; + esac mv -f -- "$file.subst" "$file" } +#: The lines every `-wip` ignores (#162): bytecode, test and type-check +#: caches, dependency trees and virtual environments - a handoff attachment +#: is evidence, and none of these is. +WIP_HYGIENE_LINES=( + "__pycache__/" + "*.pyc" + "*.pyo" + ".pytest_cache/" + ".mypy_cache/" + ".ruff_cache/" + "node_modules/" + ".venv/" + "venv/" +) + +# The hygiene block, less every line `$1` already carries; nothing at all when +# it carries them all. +wip_hygiene_lines() { # + local line missing="" + for line in "${WIP_HYGIENE_LINES[@]}"; do + command grep -Fqx -- "$line" "$1" 2>/dev/null || missing="$missing$line +" + done + [ -n "$missing" ] || return 0 + printf '\n# Bytecode, caches, dependency trees and virtual environments are never\n' + printf '# a workspace repository'"'"'s: a handoff attachment is evidence, not a build\n' + printf '# (opensoft/openRepoTools#162). `lanes-edit.sh` refuses to commit them.\n' + printf '%s' "$missing" +} + # The template's bytes, materialized EXACTLY the way `--install` gets its own # files: from the checkout beside this command when run from one, else # `gh api …/contents/` at the pinned ref, else the raw URL. *That the diff --git a/tests/test_wip_init_command.py b/tests/test_wip_init_command.py index 7e32cbb..595edcb 100644 --- a/tests/test_wip_init_command.py +++ b/tests/test_wip_init_command.py @@ -2071,3 +2071,62 @@ def test_a_commit_that_fails_prints_what_git_said(tmp_path): f"the refusal dropped what git said:\n{result.stderr}") assert not (home / ".agents" / "workspace.yaml").exists(), ( "the pointer file was written by a run that never committed") + + +# --- the workspace ignores bytecode and caches (opensoft/openRepoTools#162) -- + +#: What `openRepoTools`' `WIP_HYGIENE_LINES` adds to the seeded `.gitignore`. +HYGIENE_LINES = ("__pycache__/", "*.pyc", "*.pyo", ".pytest_cache/", ".mypy_cache/", + ".ruff_cache/", "node_modules/", ".venv/", "venv/") + + +def test_the_seeded_gitignore_ignores_bytecode_caches_and_venvs(tmp_path): + """Cause 19 of the cleanup analysis: generated bytecode was committed into + handoff attachments. The seed's `.gitignore` is the template's AND the + nine hygiene lines, so `git add` of an attachment folder never picks a + `__pycache__` up; the re-run compares the same bytes and has nothing to + do.""" + home = tmp_path / "home" + env = fake_gh(tmp_path) + first = run_wip(home, extra=env) + assert first.returncode == 0, first.stdout + first.stderr + checkout = home / "projects" / "brettheap-wip" + lines = (checkout / ".gitignore").read_text(encoding="utf-8").splitlines() + template = (TEMPLATE / ".gitignore").read_text(encoding="utf-8").splitlines() + assert lines[:len(template)] == template, "the template's own lines come first, unchanged" + for line in HYGIENE_LINES: + assert lines.count(line) == 1, (line, lines) + probe = checkout / "handoffs" / "attachments" / "x" / "__pycache__" / "m.cpython-312.pyc" + probe.parent.mkdir(parents=True) + probe.write_bytes(b"\0") + ignored = subprocess.run(["git", "-C", str(checkout), "check-ignore", "-q", str(probe)], + check=False) + assert ignored.returncode == 0, "the workspace would commit bytecode" + probe.unlink() + second = run_wip(home, extra=env) + assert second.returncode == 0, second.stdout + second.stderr + assert "nothing to do" in second.stdout + + +@pytest.mark.parametrize("have,expect", [ + ("*.swp\n", HYGIENE_LINES), + ("__pycache__/\n*.pyc\nvenv/\n", tuple(x for x in HYGIENE_LINES + if x not in ("__pycache__/", "*.pyc", "venv/"))), + ("".join(f"{x}\n" for x in HYGIENE_LINES), ()), +]) +def test_a_hygiene_line_the_template_already_carries_is_not_repeated(tmp_path, have, expect): + """`wip_hygiene_lines`, by itself: the block less every line the file + already carries, and NOTHING - not even the comment - when it carries them + all, so a template that adopts the lines upstream seeds them once.""" + gitignore = tmp_path / "gitignore" + gitignore.write_text(have, encoding="utf-8") + source = COMMAND.read_text(encoding="utf-8") + start = source.index("\nWIP_HYGIENE_LINES=(") + end = source.index("\n}\n", source.index("\nwip_hygiene_lines()", start)) + 3 + proc = subprocess.run(["bash", "-c", source[start:end] + '\nwip_hygiene_lines "$1"\n', + "bash", str(gitignore)], capture_output=True, text=True, check=False) + assert proc.returncode == 0, proc.stderr + added = [ln for ln in proc.stdout.splitlines() if ln and not ln.startswith("#")] + assert tuple(added) == expect + if not expect: + assert proc.stdout == "" From 54585716d11e6c6067229588b2ffe489a3e431d7 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:08:50 +0000 Subject: [PATCH 09/29] Refuse a workspace commit whose pathspec carries bytecode The .gitignore lines keep a new workspace clean; an older one without them still stages whatever a pathspec names. commit_push now asks git what its pathspec would stage (git add --dry-run, which honours .gitignore exactly as the real add does) and refuses, exit 2, nothing staged, when any path has a __pycache__, cache, node_modules, venv or site-packages component or is *.py[co] - and offers the .gitignore lines that workspace lacks, with the one command that adds them. Attachments are committed by hand, so the same question is a subcommand: `lanes-edit.sh pathspec-check ...` answers 0 clean, 2 with the offending paths on stdout and the offer on stderr, 64 usage. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lanes-edit.sh | 83 +++++++++++++++++++++++++++++++++++++- tests/test_lane_helpers.sh | 44 ++++++++++++++++++++ 2 files changed, 125 insertions(+), 2 deletions(-) diff --git a/lanes-edit.sh b/lanes-edit.sh index 8eb0877..27f5839 100755 --- a/lanes-edit.sh +++ b/lanes-edit.sh @@ -351,6 +351,7 @@ # [--generation ] [--operation ] # lanes-edit.sh lane-tree-now # lanes-edit.sh lane-reconcile +# lanes-edit.sh pathspec-check ... # opensoft/openRepoTools#162 # # A lane is RUNNING, SWAPPING, SWAPPED or CLOSED, and WHICH OF THOSE IT IS # WITH NO LIVE HOLDER is what says where its session stopped: `RUNNING` with @@ -417,7 +418,9 @@ # checkout that cannot be rebased, — Amendment 15 — a register holding # two rows whose lane names differ only by case, which every writer and # `canon-lane` refuse until 15(d)'s merge, or AMENDMENT 12's BLOCKED -# PROMPT. SIX MEANINGS ON ONE NUMBER, and they +# PROMPT, or — opensoft/openRepoTools#162 — a commit whose pathspec +# would stage bytecode, a cache or a virtual environment +# (`commit_push`, `pathspec-check`). SEVEN MEANINGS ON ONE NUMBER, and they # stay readable only because of where each can occur: bad arguments to a # subcommand that predates Amendment 11; `register-row`'s "not lane-shaped"; # AN UNKNOWN SUBCOMMAND (the `*)` arm below), which is how a caller detects @@ -1804,6 +1807,63 @@ git_timeout_die() { # die "timed out after ${GIT_TIMEOUT}s on '$1' — the mutex is released and your commit is local; nothing was lost." 3 } +# BYTECODE NEVER RIDES A WORKSPACE COMMIT (opensoft/openRepoTools#162, cause +# 19: a handoff's attachments were committed with a whole virtualenv's +# `__pycache__` inside them, and the workspace repository carries them for +# ever). What `git add` WOULD stage for a pathspec is asked of git itself +# (`add --dry-run`, which honours `.gitignore` exactly as the real add does), +# and every path in that answer with a cache, bytecode or environment +# component is printed. Empty means the pathspec is clean. +# ONE PER LINE AND READ WITH `read`, never word-split: `*.pyc` unquoted is a +# glob, and the shell would expand it against whatever directory this runs in. +BYTECODE_IGNORES='__pycache__/ +*.pyc +*.pyo +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +node_modules/ +.venv/ +venv/' +bytecode_in_pathspec() { # ... + bip_repo="${1-}"; shift || : + [ "$#" -gt 0 ] || return 0 + git -C "$bip_repo" add --dry-run --ignore-missing -- "$@" 2>/dev/null | + sed -n "s/^add '\(.*\)'\$/\1/p" | + awk '{ + n = split($0, part, "/"); hit = 0 + for (i = 1; i <= n; i++) + if (part[i] == "__pycache__" || part[i] == ".pytest_cache" || + part[i] == ".mypy_cache" || part[i] == ".ruff_cache" || + part[i] == "node_modules" || part[i] == ".venv" || + part[i] == "venv" || part[i] == "site-packages") hit = 1 + if ($0 ~ /\.py[co]$/) hit = 1 + if (hit) print }' +} + +# The refusal's words, and the OFFER beside them: the ignore lines the +# workspace's own `.gitignore` lacks, with the one command that adds them. +# `openRepoTools wip init` seeds them into a new workspace; an older one is +# offered them here, where they are missing. +bytecode_refusal_text() { # + brt_n="$(printf '%s\n' "${1-}" | awk 'NF { n++ } END { print n + 0 }')" + brt_missing="" + while IFS= read -r brt_l; do + [ -n "$brt_l" ] || continue + command grep -Fqx -- "$brt_l" "$LANES_REPO/.gitignore" 2>/dev/null || + brt_missing="$brt_missing '$brt_l'" + done <> %s/.gitignore\n' \ + "$brt_missing" "$brt_missing" "$LANES_REPO" + fi + printf 'Nothing was staged or committed. Unstage or delete those paths, then re-run.' +} + # commit_push "" [...] — the pathspecs default to the # register alone, which is every caller that predates Amendment 7. A LANDING # or LANDED writes TWO (the register and the lane's log) so that both halves @@ -1871,6 +1931,8 @@ commit_push() { || die "git update-index --force-remove $cp_ren failed — the log is already renamed on disk and git still holds the old path, so nothing was committed. Re-run; if it refuses again, \`git -C $LANES_REPO rm --cached -- $cp_ren\` is the same act by hand." 6 fi if [ "$cp_add_n" -gt 0 ]; then + cp_byte="$(bytecode_in_pathspec "$LANES_REPO" ${cp_add[@]+"${cp_add[@]}"})" + [ -z "$cp_byte" ] || die "$(bytecode_refusal_text "$cp_byte")" 2 git -C "$LANES_REPO" -c "$CP_EXACT" add -- ${cp_add[@]+"${cp_add[@]}"} || die "git add failed" 6 fi if git -C "$LANES_REPO" -c "$CP_EXACT" diff --cached --quiet -- "${CP_PATHS[@]}"; then @@ -15776,7 +15838,24 @@ EOF esac ;; + # opensoft/openRepoTools#162 — WHAT A HAND COMMIT OF A HANDOFF'S ATTACHMENTS + # WOULD STAGE THAT IS BYTECODE, asked BEFORE the commit. `commit_push` + # refuses such a pathspec for every write this file makes; attachments are + # committed by hand, so this is the same question for a person to ask first. + # 0 nothing in those paths would stage bytecode, a cache or an environment + # 2 something would: the paths on stdout, the offer on stderr + # 64 usage + pathspec-check) + [ "$#" -gt 0 ] || die "usage: pathspec-check ... (relative to the workspace repository $LANES_REPO)" 64 + pck_out="$(bytecode_in_pathspec "$LANES_REPO" "$@")" + if [ -n "$pck_out" ]; then + printf '%s\n' "$pck_out" + note "$(bytecode_refusal_text "$pck_out")" + exit 2 + fi + ;; + *) - die "unknown subcommand '$cmd' (verify-row|set-row-state|append-row-status|replace-in-row|append-session-id|append-line|add-row|retire-rows|archive-rows|migrate-state-cells|commit|rename-lane|log|claim|release|who|history|swapped|session-start|guard|idle-holders|live-holder|window-session|transcript-holders|binding|request-handoff|session-lane|window-lane|lane-dir|lane-profile|lane-agent|lane-transcript|lane-last|workspace-root|last-session|forks|duplicate-holder|workstation|fetch-age|lanes|lane-groups|next-free|sibling-filter|resolve-repo|lane-objects|register-row|register-row-local|retired-identity|canon-lane|resolve-home|managed-projection|lane-state|set-lane-state|lane-trees|set-lane-tree|lane-tree-now|lane-reconcile)" 2 + die "unknown subcommand '$cmd' (verify-row|set-row-state|append-row-status|replace-in-row|append-session-id|append-line|add-row|retire-rows|archive-rows|migrate-state-cells|commit|rename-lane|log|claim|release|who|history|swapped|session-start|guard|idle-holders|live-holder|window-session|transcript-holders|binding|request-handoff|session-lane|window-lane|lane-dir|lane-profile|lane-agent|lane-transcript|lane-last|workspace-root|last-session|forks|duplicate-holder|workstation|fetch-age|lanes|lane-groups|next-free|sibling-filter|resolve-repo|lane-objects|register-row|register-row-local|retired-identity|canon-lane|resolve-home|managed-projection|lane-state|set-lane-state|lane-trees|set-lane-tree|lane-tree-now|lane-reconcile|pathspec-check)" 2 ;; esac diff --git a/tests/test_lane_helpers.sh b/tests/test_lane_helpers.sh index 3031dc3..6f0803b 100755 --- a/tests/test_lane_helpers.sh +++ b/tests/test_lane_helpers.sh @@ -451,6 +451,12 @@ export LANES_NO_GITHUB=1 # index turns it back on for its own cases. export LANES_INDEX=off +# opensoft/openRepoTools#162 — AND NO CASE STARTS THE ESTATE REPORT EITHER. +# `lane-start` runs `lane-worktrees sweep --all --report` detached, once a day, +# and the state directory it stamps may be the workstation's own. Off for the +# file; the section about the report turns it on for its own cases. +export LANE_WORKTREES_REPORT=off + # Amendment 8, ruling (g): the harness exports `CLAUDE_CODE_SESSION_ID` into # every shell a session runs, INCLUDING the one running this suite. It is a # holder test now, so a case that does not set it deliberately must not inherit @@ -14740,6 +14746,44 @@ a14_three "who (a read an act makes for itself)" 0 "$E" who opensoft/repoA#1414 a14_three "managed-projection (#97's seam)" 0 "$E" managed-projection repoMG-1 a14_three "history without the flag" 0 "$E" history repoA-1 +# ========================================================================= +echo "== opensoft/openRepoTools#162: bytecode never rides a workspace commit ==" +# +# A handoff's attachments were committed with a whole virtualenv's +# `__pycache__` inside them (cause 19). `pathspec-check` is the question a +# hand commit asks first, and `commit_push` asks it of every write this file +# makes: what `git add` WOULD stage that is bytecode, a cache or an +# environment. The `.gitignore` lines are OFFERED where the workspace lacks +# them, and once they are there the same pathspec stages nothing of the kind. +B162_REL="handoffs/sandbox/attachments/b162" +B162="$WIP/$B162_REL" +mkdir -p "$B162/r1/__pycache__" "$B162/r1/venv/lib/python3.12/site-packages" +printf 'findings\n' > "$B162/r1/notes.md" +printf 'bytecode\n' > "$B162/r1/__pycache__/m.cpython-312.pyc" +printf 'pkg\n' > "$B162/r1/venv/lib/python3.12/site-packages/pkg.py" +B162_IGN="" +[ -f "$WIP/.gitignore" ] && B162_IGN="$(cat "$WIP/.gitignore")" +run "$E" pathspec-check "$B162_REL" +is "pathspec-check refuses an attachment that carries bytecode" "$rc" 2 +has "…naming the bytecode" "$out" "$B162_REL/r1/__pycache__/m.cpython-312.pyc" +has "…and the virtual environment" "$out" "$B162_REL/r1/venv/lib/python3.12/site-packages/pkg.py" +hasnt "…and not the attachment's own notes" "$out" "notes.md" +has "…and offers the ignore lines the workspace lacks" "$err" "'__pycache__/'" +run "$E" pathspec-check "$B162_REL/r1/notes.md" +is "a pathspec with nothing of the kind passes" "$rc" 0 +b162_head="$(git -C "$WIP" rev-parse HEAD)" +run env LANES_LANE=repoA-1 LANES_PATH="$B162_REL" "$E" commit "attach r1" +is "the commit path refuses the same pathspec" "$rc" 2 +has "…saying what it is" "$err" "none of them is ever a workspace repository's" +is "…and commits nothing" "$(git -C "$WIP" rev-parse HEAD)" "$b162_head" +is "…and stages nothing" "$(git -C "$WIP" diff --cached --name-only | grep -c . || :)" 0 +printf '%s\n' '__pycache__/' '*.pyc' 'venv/' >> "$WIP/.gitignore" +run "$E" pathspec-check "$B162_REL" +is "with the ignore lines in place it stages nothing of the kind" "$rc" 0 +rm -rf "$WIP/handoffs/sandbox" +if [ -n "$B162_IGN" ]; then printf '%s\n' "$B162_IGN" > "$WIP/.gitignore"; else rm -f "$WIP/.gitignore"; fi +git -C "$WIP" checkout -q -- .gitignore 2>/dev/null || : + # ------------------------------------------------------- nothing real touched # # The check is that the suite CREATED nothing here, which is not the same as From 23786c29d7efce8af3c81c8c70e56ea2285e856a Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:08:50 +0000 Subject: [PATCH 10/29] Run the estate report once a day from lane-start The report is the net under every actor that never runs lane-end, and it only works if it is read every day. lane-start now runs `lane-worktrees sweep --all --dry-run --report` once per UTC day per workstation, after the row is written and before the launch: the first start of the day takes report-.stamp under the state directory with an atomic `set -C` create, removes older stamps, and starts the report detached, stdin, stdout and stderr closed, writing reports/.md. Every step either works or is skipped in silence, so the report never delays a start and never fails one. --dry-run starts none; LANE_WORKTREES_REPORT=off is the switch, and the suite sets it. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lane-start | 38 ++++++++++++++++++++++++++++++++++++++ tests/conftest.py | 5 +++++ 2 files changed, 43 insertions(+) diff --git a/lane-start b/lane-start index 0b7afd0..09a8aa6 100755 --- a/lane-start +++ b/lane-start @@ -3901,6 +3901,44 @@ if (( first_prompt )); then fi fi +# ------------- 5e. the estate's daily report (opensoft/openRepoTools#162) +# +# ONCE A DAY PER WORKSTATION, DETACHED, AND NEVER A REASON A START WAITS OR +# FAILS. `lane-worktrees sweep --all --dry-run --report` is the net under every +# actor that never runs `lane-end`: what is FOREIGN, orphaned, unpublished, +# aging or past retention across the estate, as one table, written to the state +# directory. It changes nothing. The first start of the day takes the stamp - +# `set -C` makes taking it one atomic create, so two starts at once run one +# report - and starts it with stdin, stdout and stderr closed, in a subshell +# that exits at once, so the report is no child of the session this command is +# about to exec. Every step of this either works or is skipped in silence: +# a report that could not start is tomorrow's start's to run. +# `LANE_WORKTREES_REPORT=off` is the switch, and both suites set it. +lane_daily_report() { + [ "${LANE_WORKTREES_REPORT:-on}" = off ] && return 0 + ldr_bin="" + if [ -x "$SCRIPT_DIR/lane-worktrees" ]; then + ldr_bin="$SCRIPT_DIR/lane-worktrees" + else + ldr_bin="$(command -v lane-worktrees 2>/dev/null || :)" + fi + [ -n "$ldr_bin" ] || return 0 + ldr_state="${XDG_STATE_HOME:-$HOME/.local/state}/openRepoTools" + ldr_day="$(date -u +%Y%m%d 2>/dev/null || :)" + [ -n "$ldr_day" ] || return 0 + mkdir -p -- "$ldr_state/reports" 2>/dev/null || return 0 + ( set -C; : > "$ldr_state/report-$ldr_day.stamp" ) 2>/dev/null || return 0 + for ldr_old in "$ldr_state"/report-*.stamp; do + [ "$ldr_old" = "$ldr_state/report-$ldr_day.stamp" ] || rm -f -- "$ldr_old" 2>/dev/null || : + done + ldr_out="$ldr_state/reports/$(date -u +%Y%m%dT%H%M%SZ).md" + ( "$ldr_bin" sweep --all --dry-run --report --post "$ldr_out" \ + /dev/null 2>&1 & ) 2>/dev/null || : + step "the estate's daily report is running, detached: $ldr_out" + return 0 +} +(( dry_run )) || lane_daily_report || : + # ---------------------------------------------------------------- 6. launch # # `cmd` was decided in 5a, before the row was written. diff --git a/tests/conftest.py b/tests/conftest.py index e2c9292..0f53a61 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,6 +36,11 @@ #: builds every environment it runs from scratch and decides for itself. os.environ["LANES_INDEX"] = "off" +#: opensoft/openRepoTools#162 — and no `lane-start` a test runs starts the +#: estate's daily report: it is detached, it reads the whole estate, and the +#: state directory it stamps may be the workstation's own. +os.environ["LANE_WORKTREES_REPORT"] = "off" + #: The pinned openRepoShape checkout. Every path this suite reads out of the #: standard hangs off this one name, so a bump of the pin moves one line in #: `contracts/openreposhape-pin.yaml` and the gitlink beside it, and nothing From 7fdef8ac598694bcbf3368fd0558f6502ef6a682 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:08:50 +0000 Subject: [PATCH 11/29] Report the estate's leftovers with sweep --all --dry-run --report Workspace creation outpaces closeout, and nothing counted it. The report reads every lane's #97 inventory and snapshot (beside each checkout's parent, where #97 keeps them), the workspace register and the disk - never the derived index - and lists, as one markdown document: FOREIGN clones with size, last commit and Lane: owner; orphaned trees of ENDED lanes; unmerged branches with a missing, diverged or unpushed upstream; root-main divergence, with Amendment 4's remedy where only handoff/register paths moved; rescue branches and dirty inventory trees awaiting disposition past aging_days; caches and sandboxes by size; ignored directories over ignored_report_mb; evidence-shaped paths; archives past retention (--expire's own table, now shared as expiry_rows); the workspace's .gitignore and bytecode history; and `status --all`'s findings, reported as what they are and never counted as dirt. The head table carries the rescue-branch count and the sweeps directory's size. It changes nothing: no fetch, no index refresh, no expiry. --post writes it to a file or comments it on owner/repo#n through gh. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lane-worktrees | 565 +++++++++++++++++++++++++++- tests/test_lane_worktrees_report.py | 366 ++++++++++++++++++ 2 files changed, 929 insertions(+), 2 deletions(-) create mode 100644 tests/test_lane_worktrees_report.py diff --git a/lane-worktrees b/lane-worktrees index 211f5f9..e4e91fe 100755 --- a/lane-worktrees +++ b/lane-worktrees @@ -10,6 +10,8 @@ abandoned worktrees, branches, scratch and killed-suite sandboxes [--include-foreign --word ""] [--live ... | --live none] [--fetch] [--estate ] lane-worktrees sweep --expire [--dry-run | --yes] [--porcelain] + lane-worktrees sweep --all --dry-run --report [--post ] + [--estate ] lane-worktrees --help | --version A SWAP THAT DOES NOT FINISH GRACEFULLY LEAVES ITS WRITERS' TREES WHERE THEY @@ -46,6 +48,16 @@ sweeps/// with MANIFEST.sha256, DISPOSITION.md and rescues.tsv. retention (90 days; `sweep.conf`) whose every rescue branch is still on origin; an archive whose rescue branch is gone from origin is never expired. +THE ESTATE REPORT (`--all --dry-run --report`) is the net under every actor +that never runs `lane-end`: FOREIGN clones, trees of ENDED lanes, unmerged +branches, root-`main` divergence, rescue branches and dirty trees awaiting +disposition, caches and sandboxes, large ignored directories, evidence-shaped +paths, archives past retention, the workspace's bytecode hygiene and +`status --all`'s findings - one markdown document, printed or `--post`ed to a +file or an issue. It changes nothing. `lane-start` runs it once a day per +workstation, detached (`LANE_WORKTREES_REPORT=off` is the switch). It exits 0, +or 1 when `--post` could not deliver it (the report is then printed). + Exit codes: 0 dry run: nothing to retire · --yes: every act completed 3 dry run: something to retire (the `lane-end` gate's answer, #163) @@ -2388,8 +2400,11 @@ def _gitdir_pointer(path: str, base: str, now: float, out: list) -> None: # ================================================================== expire -def expire(opts) -> int: - conf = load_conf() +def expiry_rows(conf: dict) -> list: + """(archive, age in days, keep|expire, why) for every archive under the + sweeps directory - `sweep --expire`'s table, and the report's. An archive + past retention is `expire` only while every rescue branch it records is on + origin; one whose rescue left origin is the only copy and is kept.""" root = sweeps_dir() keep_days = conf["retention_days"] now = _dt.datetime.now(_dt.timezone.utc) @@ -2430,6 +2445,14 @@ def expire(opts) -> int: + "; ".join(missing) + " - this archive is the only copy")) else: rows.append((adir, age, "expire", f"{age} d old and every rescue branch is on origin")) + return rows + + +def expire(opts) -> int: + conf = load_conf() + root = sweeps_dir() + keep_days = conf["retention_days"] + rows = expiry_rows(conf) failed = 0 for i, (adir, age, disp, why) in enumerate(rows): if disp == "expire" and opts.yes: @@ -2474,6 +2497,535 @@ def _rescues_of(adir: str) -> list: return out +# ================================================================== report + +EVIDENCE_NAME_RE = re.compile( + r"(^canary-|^deployment-evidence$|-evidence$|^evidence$|-report\.md$|REPORT[^/]*\.md$|" + r"^junit[^/]*\.xml$|^MANIFEST)", re.IGNORECASE) +HANDOFF_PATHS = ("handoffs/", "lanes/") +#: `rescue//-` names its lane; a rescue branch of another +#: convention (`rescue/`) names none. +LANE_SHAPED_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*-[0-9]+$") + + +def estate_root_default() -> str: + for cand in (os.environ.get("PROJECTS_ROOT", ""), os.path.join(_home(), "projects"), + os.path.join(_home(), "Projects")): + if cand and os.path.isdir(cand): + return cand + return "" + + +def _under_container(path: str, root: str) -> str: + """The worktree container a path sits in - `.lane-worktrees/`, + `.claude/worktrees`, `worktrees`, `-worktrees` - or ''.""" + rel = os.path.relpath(path, root) + parts = rel.split(os.sep) + for i, p in enumerate(parts[:-1]): + if p == ".lane-worktrees" and i + 1 < len(parts) - 1: + return os.sep.join(parts[:i + 2]) + if p in ("worktrees", ".worktrees") or p.endswith("-worktrees"): + return os.sep.join(parts[:i + 1]) + return "" + + +def _du(paths: list) -> dict: + """{path: bytes} through `du -sk`, which walks in C: an estate's ignored + trees are millions of files, and a Python walk of them is minutes.""" + out: dict = {} + for i in range(0, len(paths), 64): + chunk = paths[i:i + 64] + proc = run(["du", "-sk", "--", *chunk], timeout=600) + for line in proc.stdout.splitlines(): + size, _, path = line.partition("\t") + if size.isdigit(): + out[path] = int(size) * 1024 + return out + + +def _lane_parents(root: str, repos: list) -> list: + """Every directory a lane's control root and worktree root can sit in. + #97 puts `.lane-state/` - and lane-start `.lane-worktrees/` - + beside the lane's recorded `dir`, so a lane whose checkout is nested + (`/InkRouter/IRRS`) keeps them in `/InkRouter`, not at + the estate's top.""" + out = [root] if root else [] + for r in repos: + parent = os.path.dirname(r.rstrip(os.sep)) + if parent and parent not in out: + out.append(parent) + return out + + +def _lane_state_dirs(parents: list) -> list: + dirs = [] + cands = [os.environ.get("LANES_LANE_STATE_ROOT", "")] + cands += [os.path.join(p, ".lane-state") for p in parents] + for cand in cands: + if cand and os.path.isdir(cand) and cand not in dirs: + dirs.append(cand) + return dirs + + +def _lane_tree_roots(parents: list) -> list: + """`/.lane-worktrees`, each that exists.""" + out = [] + for p in parents: + cand = os.path.join(p, ".lane-worktrees") + if os.path.isdir(cand) and cand not in out: + out.append(cand) + return out + + +def _sidecar(path: str) -> dict: + out = {} + try: + with open(path, encoding="utf-8") as fh: + for line in fh: + key, sep, value = line.partition(": ") + if sep: + out[key.strip()] = value.strip() + except OSError: + pass + return out + + +class Report: + """`sweep --all --dry-run --report`: the estate's leftovers as one + document. It reads every lane's #97 inventory, the workspace register and + the disk; it changes nothing.""" + + def __init__(self, opts): + self.opts = opts + self.conf = load_conf() + self.root = opts.estate or estate_root_default() + self.helper = Helper() + self.sections: list = [] + self.counts: dict = {} + self.now = time.time() + + def add(self, title: str, rows: list, empty: str, note_text: str = "") -> None: + self.sections.append((title, rows, empty, note_text)) + self.counts[title] = len(rows) + + # ------------------------------------------------------------- reading + def build(self) -> None: + repos = estate_repos(self.root) if self.root else [] + self.deps = dependents_map(repos) + clones = [r for r in repos if os.path.isdir(os.path.join(r, ".git"))] + canonical = self._canonical(clones) + self.canonical = canonical + parents = _lane_parents(self.root, canonical) + self.state_dirs = _lane_state_dirs(parents) + self.tree_roots = _lane_tree_roots(parents) + self._foreign(clones, canonical) + closed = self._closed_lanes() + self._orphans(closed) + self._branches(canonical) + self._root_main(canonical) + self._aging(canonical) + self._caches_and_sandboxes() + self._ignored_and_evidence(repos, canonical) + self._archives() + self._workspace_hygiene() + self._status_all() + + def _canonical(self, clones: list) -> list: + """The estate's own checkouts: a clone outside every worktree + container. Of several clones of one origin, the one whose folder is + named for the repository is the checkout and the rest are duplicates.""" + by_origin: dict = {} + for c in clones: + if _under_container(c, self.root): + continue + url = git_out(c, "config", "--get", "remote.origin.url") or "" + key = (slug_of(url) or url).lower().rstrip("/") + by_origin.setdefault(key, []).append(c) + self.duplicates: dict = {} + out = [] + for key, group in by_origin.items(): + if not key or len(group) == 1: + out.extend(group) + continue + name = key.rsplit("/", 1)[-1].replace(".git", "") + named = [g for g in group if os.path.basename(g).lower() == name] + if len(named) == 1: + out.append(named[0]) + for g in group: + if g != named[0]: + self.duplicates[g] = named[0] + else: + out.extend(group) + return sorted(out) + + def _foreign(self, clones: list, canonical: list) -> None: + rows = [] + for c in clones: + container = _under_container(c, self.root) + why = "" + if container: + why = f"a CLONE inside {container}, where a lane makes worktrees" + elif c in self.duplicates: + why = f"a second clone of {self.duplicates[c]}'s origin" + if not why: + continue + last = git_out(c, "log", "-1", "--format=%cs") or "?" + msg = git_out(c, "log", "-1", "--format=%B") or "" + owner = (re.findall(r"^Lane:\s*(\S+)", msg, re.MULTILINE) or ["-"])[-1] + lean = leaned_on(c, git_common_dir(c), self.deps) + rows.append([c, why, human_size(dir_size(c)), last, owner, + ("LOAD-BEARING: " + _lean_text(lean)) if lean else ""]) + self.add("FOREIGN repositories", rows, "none: every clone is the estate's own checkout", + "A lane creates worktrees, never clones. Each is a person's to dispose of.") + + def _closed_lanes(self) -> dict: + """{lane: why} for every lane whose snapshot is CLOSED or whose log's + last lane-kind line is ENDED or RETIRED.""" + closed: dict = {} + for d in self.state_dirs: + for lane in sorted(os.listdir(d)): + if _sidecar(os.path.join(d, lane, "lane-state.yaml")).get("state") == "CLOSED": + closed[lane] = "its snapshot is CLOSED" + wroot = self.helper.workspace_root() if self.helper.path else "" + if wroot: + branch = os.environ.get("LANES_BRANCH", "main") + proc = git(wroot, "grep", "-E", "-e", + "^(STARTED|RESUMED|PAUSED|ENDED|RETIRED) — lane ", + f"origin/{branch}", "--", "lanes/log") + last: dict = {} + for line in proc.stdout.splitlines(): + _ref, _, rest = line.partition(":") + fname, _, text = rest.partition(":") + last[os.path.basename(fname)[:-3]] = text.split(" ", 1)[0] + for lane, verb in last.items(): + if verb in ("ENDED", "RETIRED"): + closed.setdefault(lane, f"its log ends {verb}") + return closed + + def _orphans(self, closed: dict) -> None: + rows = [] + seen = set() + for d in self.state_dirs: + for lane in sorted(closed): + tdir = os.path.join(d, lane, "trees") + try: + files = sorted(os.listdir(tdir)) + except OSError: + continue + for f in files: + path = _sidecar(os.path.join(tdir, f)).get("path", "") + if path and os.path.isdir(path) and real(path) not in seen: + seen.add(real(path)) + rows.append([path, lane, closed[lane], "lane-worktrees sweep " + lane]) + for lane, lroot in [(ln, r) for ln in sorted(closed) for r in self.tree_roots]: + ldir = os.path.join(lroot, lane) + try: + names = sorted(os.listdir(ldir)) + except OSError: + continue + for n in names: + full = os.path.join(ldir, n) + if os.path.isdir(full) and real(full) not in seen: + seen.add(real(full)) + rows.append([full, lane, closed[lane], f"lane-worktrees sweep {lane} " + "--include-scratch --include-foreign --word \"\""]) + self.add("Orphaned worktrees of ENDED lanes", rows, "none") + + def _branches(self, canonical: list) -> None: + rows = [] + fmt = ("%(refname:short)%1f%(objectname:short=12)%1f%(upstream:short)%1f" + "%(upstream:track)%1f%(committerdate:iso-strict)%1f%(contents:trailers)%00") + for c in canonical: + proc = git(c, "for-each-ref", f"--format={fmt}", "refs/heads/") + if proc.returncode != 0: + continue + held = {r.get("branch", "")[len("refs/heads/"):] + for r in worktree_registrations(c) if r.get("branch")} + for rec in proc.stdout.split("\0"): + cols = rec.strip("\n").split("\x1f") + if len(cols) < 6: + continue + name, tip, up, track, date, trailers = cols[:6] + if name in PROTECTED_BRANCHES or name.startswith("rescue/"): + continue + if not up: + state = "no upstream" + elif "gone" in track: + state = f"upstream {up} gone" + elif "ahead" in track and "behind" in track: + state = f"diverged from {up} {track}" + elif "ahead" in track: + state = f"unpushed {track}" + else: + continue + owner = (re.findall(r"^Lane:\s*(\S+)", trailers, re.MULTILINE) or ["-"])[-1] + rows.append([f"{c} {name}", tip, state, f"{_age_days(date)} d", + owner, "held by a worktree" if name in held else ""]) + rows.sort(key=lambda r: -int(r[3].split()[0])) + self.add("Unmerged branches with a missing, diverged or unpushed upstream", rows, + "none", "Listed, never deleted: a person pushes each to rescue//... " + "or gives a word.") + + def _root_main(self, canonical: list) -> None: + rows = [] + for c in canonical: + if git(c, "rev-parse", "-q", "--verify", "refs/heads/main").returncode != 0 or \ + git(c, "rev-parse", "-q", "--verify", "refs/remotes/origin/main").returncode != 0: + continue + n = git_out(c, "rev-list", "--count", "refs/remotes/origin/main..refs/heads/main") or "0" + if not n.isdigit() or int(n) == 0: + continue + paths = (git_out(c, "diff", "--name-only", "refs/remotes/origin/main...refs/heads/main") + or "").splitlines() + only = bool(paths) and all(p.startswith(HANDOFF_PATHS) for p in paths) + rows.append([c, f"local main ahead of origin by {n}", + "only handoff/register paths" if only else f"{len(paths)} path(s)", + "Amendment 4: handoffs and the register live in the workspace " + "repository and are pushed per write" if only else + "push it through a pull request, or a person's word"]) + self.add("Root main divergence", rows, "none") + + def _aging(self, canonical: list) -> None: + aging = self.conf["aging_days"] + rows = [] + rescue_count = 0 + seen = set() + for c in canonical: + proc = git(c, "for-each-ref", "--format=%(refname)\t%(committerdate:iso-strict)", + "refs/heads/rescue/", "refs/remotes/origin/rescue/") + for line in proc.stdout.splitlines(): + ref, _, date = line.partition("\t") + name = ref.split("/rescue/", 1)[1] + if (c, name) in seen: + continue + seen.add((c, name)) + rescue_count += 1 + age = _age_days(date) + if age >= aging: + first = name.split("/", 1)[0] if "/" in name else "" + owner = first if LANE_SHAPED_RE.match(first) else "-" + rows.append([f"{c} rescue/{name}", f"rescue branch, {age} d", + owner, "awaiting disposition"]) + for d in self.state_dirs: + for lane in sorted(os.listdir(d)): + tdir = os.path.join(d, lane, "trees") + try: + files = sorted(os.listdir(tdir)) + except OSError: + continue + for f in files: + path = _sidecar(os.path.join(tdir, f)).get("path", "") + if not path or not os.path.isdir(path): + continue + dirt = git_out(path, "status", "--porcelain") + if not dirt: + continue + newest = _gitdir_activity(path) + age = int((self.now - newest) // 86400) if newest else 0 + if age >= aging: + rows.append([path, f"dirty inventory tree, {age} d", lane, + "awaiting disposition"]) + self.rescue_count = rescue_count + self.add(f"Awaiting disposition (older than {aging} days)", rows, "none") + + def _caches_and_sandboxes(self) -> None: + rows = [] + roots = [] + for lroot in getattr(self, "tree_roots", []): + try: + roots += [os.path.join(lroot, n) for n in sorted(os.listdir(lroot))] + except OSError: + pass + for c in getattr(self, "canonical", []): + wt = os.path.join(c, ".claude", "worktrees") + if os.path.isdir(wt): + roots.append(wt) + for root in roots: + for path, size in find_caches(root): + rows.append([path, "cache", size]) + for it in sandbox_items(ProcScan(), self.conf): + rows.append([it.path, "sandbox", it.size]) + rows.sort(key=lambda r: -r[2]) + total = sum(r[2] for r in rows) + self.add(f"Caches and sandboxes ({human_size(total)} in all)", + [[p, k, human_size(s)] for p, k, s in rows], "none", + "Removed without archiving by `--include-caches` / `--include-sandboxes`.") + + def _ignored_and_evidence(self, repos: list, canonical: list) -> None: + threshold = self.conf["ignored_report_mb"] * 1024 * 1024 + big, evidence = [], [] + for r in repos: + proc = git(r, "ls-files", "--others", "--ignored", "--exclude-standard", + "--directory", "-z") + ignored = [p.rstrip("/") for p in proc.stdout.split("\0") if p.endswith("/")] + sizes = _du([os.path.join(r, p) for p in ignored]) if ignored else {} + for path, size in sizes.items(): + if size >= threshold: + big.append([path, human_size(size)]) + unt = git(r, "ls-files", "--others", "--exclude-standard", "--directory", "-z") + for p in (unt.stdout.split("\0") + [i + "/" for i in ignored]): + name = p.rstrip("/").rsplit("/", 1)[-1] + if p and EVIDENCE_NAME_RE.search(name): + evidence.append([os.path.join(r, p.rstrip("/")), "untracked or ignored in " + + os.path.basename(r)]) + if self.root: + repo_set = set(repos) + for n in sorted(os.listdir(self.root)): + full = os.path.join(self.root, n) + if full in repo_set or not os.path.isdir(full) or n.startswith(".lane-") \ + or n.endswith("-worktrees"): + continue + if EVIDENCE_NAME_RE.search(n) or self._holds_reports(full): + evidence.append([full, "a directory of reports beside the repositories"]) + big.sort(key=lambda r: r[0]) + self.add(f"Ignored directories over {self.conf['ignored_report_mb']} MB", big, "none") + self.add("Evidence-shaped paths inside the estate", evidence, "none", + "One evidence root per repository: ${XDG_STATE_HOME:-$HOME/.local/state}/" + "openRepoTools/evidence//-/. Each is a finding to move.") + + @staticmethod + def _holds_reports(d: str) -> bool: + try: + for e in os.scandir(d): + if e.is_file() and EVIDENCE_NAME_RE.search(e.name): + return True + if e.is_dir(follow_symlinks=False) and not e.name.startswith("."): + for f in os.scandir(e.path): + if f.is_file() and EVIDENCE_NAME_RE.search(f.name): + return True + except OSError: + return False + return False + + def _archives(self) -> None: + """`sweep --expire`'s dry run, folded in: what WOULD expire, and what + is past retention but kept because its rescue left origin. Expiring + stays an act (`sweep --expire --yes`); the report changes nothing.""" + root = sweeps_dir() + keep = self.conf["retention_days"] + rows = [] + for adir, age, disp, why in expiry_rows(self.conf): + if disp == "expire": + rows.append([adir, f"{age} d", "would expire: lane-worktrees sweep --expire --yes"]) + elif age >= keep: + rows.append([adir, f"{age} d", "kept past retention: " + why]) + self.sweeps_size = dir_size(root) if os.path.isdir(root) else 0 + self.add(f"Sweep archives past retention ({keep} days)", rows, "none") + + def _workspace_hygiene(self) -> None: + rows = [] + wroot = self.helper.workspace_root() if self.helper.path else "" + if wroot: + try: + with open(os.path.join(wroot, ".gitignore"), encoding="utf-8") as fh: + have = set(fh.read().splitlines()) + except OSError: + have = set() + want = ["__pycache__/", "*.pyc", "*.pyo", ".pytest_cache/", ".mypy_cache/", + ".ruff_cache/", "node_modules/", ".venv/", "venv/"] + missing = [w for w in want if w not in have] + if missing: + rows.append([os.path.join(wroot, ".gitignore"), "lacks " + " ".join(missing), + "printf '%s\\n' " + " ".join(f"'{m}'" for m in missing) + + f" >> {wroot}/.gitignore"]) + hist = git(wroot, "log", "--all", "--format=", "--name-only", "--", + "*.pyc", "*__pycache__*") + paths = sorted({p for p in hist.stdout.splitlines() if p}) + if paths: + rows.append([wroot, f"{len(paths)} bytecode path(s) in history, e.g. {paths[0]}", + "a person's word: history is rewritten only on one"]) + self.add("Workspace repository hygiene", rows, "none") + + def _status_all(self) -> None: + # `LANE_WORKTREES_STATUS` names the estate command when it is not + # beside this one (a test names a fake; a person, their own build). + exe = os.environ.get("LANE_WORKTREES_STATUS", "") + if not exe: + exe = os.path.join(os.path.dirname(os.path.realpath(__file__)), "status") + if not os.access(exe, os.X_OK): + exe = shutil.which("status") or "" + rows = [] + why = "" + if not exe: + why = "no `status` beside this command or on PATH" + else: + env = dict(os.environ) + if self.opts.estate: + env["PROJECTS_DIR"] = self.opts.estate + proc = run([exe, "--all"], env=env, timeout=600) + if proc.returncode not in (0, 1): + why = f"`status --all` exited {proc.returncode}: " + _first_line(proc.stderr) + context = "" + for line in proc.stdout.splitlines(): + s = line.strip() + if line.startswith("=== "): + context = s.strip("= ").replace("status ", "") + elif s.startswith("- "): + rows.append([context, s[2:]]) + self.add("`status --all` findings (pin drift and tracking are reported as what they " + "are, never counted as dirt)", rows, "none", why) + + # ------------------------------------------------------------ writing + def markdown(self) -> str: + host = os.environ.get("LANES_WORKSTATION") or socket.gethostname() + out = [f"# Estate report - {host} - {utc_now()}", "", + f"`{PROG} sweep --all --dry-run --report` over `{self.root or '(no estate)'}`. " + "It changed nothing.", "", + "| finding | count |", "|---|---|"] + for title, rows, _e, _n in self.sections: + out.append(f"| {title} | {len(rows)} |") + out.append(f"| rescue branches (all ages) | {getattr(self, 'rescue_count', 0)} |") + out.append(f"| the sweeps directory | {human_size(getattr(self, 'sweeps_size', 0))} |") + for title, rows, empty, note_text in self.sections: + out += ["", f"## {title}", ""] + if note_text: + out += [note_text, ""] + if not rows: + out.append(f"_{empty}_") + continue + for r in rows[:500]: + out.append("- " + " · ".join(str(c).replace("\n", " ") for c in r if c != "")) + if len(rows) > 500: + out.append(f"- ... and {len(rows) - 500} more") + return "\n".join(out) + "\n" + + +def report_all(opts) -> int: + rep = Report(opts) + rep.build() + text = rep.markdown() + post = opts.post or "" + if not post: + sys.stdout.write(text) + return 0 + m = re.match(r"^([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)#(\d+)$", post) + if m: + fd, tmp = tempfile.mkstemp(prefix="lane-worktrees-report.", suffix=".md") + with os.fdopen(fd, "w", encoding="utf-8") as fh: + fh.write(text) + try: + proc = run(["gh", "issue", "comment", m.group(2), "--repo", m.group(1), + "--body-file", tmp], timeout=120) + finally: + os.unlink(tmp) + if proc.returncode != 0: + note(f"the report could not be posted to {post}: " + _first_line(proc.stderr)) + sys.stdout.write(text) + return 1 + say(f"{PROG}: the report was posted to {post}: {proc.stdout.strip()}") + return 0 + try: + os.makedirs(os.path.dirname(os.path.abspath(post)), exist_ok=True) + with open(post, "w", encoding="utf-8") as fh: + fh.write(text) + except OSError as exc: + note(f"the report could not be written to {post}: {exc}") + sys.stdout.write(text) + return 1 + say(f"{PROG}: the report is at {post}") + return 0 + + # ==================================================================== main class _Parser(argparse.ArgumentParser): @@ -2501,6 +3053,9 @@ def parse(argv: list): parser.add_argument("--fetch", action="store_true") parser.add_argument("--estate") parser.add_argument("--expire", action="store_true") + parser.add_argument("--all", action="store_true") + parser.add_argument("--report", action="store_true") + parser.add_argument("--post") parser.add_argument("-h", "--help", action="store_true") parser.add_argument("--version", action="store_true") return parser.parse_args(argv) @@ -2521,6 +3076,12 @@ def main(argv: list) -> int: if ns.verb != "sweep": note("usage: lane-worktrees sweep [--dry-run | --yes] ... (see --help)") return 64 + if ns.all or ns.report or ns.post is not None: + if not (ns.all and ns.report) or ns.lane or ns.yes or ns.expire: + note("usage: the estate-wide sweep only reports: lane-worktrees sweep --all " + "--dry-run --report [--post ]") + return 64 + return report_all(ns) if ns.expire: if ns.lane: note("usage: sweep --expire takes no lane: it reads every lane's archives") diff --git a/tests/test_lane_worktrees_report.py b/tests/test_lane_worktrees_report.py new file mode 100644 index 0000000..bb0e43d --- /dev/null +++ b/tests/test_lane_worktrees_report.py @@ -0,0 +1,366 @@ +# SPDX-License-Identifier: Apache-2.0 +"""`lane-worktrees sweep --all --dry-run --report` and the start that runs it +once a day (opensoft/openRepoTools#162, the "nightly report" fold). + +The report is the net under every actor that never runs `lane-end`. Held here: + + * every section the issue names finds what it is for, in an estate built + under `tmp_path`: a FOREIGN clone (inside a worktree container, and a + second clone of one origin), an orphaned tree of an ENDED lane, unmerged + branches with a missing or diverged upstream, root-`main` divergence that + is handoff and register paths only, a rescue branch and a dirty inventory + tree past the aging threshold, caches and a killed suite's sandbox, + ignored directories over the threshold, evidence-shaped paths, archives + past retention, the workspace's `.gitignore` and its bytecode history, + and `status --all`'s findings; + * it changes nothing - the estate is snapshotted around it; + * `--post ` writes it, `--post owner/repo#n` comments it through `gh`; + * `lane-start` runs it detached, once a day per workstation, and a report + that is slow, failing or switched off neither delays nor fails a start. + +The `lane-start` cases reuse `tests/test_lane_start_claude_current.py`'s +sandbox: a real `lane-start` against a register in `tmp_path`. +""" + +from __future__ import annotations + +import datetime as _dt +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from conftest import REPO, WINDOWS_SKIP +import test_lane_worktrees as LW +import test_lane_start_claude_current as LS + +pytestmark = [WINDOWS_SKIP, pytest.mark.skipif( + shutil.which("git") is None or shutil.which("bash") is None, + reason="the report reads git, and lane-start is bash")] + +FAKE_STATUS = """#!/bin/sh +cat <<'OUT' +status --all: reading every estate: + +=== status repoA === + root main /x/repoA + - shape pin 71cf5de is 1 commit(s) behind opensoft/openRepoShape main +OUT +exit 1 +""" + + +def _commit_files(e, cwd, message, when, files): + """A commit whose author and committer dates are `when`.""" + for rel, text in files.items(): + path = Path(cwd) / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + e.git("add", "-A", cwd=cwd) + e.git("commit", "-q", "-m", message, cwd=cwd, + env={"GIT_COMMITTER_DATE": when, "GIT_AUTHOR_DATE": when}) + return e.git("rev-parse", "HEAD", cwd=cwd) + + +def build_estate(e) -> dict: + """An estate with one finding of every kind the report lists.""" + f = {} + # FOREIGN: a clone inside the lane's worktree root, and a second clone of + # repoA's origin beside the repositories. + f["clone"] = e.lane_root / "cloned" + e.git("clone", "-q", LW.GH_URL, f["clone"], cwd=e.root) + f["dup"] = e.projects / "repoA-copy" + e.git("clone", "-q", LW.GH_URL, f["dup"], cwd=e.root) + # ORPHANED: a tree in the inventory of a lane whose snapshot is CLOSED. + f["orphan"] = e.projects / ".lane-worktrees" / "repoA-9" / "left" + e.git("worktree", "add", "-q", "-b", "feat/left", f["orphan"], "origin/main") + closed = e.projects / ".lane-state" / "repoA-9" + (closed / "trees").mkdir(parents=True) + (closed / "lane-state.yaml").write_text("schema: 1\nstate: CLOSED\n") + (closed / "trees" / "c1.yaml").write_text(f"schema: 1\npath: {f['orphan']}\n") + # ... and one of a lane whose checkout is NESTED: #97 keeps its control + # root, and lane-start its worktree root, beside the checkout's own + # parent, `projects/group/`, never at the estate's top. + nested_origin = e.root / "remotes" / "repoB.git" + e.git("init", "-q", "--bare", "-b", "main", nested_origin, cwd=e.root) + e.git("clone", "-q", nested_origin, e.projects / "group" / "repoB", cwd=e.root) + group_closed = e.projects / "group" / ".lane-state" / "repoB-2" + group_closed.mkdir(parents=True) + (group_closed / "lane-state.yaml").write_text("schema: 1\nstate: CLOSED\n") + f["nested-orphan"] = e.projects / "group" / ".lane-worktrees" / "repoB-2" / "leftover" + f["nested-orphan"].mkdir(parents=True) + # UNMERGED: no upstream at all, and an upstream that diverged. + e.git("branch", "--no-track", "feat/nowhere", "origin/main") + e.git("checkout", "-q", "-b", "feat/diverged", "origin/main") + e.commit(e.checkout, "d1", {"d.txt": "1\n"}) + e.git("push", "-q", "-u", "origin", "feat/diverged") + e.git("reset", "-q", "--hard", "HEAD~1") + e.commit(e.checkout, "d2", {"d.txt": "2\n"}) + e.git("checkout", "-q", "main") + # ROOT MAIN DIVERGENCE: local main ahead by a handoff only. + e.commit(e.checkout, "handoff(repoA-1@Eagle): x", {"handoffs/repoA/h.md": "h\n"}) + # AGING: a rescue branch 30 days old, and a dirty inventory tree of a live + # lane whose git directory has been still for 30 days. + e.git("checkout", "-q", "-b", "rescue/repoA-1/old-20260101T000000Z", "origin/main") + _commit_files(e, e.checkout, "old rescue", (_dt.datetime.now(_dt.timezone.utc) + - _dt.timedelta(days=30)).strftime("%Y-%m-%dT%H:%M:%SZ"), {"r.txt": "r\n"}) + e.git("checkout", "-q", "main") + f["stale-dirty"] = e.lane_root / "stale" + e.git("worktree", "add", "-q", "-b", "feat/stale", f["stale-dirty"], "origin/main") + (f["stale-dirty"] / "wip.txt").write_text("unsaved\n") + live = e.projects / ".lane-state" / LW.LANE / "trees" + live.mkdir(parents=True) + (live / "s.yaml").write_text(f"schema: 1\npath: {f['stale-dirty']}\n") + old = time.time() - 30 * 86400 + gitdir = Path(e.git("rev-parse", "--absolute-git-dir", cwd=f["stale-dirty"])) + for name in ("index", "HEAD", "logs/HEAD"): + if (gitdir / name).exists(): + os.utime(gitdir / name, (old, old)) + # CACHES and a killed suite's SANDBOX. + (e.lane_root / "x-scratch" / "__pycache__").mkdir(parents=True) + (e.lane_root / "x-scratch" / "__pycache__" / "a.pyc").write_bytes(b"\0" * 2048) + f["sandbox"] = e.sandboxes / "tmp.killedrun1" + f["sandbox"].mkdir() + (f["sandbox"] / "f").write_text("x") + os.utime(f["sandbox"], (old, old)) + # IGNORED over the threshold (0 MB in this estate's sweep.conf), and + # EVIDENCE: an untracked JUnit file, and a directory of reports. + (e.checkout / "build").mkdir() + (e.checkout / "build" / "out.bin").write_bytes(b"\0" * 4096) + (e.checkout / ".gitignore").write_text((e.checkout / ".gitignore").read_text() + "build/\n") + (e.checkout / "junit-results.xml").write_text("\n") + f["reports"] = e.projects / "cleanup-20261005" + f["reports"].mkdir() + (f["reports"] / "FINAL-REPORT.md").write_text("# report\n") + # PAST RETENTION: an archive 120 days old that would expire, and one 150 + # days old kept because the rescue branch it records left origin. + f["archive"] = LW._archive(e, 120, None) + f["kept"] = LW._archive(e, 150, "rescue/repoA-1/gone-1") + # THE WORKSPACE: a .gitignore without the bytecode lines, and a .pyc in + # its history. + wip_origin = e.root / "wip.git" + wip = e.projects / "wip" + e.git("init", "-q", "--bare", "-b", "main", wip_origin, cwd=e.root) + e.git("clone", "-q", wip_origin, wip, cwd=e.root) + (wip / "lanes" / "log").mkdir(parents=True) + (wip / "lanes" / "LANES.md").write_text("# register\n") + (wip / "lanes" / "log" / "repoA-9.md").write_text( + f"STARTED — lane repoA-9, session {LW.ME}@Eagle, 2026-09-01T00:00:00Z, lane:repoA-9\n" + f"ENDED — lane repoA-9, session {LW.ME}@Eagle, 2026-09-02T00:00:00Z, lane:repoA-9\n") + (wip / ".gitignore").write_text("*.swp\n") + (wip / "handoffs" / "a" / "__pycache__").mkdir(parents=True) + (wip / "handoffs" / "a" / "__pycache__" / "m.cpython-312.pyc").write_bytes(b"\0") + e.git("add", "-A", cwd=wip) + e.git("commit", "-q", "-m", "seed", cwd=wip) + e.git("push", "-q", "origin", "main", cwd=wip) + e.workspace = str(wip) + f["wip"] = wip + (e.fakebin / "status").write_text(FAKE_STATUS) + (e.fakebin / "status").chmod(0o755) + e.env["LANE_WORKTREES_STATUS"] = str(e.fakebin / "status") + conf = e.root / "sweep.conf" + conf.write_text("ignored_report_mb=0\n") + e.env["LANE_WORKTREES_CONF"] = str(conf) + return f + + +def report(e, *args, env=None): + e.write_spec() + child = dict(e.env) + child.update(env or {}) + return subprocess.run([sys.executable, str(LW.LW), "sweep", "--all", "--dry-run", "--report", + "--estate", str(e.projects), *args], + capture_output=True, text=True, env=child, timeout=300, + cwd=str(e.root), stdin=subprocess.DEVNULL) + + +def section(text: str, title: str) -> str: + start = text.index(f"## {title}") + end = text.find("\n## ", start + 3) + return text[start:end if end != -1 else len(text)] + + +def test_the_report_finds_every_kind_of_leftover_and_changes_nothing(tmp_path): + e = LW.Estate(tmp_path) + f = build_estate(e) + before = LW.snapshot(e.root, skip=("helper.log", "helper-spec.json", "prs.json")) + proc = report(e) + after = LW.snapshot(e.root, skip=("helper.log", "helper-spec.json", "prs.json")) + assert proc.returncode == 0, proc.stdout + proc.stderr + out = proc.stdout + assert out.startswith("# Estate report") + foreign = section(out, "FOREIGN repositories") + assert str(f["clone"]) in foreign and "a CLONE inside" in foreign + assert str(f["dup"]) in foreign and "a second clone of" in foreign + assert f"{e.checkout} ·" not in foreign, "the estate's own checkout is not foreign" + orphans = section(out, "Orphaned worktrees of ENDED lanes") + assert str(f["orphan"]) in orphans and "repoA-9" in orphans + assert f"{f['nested-orphan']} \u00b7 repoB-2 \u00b7 its snapshot is CLOSED" in orphans + branches = section(out, "Unmerged branches") + assert "feat/nowhere" in branches and "no upstream" in branches + assert "feat/diverged" in branches and "diverged from origin/feat/diverged" in branches + assert "rescue/repoA-1/old" not in branches + root_main = section(out, "Root main divergence") + assert "ahead of origin by 1" in root_main and "only handoff/register paths" in root_main + aging = section(out, "Awaiting disposition") + assert "rescue/repoA-1/old-20260101T000000Z \u00b7 rescue branch, 30 d \u00b7 repoA-1" in aging + assert str(f["stale-dirty"]) in aging and "dirty inventory tree" in aging + caches = section(out, "Caches and sandboxes") + assert "x-scratch/__pycache__" in caches and str(f["sandbox"]) in caches + ignored = section(out, "Ignored directories over") + assert str(e.checkout / "build") in ignored + evidence = section(out, "Evidence-shaped paths") + assert "junit-results.xml" in evidence and str(f["reports"]) in evidence + archives = section(out, "Sweep archives past retention") + assert f"{f['archive']} \u00b7 120 d \u00b7 would expire" in archives + assert f"{f['kept']} \u00b7 150 d \u00b7 kept past retention" in archives + assert "gone from origin" in archives + hygiene = section(out, "Workspace repository hygiene") + assert "lacks __pycache__/" in hygiene and "bytecode path(s) in history" in hygiene + status = section(out, "`status --all` findings") + assert "shape pin 71cf5de is 1 commit(s) behind" in status + assert "| rescue branches (all ages) | 1 |" in out + assert before == after, sorted(set(before.items()) ^ set(after.items()))[:10] + + +def test_post_writes_a_file_or_comments_on_an_issue(tmp_path): + e = LW.Estate(tmp_path) + build_estate(e) + target = e.state / "openRepoTools" / "reports" / "today.md" + proc = report(e, "--post", str(target)) + assert proc.returncode == 0, proc.stderr + assert target.read_text().startswith("# Estate report") + log = e.root / "gh.log" + (e.fakebin / "gh").write_text( + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$FAKE_GH_LOG\"\n" + "[ \"$1 $2\" = 'issue comment' ] && { cat \"$7\" >> \"$FAKE_GH_LOG\"; " + "echo https://github.com/opensoft/repoA/issues/9#c1; exit 0; }\nexit 1\n") + proc = report(e, "--post", "opensoft/repoA#9", env={"FAKE_GH_LOG": str(log)}) + assert proc.returncode == 0, proc.stderr + text = log.read_text() + assert "issue comment 9 --repo opensoft/repoA --body-file" in text + assert "# Estate report" in text + + +def test_the_report_is_only_a_report(tmp_path): + e = LW.Estate(tmp_path) + for argv in (["--all", "--report", "--yes"], ["--all"], ["--report"], + [LW.LANE, "--all", "--report"]): + e.write_spec() + proc = subprocess.run([sys.executable, str(LW.LW), "sweep", *argv], capture_output=True, + text=True, env=e.env, timeout=60) + assert proc.returncode == 64, (argv, proc.stdout, proc.stderr) + + +# ================================================== lane-start runs it daily + +FAKE_REPORTER = """#!/bin/sh +printf '%s\\n' "$*" >> "$FAKE_REPORT_LOG" +if [ -n "${FAKE_REPORT_SLEEP:-}" ]; then + sleep "$FAKE_REPORT_SLEEP" & + echo "$!" > "$FAKE_REPORT_LOG.sleeper" + wait + echo finished >> "$FAKE_REPORT_LOG" +fi +exit "${FAKE_REPORT_RC:-0}" +""" + + +def _reporter(box, state: Path) -> Path: + log = box.root / "report.log" + LS._write(box.bin / "lane-worktrees", FAKE_REPORTER) + box.env.update(LANE_WORKTREES_REPORT="on", XDG_STATE_HOME=str(state), + FAKE_REPORT_LOG=str(log)) + return log + + +def _no_report_state(state: Path) -> bool: + """No stamp and no reports directory: the report's own state only - the + start writes other state of its own there (#164's index nudge).""" + base = state / "openRepoTools" + return not list(base.glob("report-*.stamp")) and not (base / "reports").exists() + + +def _wait_for(path: Path, timeout: float = 15) -> str: + deadline = time.time() + timeout + while time.time() < deadline: + if path.exists() and path.read_text(): + return path.read_text() + time.sleep(0.1) + return "" + + +def test_lane_start_runs_the_report_detached_once_a_day(tmp_path): + box = LS.Sandbox(tmp_path) + state = tmp_path / "state" + log = _reporter(box, state) + try: + result = box.start("--no-launch", FAKE_REPORT_SLEEP="90") + assert result.returncode == 0, result.stderr + calls = _wait_for(log) + # The report sleeps 90 s and then writes `finished`: a start that + # waited on it would have returned after that line. + assert "finished" not in calls, "the start waited on the report" + finally: + sleeper = Path(str(log) + ".sleeper") + if _wait_for(sleeper, 5).strip().isdigit(): + try: + os.kill(int(sleeper.read_text()), 15) + except OSError: + pass + assert calls.startswith("sweep --all --dry-run --report --post "), calls + assert f"{state}/openRepoTools/reports/" in calls + day = _dt.datetime.now(_dt.timezone.utc).strftime("%Y%m%d") + assert (state / "openRepoTools" / f"report-{day}.stamp").exists() + assert "the estate's daily report is running, detached" in result.stderr + + +def test_a_stamp_from_today_runs_no_second_report_and_yesterdays_is_replaced(tmp_path): + box = LS.Sandbox(tmp_path) + state = tmp_path / "state" + log = _reporter(box, state) + day = _dt.datetime.now(_dt.timezone.utc).strftime("%Y%m%d") + (state / "openRepoTools").mkdir(parents=True) + (state / "openRepoTools" / f"report-{day}.stamp").write_text("") + result = box.start("--no-launch") + assert result.returncode == 0, result.stderr + time.sleep(1) + assert not log.exists() or log.read_text() == "", "a second report ran today" + (state / "openRepoTools" / f"report-{day}.stamp").unlink() + (state / "openRepoTools" / "report-20200101.stamp").write_text("") + box2 = LS.Sandbox(tmp_path / "second") + log2 = _reporter(box2, state) + assert box2.start("--no-launch").returncode == 0 + assert _wait_for(log2) + assert not (state / "openRepoTools" / "report-20200101.stamp").exists() + + +def test_a_failing_or_switched_off_report_never_fails_a_start(tmp_path): + box = LS.Sandbox(tmp_path) + state = tmp_path / "state" + log = _reporter(box, state) + result = box.start("--no-launch", FAKE_REPORT_RC="1") + assert result.returncode == 0, result.stderr + assert _wait_for(log), "the failing reporter was never run" + off = LS.Sandbox(tmp_path / "off") + off_log = _reporter(off, tmp_path / "off-state") + result = off.start("--no-launch", LANE_WORKTREES_REPORT="off") + assert result.returncode == 0, result.stderr + time.sleep(1) + assert not off_log.exists() + assert _no_report_state(tmp_path / "off-state") + + +def test_a_dry_run_start_runs_no_report(tmp_path): + box = LS.Sandbox(tmp_path) + state = tmp_path / "state" + log = _reporter(box, state) + result = box.start("--dry-run") + assert result.returncode == 0, result.stderr + time.sleep(1) + assert not log.exists() + assert _no_report_state(state) From 38e090f9d1c9c9c0d6e98e6f31e159e5cc79d405 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:08:50 +0000 Subject: [PATCH 12/29] Document the estate report and prevention at creation The lanes manual gains the report (its sections, --post, the daily run from lane-start and its switch) and what keeps leftovers from arriving: the run.sh relocation, one venv per repository outside the estate, one evidence root per repository under the state directory, and the workspace's bytecode rules with lanes-edit.sh's refusal and pathspec-check. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- docs/README-lanes.md | 79 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/docs/README-lanes.md b/docs/README-lanes.md index 6beff9c..308f224 100644 --- a/docs/README-lanes.md +++ b/docs/README-lanes.md @@ -3468,6 +3468,85 @@ the table and in `DISPOSITION.md`); usage is 64. goes to `${XDG_STATE_HOME:-$HOME/.local/state}/openRepoTools/lanes//scratch/` or the harness scratchpad. +### The estate report — `sweep --all --dry-run --report` + +The net under every actor that never runs `lane-end`: **creation outpacing +closeout as a number that is read every day.** It reads every lane's #97 +inventory and snapshot under `/.lane-state` (or +`$LANES_LANE_STATE_ROOT`), the workspace repository's register on +`origin/`, and the disk — **never the derived index** (Amendment 14 +clause (b)) — and CHANGES NOTHING: no fetch, no index refresh +(`GIT_OPTIONAL_LOCKS=0`), no expiry. + +```sh +lane-worktrees sweep --all --dry-run --report [--estate ] [--post | --post #] +``` + +| section | what is listed | +|---|---| +| FOREIGN repositories | a CLONE inside a worktree container (`.lane-worktrees/`, `.claude/worktrees`, `worktrees`, `-worktrees`), and a second clone of an origin the estate already has a checkout of: path, size, last commit, owner (the last `Lane:` trailer), and LOAD-BEARING with its dependents where something leans on it | +| Orphaned worktrees of ENDED lanes | trees an inventory records, and directories under `.lane-worktrees/`, of a lane whose snapshot is `CLOSED` or whose log's last lane line is `ENDED` or `RETIRED` — with the sweep that retires them | +| Unmerged branches | local branches of every checkout with no upstream, a gone upstream, a diverged one, or unpushed commits: tip, state, age, owner, and whether a worktree holds it. Never deleted; `main`, `master` and `rescue/*` are not listed | +| Root main divergence | a checkout whose local `main` is ahead of `origin/main`; where every changed path is under `handoffs/` or `lanes/`, the remedy is Amendment 4's — those live in the workspace repository and are pushed per write | +| Awaiting disposition | a `rescue/*` branch (local or on origin) and a DIRTY inventory tree whose git directory has been still for `aging_days` (14), with owner and age | +| Caches and sandboxes | `--include-caches`' and `--include-sandboxes`' rows across every lane, by size | +| Ignored directories over `ignored_report_mb` | every ignored directory of every repository at or over 50 MB (`du -sk`) | +| Evidence-shaped paths | untracked or ignored `junit*.xml`, `MANIFEST*`, `*-report.md`, `*REPORT*.md`, `canary-*`, `*-evidence`, `deployment-evidence` inside a repository, and a directory of reports beside the repositories — each a finding to move to the evidence root below | +| Sweep archives past retention | `sweep --expire`'s own dry run: what WOULD expire, and what is past retention but kept because its rescue left origin. Expiring stays an act (`sweep --expire --yes`) | +| Workspace repository hygiene | the workspace's `.gitignore` lines it lacks (with the one command that adds them), and bytecode already in its history (for a person's word: history is rewritten only on one) | +| `status --all` findings | the estate command's ahead/behind, fork, shape-pin and parked-record lines, reported as what they are and never counted as dirt (`LANE_WORKTREES_STATUS` names the command when it is not beside this one) | + +The summary table at its head also carries **the count of rescue branches** and +**the sweeps directory's size**, so the cleanup's own footprint is watched. +`--post ` writes the report there; `--post /#` comments it +on that issue through `gh`; with no `--post` it is printed. `--all` and +`--report` come together, with no lane, no `--yes` and no `--expire` (usage, 64). + +**`lane-start` runs it once a day per workstation**, after the row is written +and before the launch: the first start of a UTC day takes the stamp +`${XDG_STATE_HOME:-~/.local/state}/openRepoTools/report-.stamp` +(an atomic `set -C` create, so two starts at once run one report), removes +older stamps, and starts the report DETACHED — stdin, stdout and stderr closed, +in a subshell that exits at once — writing +`…/openRepoTools/reports/.md`. Every step either works or is skipped in +silence: the report never delays a start and never fails one. `--dry-run` +starts none; `LANE_WORKTREES_REPORT=off` is the switch (both suites set it). + +### Nothing new lands beside the code + +The sweep retires what is already there; these keep more from arriving. + +* **`tests/run.sh`** sends bytecode to + `${XDG_CACHE_HOME:-~/.cache}/openRepoTools/pycache`, turns pytest's cache off, + and roots every temporary directory of a run under + `${XDG_STATE_HOME:-~/.local/state}/openRepoTools/tmp/-/`, which its + EXIT trap removes however the run ends. A SIGKILL is the one end no trap sees; + the pid in the name is what lets `--include-sandboxes` tell its owner is gone. +* **One virtual environment per repository, outside the estate:** + `${XDG_CACHE_HOME:-~/.cache}/openRepoTools/venvs//`. `openRepoTools + --install` names this repository's and says whether it is there; `tests/run.sh` + runs from it when it has `pytest`. A `venv/` or `.venv/` inside a worktree is + a legacy leftover for `--include-caches`. +* **One evidence root per repository:** + `${XDG_STATE_HOME:-~/.local/state}/openRepoTools/evidence//-/` + for JUnit files, manifests, reports, canary and deployment evidence — never + inside a checkout and never beside the repositories. The report lists what is + still found inside the estate. +* **The workspace repository never carries bytecode.** `openRepoTools wip init` + seeds `__pycache__/`, `*.pyc`, `*.pyo`, `.pytest_cache/`, `.mypy_cache/`, + `.ruff_cache/`, `node_modules/`, `.venv/` and `venv/` into the new + workspace's `.gitignore` (a line the template already carries is not + repeated). Every commit `lanes-edit.sh` makes asks git what its pathspec would + stage (`git add --dry-run`) and REFUSES, exit **2**, nothing staged, when any + of it is bytecode, a cache, a dependency tree or a virtual environment — and + offers the `.gitignore` lines a workspace without them lacks. Attachments are + committed by hand, so the same question is a subcommand: + + ```sh + lanes-edit.sh pathspec-check handoffs//attachments/ + # 0 clean · 2 the offending paths on stdout, the offer on stderr · 64 usage + ``` + ## Hand edits After **any** hand edit made with an allowed tool (python read/write, `sed -i From bf066fdb2667d9114be3b891fc2d99cbef6a6e0c Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:22:52 +0000 Subject: [PATCH 13/29] Take Copilot round 1's defects in the sweep Every one of these let a destructive act rest on a read that was stale, failed, or answered a different question: - Merged now needs the PR's base to be the default branch (origin's HEAD, else main/master): a merge into a release branch is not a landing. An OPEN PR protects its branch before any older merged PR on that branch counts, unless the register LANDED that very PR. - A failed `rev-list` distance is a read error, never "0 ahead", for a tree and for --branches' upstream state alike. - A clone whose git directory linked worktrees share is kept. Removing it would take their repository with it. - A tree that another lane's inventory also names is kept. An inventory record that cannot be read refuses the sweep (exit 2) instead of being skipped. The lane name goes through `canon-lane` before any path is derived from it, and an unreadable alias table refuses. - Liveness is asked again before the rescue and again just before removal, afresh (processes, tmux, the recording session). A scan that cannot be made leaves the tree. Scratch is removed only if no writer arrived and nothing outside its caches changed while the tar ran. A cache is rechecked against its owning tree. - Caches never come from a FOREIGN tree, a live one, or one whose liveness is unknown. - Branches are deleted at the SHA that was judged: `update-ref -d ` locally, a --force-with-lease push for the remote. Each failure is counted, and the register claims "+ remote" only for a remote delete that happened. - --branches fetches the lane's own checkout too, and a failed fetch deletes nothing. - A stale registration is removed by `git worktree remove ` alone, never by a repository-wide prune. - An ignored-file listing that failed stops the act. It is never read as "nothing ignored". - Each invocation gets its own archive directory, created exclusively. - An origin-less worktree under --bundle --yes is bundled and removed, as its dry run says, instead of being read as a failed fetch. - --expire requires the rescued SHA itself on origin and an archive that is whole: a MANIFEST.sha256 listing every file at its digest, and a rescues.tsv whose rows parse. Ten new or extended cases. Each one fails against the previous head and passes here. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lane-worktrees | 472 +++++++++++++++++++++++++++++------ tests/test_lane_worktrees.py | 187 +++++++++++++- 2 files changed, 571 insertions(+), 88 deletions(-) diff --git a/lane-worktrees b/lane-worktrees index 211f5f9..5036229 100755 --- a/lane-worktrees +++ b/lane-worktrees @@ -44,7 +44,9 @@ The sweeps directory: ${XDG_STATE_HOME:-$HOME/.local/state}/openRepoTools/ sweeps/// with MANIFEST.sha256, DISPOSITION.md and rescues.tsv. `sweep --expire` lists (and with `--yes` removes) archives older than the retention (90 days; `sweep.conf`) whose every rescue branch is still on -origin; an archive whose rescue branch is gone from origin is never expired. +origin at the rescued SHA; an archive whose rescue branch is gone or moved, +or that is not whole (its MANIFEST.sha256 missing or not matching, its +rescues.tsv unreadable), is never expired. Exit codes: 0 dry run: nothing to retire · --yes: every act completed @@ -387,6 +389,24 @@ class Helper: self._fetched = True return proc + def canon_lane(self, lane: str): + """(canonical spelling, refusal) - Amendment 15's resolver, as + `lane-end` reads it: 0 with the row's spelling; 66 the alias table + unreadable and 2 with 15(d)'s words a published case pair, both + refusals; any other answer (a helper predating the subcommand) keeps + the typed spelling.""" + proc = self.call("canon-lane", lane, fetch=True) + out = proc.stdout.strip() + if proc.returncode == 0 and out: + return out, "" + if proc.returncode == 66: + return lane, ("the lane alias table could not be read (exit 66), so a renamed " + f"lane's old name cannot be resolved: {_first_line(proc.stderr)}") + if proc.returncode == 2 and "Amendment 15(d)" in (proc.stderr or ""): + return lane, ("the published register spells this lane two ways (Amendment " + f"15(d)): {_first_line(proc.stderr)}") + return lane, "" + def lane_facts(self, lane: str) -> LaneFacts: facts = LaneFacts(lane) proc = self.call("lane-reconcile", lane, fetch=True) @@ -486,11 +506,12 @@ class Helper: proc = self.call("workspace-root") return proc.stdout.strip() if proc.returncode == 0 else "" - def transcript_live(self, uuid: str) -> str: + def transcript_live(self, uuid: str, fresh: bool = False) -> str: """live | none | unknown - one live process carrying that transcript, - asked once per session however many trees it recorded.""" + asked once per session however many trees it recorded (`fresh` asks + again: at the moment of an act the cached answer is minutes old).""" cache = self.__dict__.setdefault("_live", {}) - if uuid not in cache: + if fresh or uuid not in cache: proc = self.call("transcript-holders", uuid) cache[uuid] = {0: "live", 8: "none"}.get(proc.returncode, "unknown") return cache[uuid] @@ -700,7 +721,8 @@ class MergeEvidence: return self._prs[slug] env = dict(os.environ, GH_PROMPT_DISABLED="1", GH_NO_UPDATE_NOTIFIER="1") proc = run(["gh", "pr", "list", "--repo", slug, "--state", "all", "--limit", "1000", - "--json", "number,state,mergedAt,headRefName,headRefOid,isCrossRepository"], + "--json", "number,state,mergedAt,headRefName,headRefOid,baseRefName," + "isCrossRepository"], env=env, timeout=120) rows = None if proc.returncode == 0: @@ -713,9 +735,18 @@ class MergeEvidence: self._prs[slug] = rows return rows + @staticmethod + def default_branches(repo: str) -> set: + """The branch a landing goes into: origin's HEAD where this clone + knows it, else `main` or `master`.""" + head = git_out(repo, "symbolic-ref", "-q", "--short", "refs/remotes/origin/HEAD") or "" + if head.startswith("origin/") and len(head) > len("origin/"): + return {head[len("origin/"):]} + return set(PROTECTED_BRANCHES) + def judge(self, repo: str, slug: str, branch: str, tip: str) -> dict: - """{'state': merged|open|closed|none|unknown, 'pr': n, 'source': ..., - 'why': ...} for `branch` at `tip`.""" + """{'state': merged|moved|elsewhere|open|closed|none|unknown, 'pr': n, + 'source': ..., 'why': ...} for `branch` at `tip`.""" rows = self.prs(slug) if rows is None: return {"state": "unknown", "pr": 0, "source": "", @@ -726,11 +757,29 @@ class MergeEvidence: if not mine: return {"state": "none", "pr": 0, "source": "gh", "why": "no pull request names it"} landed = self.landed() + # AN OPEN PULL REQUEST PROTECTS ITS BRANCH BEFORE ANYTHING ELSE: an + # older merged PR on the same branch says nothing about the work the + # open one still carries - unless the register has LANDED that very PR. + for row in sorted(mine, key=lambda r: -int(r.get("number", 0))): + if row.get("state") == "OPEN" and (slug.lower(), int(row.get("number", 0))) not in landed: + return {"state": "open", "pr": int(row["number"]), "source": "gh", + "why": f"PR #{row['number']} is open"} + defaults = self.default_branches(repo) + elsewhere = None for row in sorted(mine, key=lambda r: -int(r.get("number", 0))): number = int(row.get("number", 0)) by_register = (slug.lower(), number) in landed by_gh = row.get("state") == "MERGED" if by_register or by_gh: + # A MERGE INTO A RELEASE OR FEATURE BRANCH IS NOT A LANDING: only + # a PR whose base is the default branch makes its branch merged. + base = row.get("baseRefName") or "" + if base not in defaults: + elsewhere = elsewhere or { + "state": "elsewhere", "pr": number, "source": "gh", + "why": f"PR #{number} was merged into {base or 'a base gh did not name'}, " + f"not {'/'.join(sorted(defaults))}: not a landing"} + continue head = row.get("headRefOid") or "" contained = bool(tip) and (tip == head or (bool(head) and git( repo, "merge-base", "--is-ancestor", tip, head).returncode == 0)) @@ -741,10 +790,8 @@ class MergeEvidence: return {"state": "moved", "pr": number, "source": source, "head": head, "why": f"PR #{number} is merged ({source}) but this branch has moved " f"past its head {short(head)}"} - for row in mine: - if row.get("state") == "OPEN": - return {"state": "open", "pr": int(row["number"]), "source": "gh", - "why": f"PR #{row['number']} is open"} + if elsewhere: + return elsewhere return {"state": "closed", "pr": int(mine[0].get("number", 0)), "source": "gh", "why": f"PR #{mine[0].get('number')} was closed unmerged"} @@ -933,6 +980,7 @@ class Tree: self.bundle = "" self.ignored_archive = "" self.register = "" + self.contested = False # another lane's inventory names it too @property def slice(self) -> str: @@ -1032,8 +1080,14 @@ def read_tree(tree: Tree) -> None: if tree.remote_tip: counts = git_out(tree.path, "rev-list", "--left-right", "--count", f"{tree.remote_tip}...{tree.head}") - if counts and len(counts.split()) == 2: - tree.behind, tree.ahead = (int(x) for x in counts.split()) + parts = (counts or "").split() + if len(parts) != 2 or not all(x.isdigit() for x in parts): + # NEVER ZERO FOR UNKNOWN: "0 ahead" is what lets a tree go + # without a rescue. + tree.read_error = (f"its distance from origin/{tree.remote_branch} could not " + "be read (git rev-list failed)") + return + tree.behind, tree.ahead = int(parts[0]), int(parts[1]) contains = git(tree.path, "for-each-ref", "--count=1", "--contains", tree.head, "--format=%(refname)", "refs/remotes/origin/") if contains.returncode == 0: @@ -1080,6 +1134,18 @@ def tree_guards(tree: Tree) -> str: return ("its worktree registration is LOCKED - someone pinned it on purpose; " f"`git -C {tree.checkout or tree.path} worktree unlock {tree.path}` is theirs to run") if tree.kind == "clone": + # A CLONE TAKES ITS LINKED WORKTREES' REPOSITORY WITH IT: a worktree + # elsewhere - perhaps a live writer's - keeps its files and loses its + # git directory. + wl = git(tree.path, "worktree", "list", "--porcelain") + if wl.returncode != 0: + return "its worktree registrations could not be read" + linked = [line[len("worktree "):] for line in wl.stdout.splitlines() + if line.startswith("worktree ")][1:] + if linked: + return (f"it is a CLONE whose git directory {len(linked)} linked worktree(s) share " + f"(e.g. {linked[0]}); its removal would take their repository with it - " + "remove or move them first") # A CLONE TAKES ITS BRANCHES AND ITS STASH WITH IT, which a worktree # never does: every one of them must be on origin first. heads = git(tree.path, "for-each-ref", "--format=%(refname:short) %(objectname)", @@ -1132,13 +1198,14 @@ def tree_guards(tree: Tree) -> str: return "" -def ignored_work(tree: Tree) -> list: +def ignored_work(tree: Tree): """Ignored paths that are NOT caches: a commit cannot carry them, so they - are archived beside the bundle before the tree goes.""" + are archived beside the bundle before the tree goes. None where git could + not list them - which is never read as "there are none".""" proc = git(tree.path, "ls-files", "--others", "--ignored", "--exclude-standard", "--directory", "-z") if proc.returncode != 0: - return [] + return None out = [] for rel in proc.stdout.split("\0"): rel = rel.rstrip("/") @@ -1225,6 +1292,17 @@ class Sweep: self.refusal = ("no lanes-edit.sh beside this command or on PATH: which trees " "are this lane's is #97's inventory, and nothing else answers it") return + # THE CANONICAL SPELLING BEFORE ANY PATH IS DERIVED FROM IT: scratch, + # caches, rescue names and the Lane: trailer comparison all use it. + canon, why = self.helper.canon_lane(self.lane) + if why: + self.refusal = f"lane {self.lane}'s name could not be resolved: {why}" + return + if canon != self.lane: + self.notes.append(f"lane {self.lane} -> {canon} (Amendment 15: the register row's " + "spelling is canonical)") + self.lane = canon + self.facts = LaneFacts(canon) self.facts = self.helper.lane_facts(self.lane) f = self.facts if f.why: @@ -1278,10 +1356,13 @@ class Sweep: for rec in self.facts.inventory: path = rec.get("path") or "" if not path or rec.get("schema") not in ("1",): - if rec.get("id"): - self.notes.append(f"inventory record {rec.get('id')} is unreadable or of " - f"schema {rec.get('schema') or '?'}; nothing of it is read " - "and it is never swept") + # A RECORD THAT CANNOT BE READ IS A TREE WHOSE OWNER IS UNKNOWN: + # skipping it would let the table say "nothing to retire", and + # --include-foreign find the same path as nobody's. + self.refusal = self.refusal or ( + f"inventory record {rec.get('id') or '?'} is unreadable or of schema " + f"{rec.get('schema') or '?'}, so which trees are this lane's is not known " + "(Amendment 7(d)); read it: lanes-edit.sh lane-trees " + self.lane) continue if lane_dir and real(path) == lane_dir: continue @@ -1360,14 +1441,33 @@ class Sweep: except OSError: continue for tree in self.trees: - if tree.owner == "foreign" and tree.real in claimed: + if tree.real not in claimed: + continue + tree.other_lane = claimed[tree.real] + if tree.owner == "foreign": tree.foreign_kind = "other-lane" - tree.other_lane = claimed[tree.real] + else: + # BOTH INVENTORIES NAME IT: records are history and a path can + # be reused, so which lane owns it now is not this sweep's to + # decide. + tree.contested = True + + def _branch_checkouts(self) -> list: + checkouts = [] + if self.facts.dir and os.path.isdir(self.facts.dir): + checkouts.append(self.facts.dir) + for t in self.trees: + if t.owner == "inventory" and t.checkout and os.path.isdir(t.checkout) \ + and real(t.checkout) not in [real(c) for c in checkouts]: + checkouts.append(t.checkout) + return checkouts def _fetch_all(self) -> None: commons: dict = {} - for tree in self.trees: - probe = tree.path if tree.exists else tree.checkout + probes = [t.path if t.exists else t.checkout for t in self.trees] + if self.opts.branches: + probes += self._branch_checkouts() + for probe in probes: if not probe or not os.path.isdir(probe): continue common = git_out(probe, "rev-parse", "--git-common-dir") @@ -1494,6 +1594,10 @@ class Sweep: if t.lean: t.reason += "; LOAD-BEARING: " + _lean_text(t.lean) return + if t.contested: + put("keep", f"lane {t.other_lane}'s inventory names it too; which lane owns it " + "now is a person's to say") + return if not t.exists: if t.registered: if t.locked: @@ -1528,7 +1632,7 @@ class Sweep: return if self.opts.fetch: state = self.fetched.get(real(t.common), "") - if state and state != "ok": + if state and state not in ("ok", "no origin remote"): put("keep", f"origin could not be read ({state}), so what is published is unknown") return if not t.origin_url: @@ -1562,7 +1666,8 @@ class Sweep: return if t.remote_tip and t.ahead == 0: put("remove", "clean, and every commit is on origin/" + t.remote_branch - + (f" ({m.get('why')})" if m.get("state") in ("open", "closed", "moved") else "")) + + (f" ({m.get('why')})" if m.get("state") in ("open", "closed", "moved", + "elsewhere") else "")) return if not t.remote_tip and t.on_origin: put("remove", "clean, and its head is on origin") @@ -1578,15 +1683,22 @@ class Sweep: + ": pushed as is, then removed") def _branches(self) -> None: - checkouts = [] - if self.facts.dir and os.path.isdir(self.facts.dir): - checkouts.append(self.facts.dir) - for t in self.trees: - if t.owner == "inventory" and t.checkout and os.path.isdir(t.checkout) \ - and real(t.checkout) not in [real(c) for c in checkouts]: - checkouts.append(t.checkout) - for co in checkouts: - self.items.extend(branch_items(co, self.lane, self.evidence)) + for co in self._branch_checkouts(): + items = branch_items(co, self.lane, self.evidence) + if self.opts.fetch: + common = git_out(co, "rev-parse", "--git-common-dir") or "" + if common and not os.path.isabs(common): + common = os.path.normpath(os.path.join(co, common)) + state = self.fetched.get(real(common), "") if common else "" + if state != "ok": + # A DELETE DECIDED ON STALE REFS IS NO DELETE: what origin + # holds now was not read. + for it in items: + if it.disposition in ("delete", "delete+remote"): + it.disposition = "keep" + it.reason = (f"origin could not be fetched in {co} " + f"({state or 'not fetched'}); " + it.reason) + self.items.extend(items) def _scratch(self) -> None: if not self.facts.dir: @@ -1633,24 +1745,53 @@ class Sweep: if t.owner == "inventory" and t.exists and t.is_git \ and not any(under(t.path, r) for r in roots): roots.append(t.path) - live_paths = [t.path for t in self.trees if t.disposition == "live"] + # NEVER FROM A TREE THAT IS NOT THIS LANE'S OR NOT PROVABLY FREE: a + # FOREIGN tree (whoever's it is, live or not), a live one, one whose + # liveness could not be read, one another lane also claims, one + # something leans on. A tree kept for its git state (a locked + # registration, a submodule's work) still gives up its caches: they + # hold no work, and its liveness is asked below. + protected = [t.path for t in self.trees if self._cache_protected(t)] # A CACHE INSIDE SCRATCH THAT IS BEING ARCHIVED GOES WITH THE SCRATCH # (and the archive leaves caches out): two acts on one path is one too many. archived = [i.path for i in self.items if i.kind == "scratch" and i.disposition == "archive+remove"] + trees = sorted((t.path for t in self.trees if t.exists), key=len, reverse=True) for root in roots: for path, size in find_caches(root): - if any(under(path, lp) for lp in live_paths + archived): + if any(under(path, lp) for lp in protected + archived): continue - if self.procs and self.procs.available and self.procs.holders(path): + owner = self._cache_owner(path, roots, trees) + # LIVENESS IS THE OWNER'S, not the cache's: a writer stands in + # the tree, not in its __pycache__. + if self.procs is None or not self.procs.available or self.procs.holders(owner): continue if tracked(path): continue it = Item("cache", path, "remove", "a dependency cache or bytecode: it holds no " "work and is rebuilt", False) it.size = size + it.extra["owner"] = owner self.items.append(it) + @staticmethod + def _cache_protected(t: Tree) -> bool: + return (t.disposition in ("foreign", "live", "load-bearing") or t.live is not None + or t.contested or t.result.startswith(("FAILED", "left"))) + + @staticmethod + def _cache_owner(path: str, roots: list, trees: list) -> str: + """The tree a cache is in, else the top directory under the lane's + root it is in, else the root.""" + for t in trees: + if under(path, t): + return t + for r in roots: + if under(path, r): + rel = os.path.relpath(path, r).split(os.sep) + return os.path.join(r, rel[0]) if rel and rel[0] not in ("", ".") else r + return path + # ------------------------------------------------------------ output def retire_count(self) -> int: return (sum(1 for t in self.trees if t.retire) @@ -1751,9 +1892,19 @@ class Sweep: self._write_disposition() def _archive_dir(self) -> str: + """This invocation's archive, created EXCLUSIVELY: two sweeps started + in one second never share - and overwrite - one ledger.""" if not self.archive: - path = os.path.join(sweeps_dir(), safe_ref_part(self.lane), self.stamp) - os.makedirs(path, mode=0o700, exist_ok=True) + parent = os.path.join(sweeps_dir(), safe_ref_part(self.lane)) + os.makedirs(parent, mode=0o700, exist_ok=True) + path, n = os.path.join(parent, self.stamp), 1 + while True: + try: + os.mkdir(path, 0o700) + break + except FileExistsError: + n += 1 + path = os.path.join(parent, f"{self.stamp}-{n}") self.archive = path return self.archive @@ -1765,22 +1916,42 @@ class Sweep: name = f"{base}-{n}" return name + def _occupied(self, path: str, writer: str = "") -> str: + """Why `path` may NOT be touched now, or '' - asked afresh: processes, + open files and tmux panes, and the session that recorded it. A scan + that could not be made is a reason, never a pass.""" + fresh = ProcScan() + if not fresh.available: + return "liveness could not be read: " + (fresh.why or "no process scan") + held = fresh.holders(path) + if held: + pid, comm, kind, _w = held[0] + return f"process {pid} ({comm or '?'}) has its {kind} there" + if writer and UUID_RE.match(writer) and writer != self.caller: + answer = self.helper.transcript_live(writer, fresh=True) + if answer != "none": + return (f"the session that recorded it ({writer[:8]}) is " + + ("live" if answer == "live" else "of unknown liveness")) + return "" + def _act_tree(self, t: Tree) -> None: # LIVENESS IS ASKED AGAIN AT THE MOMENT OF THE ACT, not trusted from the # read: a writer may have started in the tree since the table was made. - if t.exists and self.procs is not None: - fresh = ProcScan() - if fresh.available and fresh.holders(t.path): - pid, comm, kind, _w = fresh.holders(t.path)[0] - t.disposition = "live" - t.result = f"left: process {pid} ({comm}) arrived in it" + writer = (t.sidecar or {}).get("writer", "") + if t.exists: + why = self._occupied(t.path, writer) + if why: + t.disposition = "live" if not why.startswith("liveness") else "keep" + t.result = f"left: {why}" return if t.disposition == "prune": + # THAT ONE REGISTRATION: `git worktree prune` would unregister every + # missing tree of the repository - another lane's, a FOREIGN one, + # one on a disk that is not mounted - and say nothing of it. proc = git(t.checkout, "worktree", "remove", t.path) if proc.returncode != 0: - proc = git(t.checkout, "worktree", "prune") - if proc.returncode != 0: - raise _ActFailed("git worktree prune: " + _first_line(proc.stderr)) + raise _ActFailed("git worktree remove " + t.path + ": " + _first_line(proc.stderr) + + " - a repository-wide prune is never run in its place") regs = [r["path"] for r in worktree_registrations(t.checkout)] if t.path in regs: raise _ActFailed("the registration is still there after the prune") @@ -1836,6 +2007,10 @@ class Sweep: t.bundle = self._bundle(t, ref_to_bundle) # 3. IGNORED WORK no commit could carry. extra = ignored_work(t) + if extra is None: + raise _ActFailed("its ignored files could not be listed, so whether it holds an " + ".env or local configuration no commit carries is unknown; left " + "in place" + (f"; its rescue is at {t.rescue}" if t.rescue else "")) if extra: size = sum(dir_size(os.path.join(t.path, r)) if os.path.isdir(os.path.join(t.path, r)) else os.lstat(os.path.join(t.path, r)).st_size for r in extra @@ -1848,9 +2023,14 @@ class Sweep: + (f"; its rescue is at {t.rescue}" if t.rescue else "")) t.ignored_archive = self._tar(t.path, extra, f"{t.slice}-ignored.tar.gz") - # 4. NOTHING MOVED, PROVED AGAIN AT THE MOMENT OF REMOVAL: the head is the - # one classified, and the content is clean - or, for a WIP rescue, is + # 4. NOTHING MOVED, PROVED AGAIN AT THE MOMENT OF REMOVAL: no writer + # arrived while the pushes, bundles and archives ran; the head is the + # one classified; and the content is clean - or, for a WIP rescue, is # byte for byte the tree the rescue commit carries. + why = self._occupied(t.path, writer) + if why: + raise _ActFailed(f"{why} - it arrived while the rescue was made; left in place" + + (f"; its rescue is at {t.rescue}" if t.rescue else "")) head_now = git_out(t.path, "rev-parse", "-q", "--verify", "HEAD") or "" if head_now != t.head: raise _ActFailed(f"HEAD moved from {short(t.head) or 'none'} to " @@ -1879,20 +2059,22 @@ class Sweep: raise _ActFailed("the directory is still there after the removal") t.result = "removed" + ((f"; pushed to {t.rescue}" if disp == "push+remove" else f"; rescue {t.rescue}") if t.rescue else "") - # 5. THE BRANCHES OF A MERGED TREE. + # 5. THE BRANCHES OF A MERGED TREE - each deleted only AT THE SHA THAT + # WAS JUDGED merged, so a branch that moved under the sweep is kept. if disp == "remove+delete-branch" and t.branch: co = t.checkout if t.checkout and os.path.isdir(t.checkout) else checkout - proc = git(co, "branch", "-D", t.branch) - t.result += "; local branch deleted" if proc.returncode == 0 else ( - "; local branch NOT deleted: " + _first_line(proc.stderr)) + done, why = delete_local_branch(co, t.branch, t.head) + t.result += "; local branch deleted" if done else "; local branch NOT deleted: " + why + if not done: + self.failed += 1 head = (t.merge or {}).get("head", "") ours = t.lane_trailer.lower() == self.lane.lower() if t.remote_tip and ours and head and (t.remote_tip == head or git( co, "merge-base", "--is-ancestor", t.remote_tip, head).returncode == 0): - proc = git(co, "push", "--no-verify", "origin", "--delete", t.remote_branch, - timeout=GIT_NET_TIMEOUT) - t.result += "; remote branch deleted" if proc.returncode == 0 else ( - "; remote branch NOT deleted: " + _first_line(proc.stderr)) + done, why = delete_remote_branch(co, t.remote_branch, t.remote_tip) + t.result += "; remote branch deleted" if done else "; remote branch NOT deleted: " + why + if not done: + self.failed += 1 elif t.remote_tip: t.result += ("; remote branch kept (" + ("its Lane: trailer is " + (t.lane_trailer or "absent") if not ours else @@ -1986,15 +2168,36 @@ class Sweep: def _act_item(self, i: Item) -> None: if i.kind == "scratch" and i.disposition == "archive+remove": - fresh = ProcScan() - if fresh.available and fresh.holders(i.path): - i.disposition, i.result = "live", "left: a process arrived in it" + why = self._occupied(i.path) + if why: + i.disposition = "live" if not why.startswith("liveness") else "keep" + i.result = "left: " + why return + before = content_print(i.path) parent, name = os.path.dirname(i.path), os.path.basename(i.path) i.extra["archive"] = self._tar(parent, [name], f"scratch-{safe_ref_part(name)}.tar.gz") + # WHAT IS REMOVED IS WHAT WAS ARCHIVED: no writer arrived while the + # tar ran, and nothing outside the caches changed. + why = self._occupied(i.path) + if why: + raise _ActFailed(f"{why} - it arrived while the archive was made; left in place; " + f"the archive is {tilde(i.extra['archive'])}") + if content_print(i.path) != before: + raise _ActFailed("it changed while it was archived; left in place; the archive " + f"{tilde(i.extra['archive'])} is of what it was") rmtree(i.path) i.result = "archived and removed" elif i.kind == "cache" and i.disposition == "remove": + # (14) THE OWNING TREE IS ASKED AGAIN: a writer may have started + # there while the rescues ran, or the tree's own act left it live. + owner = i.extra.get("owner") or i.path + left = [t for t in self.trees if under(i.path, t.path) and self._cache_protected(t)] + why = ("its tree was left: " + (left[0].result or left[0].reason)) if left \ + else self._occupied(owner) + if why: + i.disposition = "keep" + i.result = "left: " + why + return rmtree(i.path) i.result = "removed" elif i.kind == "sandbox" and i.disposition == "remove": @@ -2006,15 +2209,20 @@ class Sweep: rmtree(i.path) i.result = "removed" elif i.kind == "branch" and i.disposition in ("delete", "delete+remote"): - proc = git(i.extra["checkout"], "branch", "-D", i.extra["branch"]) - if proc.returncode != 0: - raise _ActFailed("git branch -D: " + _first_line(proc.stderr)) + done, why = delete_local_branch(i.extra["checkout"], i.extra["branch"], + i.extra.get("tip", "")) + if not done: + raise _ActFailed("the local branch was not deleted: " + why) i.result = "local branch deleted" if i.disposition == "delete+remote": - proc = git(i.extra["checkout"], "push", "--no-verify", "origin", "--delete", - i.extra["branch"], timeout=GIT_NET_TIMEOUT) - i.result += "; remote branch deleted" if proc.returncode == 0 else ( - "; remote branch NOT deleted: " + _first_line(proc.stderr)) + done, why = delete_remote_branch(i.extra["checkout"], i.extra["branch"], + i.extra.get("remote_tip", "")) + i.extra["remote_deleted"] = done + if done: + i.result += "; remote branch deleted" + else: + i.result += "; remote branch NOT deleted: " + why + self.failed += 1 def _register_items(self) -> None: done = [i for i in self.items if i.result and not i.result.startswith(("FAILED", "left"))] @@ -2027,7 +2235,8 @@ class Sweep: for i in scratch)) if branches: self._note(None, "swept branches merged by PR: " + "; ".join( - f"{i.extra['branch']} (PR #{i.extra.get('pr')}{' + remote' if 'remote' in i.result else ''})" + f"{i.extra['branch']} (PR #{i.extra.get('pr')}" + f"{' + remote' if i.extra.get('remote_deleted') else ''})" for i in branches)) if cleaned: total = sum(i.size for i in cleaned) @@ -2099,6 +2308,34 @@ class _ActFailed(Exception): pass +def delete_local_branch(checkout: str, branch: str, expect: str): + """(done, why): `refs/heads/` deleted only while it is still at + `expect` and no worktree has it checked out - `update-ref -d` with the + old value, so a branch that moved under the sweep is never lost.""" + if not expect: + return False, "the SHA it was judged at is not known" + for reg in worktree_registrations(checkout): + if reg.get("branch") == f"refs/heads/{branch}": + return False, f"it is checked out in {reg.get('path')} now" + proc = git(checkout, "update-ref", "-d", f"refs/heads/{branch}", expect) + if proc.returncode != 0: + return False, (f"it is no longer at {short(expect)} (" + _first_line(proc.stderr) + ")") + return True, "" + + +def delete_remote_branch(checkout: str, branch: str, expect: str): + """(done, why): origin's branch deleted only while origin still has it at + `expect` - a lease, so commits pushed there since are never discarded.""" + if not expect: + return False, "the SHA it was judged at is not known" + proc = git(checkout, "push", "--no-verify", "--porcelain", + f"--force-with-lease=refs/heads/{branch}:{expect}", "origin", + f":refs/heads/{branch}", timeout=GIT_NET_TIMEOUT) + if proc.returncode != 0: + return False, _first_line(proc.stderr or proc.stdout) + return True, "" + + def _lean_text(lean: list) -> str: return "; ".join(f"{dep} leans on it ({how})" for dep, how in lean[:5]) + ( f"; and {len(lean) - 5} more" if len(lean) > 5 else "") @@ -2117,6 +2354,25 @@ def _gitdir_activity(path: str) -> float: return newest +def content_print(root: str) -> list: + """Every non-cache file under `root` with its size and mtime - what an + archive of it carries, so a change made while it was archived shows.""" + out = [] + for base, dirs, files in os.walk(root): + dirs[:] = sorted(d for d in dirs if d not in CACHE_NAMES and d not in VENV_NAMES) + for name in sorted(files): + if name.endswith((".pyc", ".pyo")): + continue + full = os.path.join(base, name) + try: + st = os.lstat(full) + except OSError: + out.append((os.path.relpath(full, root), -1, -1)) + continue + out.append((os.path.relpath(full, root), st.st_size, st.st_mtime_ns)) + return out + + def _no_caches(info): """A tar member filter: caches and bytecode hold no work, so an archive never carries them.""" @@ -2215,7 +2471,7 @@ def branch_items(checkout: str, lane: str, evidence: MergeEvidence) -> list: it = Item("branch", name, "delete+remote" if remote_ok else "delete", judged["why"] + ("; the remote branch too (this lane's, nothing beyond the PR)" if remote_ok else ""), ours, detail) - it.extra["pr"] = judged.get("pr") + it.extra.update(pr=judged.get("pr"), tip=tip, remote_tip=remote_tip if remote_ok else "") else: up_state = "missing" if upstream: @@ -2223,10 +2479,15 @@ def branch_items(checkout: str, lane: str, evidence: MergeEvidence) -> list: if not upstream.startswith("refs/") else None if up_tip: c = (git_out(checkout, "rev-list", "--left-right", "--count", - f"{up_tip}...{tip}") or "0 0").split() - b, a = int(c[0]), int(c[1]) - up_state = ("diverged" if a and b else "ahead" if a else - "behind" if b else "published") + f"{up_tip}...{tip}") or "").split() + if len(c) == 2 and all(x.isdigit() for x in c): + b, a = int(c[0]), int(c[1]) + up_state = ("diverged" if a and b else "ahead" if a else + "behind" if b else "published") + else: + # NEVER "0 0" FOR UNKNOWN: that would read as published + # and drop the branch from the list. + up_state = "unreadable (git rev-list failed)" else: up_state = "missing" if up_state in ("published", "behind"): @@ -2416,8 +2677,13 @@ def expire(opts) -> int: if age < keep_days: rows.append((adir, age, "keep", f"{age} d old; retention {keep_days} d")) continue + ledger, why = archive_ledger(adir) + if ledger is None: + rows.append((adir, age, "keep", why + " - its rescues cannot be checked, so it " + "is kept")) + continue missing = [] - for origin, branch, sha in _rescues_of(adir): + for origin, branch, sha in ledger: proc = run(["git", "ls-remote", origin, f"refs/heads/{branch}"], env=git_env(), timeout=GIT_NET_TIMEOUT) got = proc.stdout.split()[0] if proc.returncode == 0 and proc.stdout.split() else "" @@ -2425,6 +2691,11 @@ def expire(opts) -> int: missing.append(f"{branch} (origin could not be read)") elif not got: missing.append(f"{branch} (gone from origin)") + elif got != sha: + # THE RESCUED COMMIT, NOT THE NAME: a branch reset or remade + # at another commit no longer holds the work. + missing.append(f"{branch} (on origin at {short(got)}, not the rescued " + f"{short(sha)})") if missing: rows.append((adir, age, "keep", "a rescue branch is not on origin: " + "; ".join(missing) + " - this archive is the only copy")) @@ -2459,7 +2730,45 @@ def expire(opts) -> int: return 0 -def _rescues_of(adir: str) -> list: +SHA_RE = re.compile(r"^[0-9a-f]{40}([0-9a-f]{24})?$") + + +def archive_ledger(adir: str): + """(rescues, why): the archive's rescue ledger, or None with why it cannot + be trusted. An archive is trusted only when it is WHOLE: `MANIFEST.sha256` + is there and every file in the archive is in it at its digest (an + interrupted sweep leaves a bundle the ledger never recorded), and + `rescues.tsv` is there and every row of it is origin, branch and a full + SHA. A header-only ledger in a whole archive is a verified "no rescues".""" + manifest = os.path.join(adir, "MANIFEST.sha256") + listed: dict = {} + try: + with open(manifest, encoding="utf-8") as fh: + for line in fh: + if not line.strip(): + continue + digest, sep, rel = line.rstrip("\n").partition(" ") + if not sep or not re.match(r"^[0-9a-f]{64}$", digest): + return None, "its MANIFEST.sha256 is malformed" + listed[rel] = digest + except OSError: + return None, "it has no readable MANIFEST.sha256 (the sweep that made it did not finish)" + for base, _dirs, files in os.walk(adir): + for name in files: + full = os.path.join(base, name) + rel = os.path.relpath(full, adir) + if rel == "MANIFEST.sha256": + continue + if rel not in listed: + return None, f"{rel} is in it and not in its MANIFEST.sha256" + try: + with open(full, "rb") as fh: + if hashlib.sha256(fh.read()).hexdigest() != listed[rel]: + return None, f"{rel} does not match its MANIFEST.sha256" + except OSError: + return None, f"{rel} could not be read" + if "rescues.tsv" not in listed: + return None, "it has no rescues.tsv" out = [] try: with open(os.path.join(adir, "rescues.tsv"), encoding="utf-8") as fh: @@ -2467,11 +2776,12 @@ def _rescues_of(adir: str) -> list: if line.startswith("#") or not line.strip(): continue cols = line.rstrip("\n").split("\t") - if len(cols) >= 3 and cols[0] and cols[1]: - out.append((cols[0], cols[1], cols[2])) + if len(cols) < 3 or not cols[0] or not cols[1] or not SHA_RE.match(cols[2]): + return None, "its rescues.tsv has a row that is not origin, branch, sha" + out.append((cols[0], cols[1], cols[2])) except OSError: - pass - return out + return None, "its rescues.tsv could not be read" + return out, "" # ==================================================================== main diff --git a/tests/test_lane_worktrees.py b/tests/test_lane_worktrees.py index 43d39bc..587951f 100644 --- a/tests/test_lane_worktrees.py +++ b/tests/test_lane_worktrees.py @@ -238,9 +238,9 @@ def write_spec(self) -> None: self.spec_file.write_text(json.dumps(spec)) self.prs_file.write_text(json.dumps(self.prs)) - def pr(self, number: int, branch: str, state: str, head: str) -> None: + def pr(self, number: int, branch: str, state: str, head: str, base: str = "main") -> None: self.prs.append({"number": number, "headRefName": branch, "state": state, - "headRefOid": head, "isCrossRepository": False, + "headRefOid": head, "baseRefName": base, "isCrossRepository": False, "mergedAt": "2026-10-01T00:00:00Z" if state == "MERGED" else None}) # ------------------------------------------------------------ running @@ -625,6 +625,30 @@ def test_merged_is_the_register_or_gh_never_ancestry(estate, tmp_path): assert estate.git("branch", "--list", "feat/reg") == "" +def test_a_merge_into_another_base_is_no_landing_and_an_open_pr_comes_first(estate): + """Copilot on #168: a PR merged into a release branch is no landing on + main, and an OPEN PR protects its branch however an older PR of the same + branch was merged.""" + elsewhere = estate.worktree("elsewhere", "feat/rel") + tip = estate.commit(elsewhere, "e", {"e.txt": "e\n"}) + estate.git("push", "-q", "-u", "origin", "feat/rel", cwd=elsewhere) + estate.pr(41, "feat/rel", "MERGED", tip, base="release/1") + reopened = estate.worktree("reopened", "feat/again") + old = estate.commit(reopened, "o", {"o.txt": "o\n"}) + estate.git("push", "-q", "-u", "origin", "feat/again", cwd=reopened) + estate.pr(42, "feat/again", "MERGED", old) + estate.pr(43, "feat/again", "OPEN", old) + rows = rows_of(estate.sweep(LANE, "--porcelain").stdout) + assert rows[str(elsewhere)][0] == "remove", rows[str(elsewhere)] + assert "merged into release/1" in rows[str(elsewhere)][2] + assert rows[str(reopened)][0] == "remove" and "PR #43 is open" in rows[str(reopened)][2] + yes = estate.sweep(LANE, "--yes") + assert yes.returncode == 0, yes.stdout + yes.stderr + local = estate.git("branch", "--format=%(refname:short)").splitlines() + assert "feat/rel" in local and "feat/again" in local, "neither branch is merged" + assert {"feat/rel", "feat/again"} <= set(estate.remote_heads()) + + def test_without_gh_nothing_is_called_merged(estate): tree = estate.worktree("m", "feat/m") tip = estate.commit(tree, "m", {"m.txt": "m\n"}) @@ -635,6 +659,64 @@ def test_without_gh_nothing_is_called_merged(estate): assert "LANES_NO_GITHUB=1" in estate.sweep(LANE, env={"LANES_NO_GITHUB": "1"}).stdout +def test_a_tree_another_lanes_inventory_also_names_is_kept(estate): + """Copilot on #168: inventory records are history and a path can be + reused, so a tree BOTH lanes' inventories name is neither's to sweep.""" + tree = estate.worktree("shared", "feat/shared") + estate.git("push", "-q", "-u", "origin", "feat/shared", cwd=tree) + other = estate.root / "lane-state" / "repoA-7" / "trees" + other.mkdir(parents=True) + (other / "c1.yaml").write_text(f"schema: 1\npath: {tree}\n") + rows = rows_of(estate.sweep(LANE, "--porcelain").stdout) + assert rows[str(tree)][0] == "keep" and "repoA-7's inventory names it too" in rows[str(tree)][2] + assert estate.sweep(LANE, "--yes").returncode == 0 + assert tree.is_dir() + + +def test_a_clone_its_linked_worktrees_share_is_kept(estate): + """Copilot on #168: removing a clone takes the git directory every + worktree linked to it shares, a live writer's included.""" + clone = estate.lane_root / "cl" + estate.git("clone", "-q", GH_URL, clone, cwd=estate.root) + linked = estate.root / "elsewhere-wt" + estate.git("worktree", "add", "-q", "-b", "feat/linked", linked, "origin/main", cwd=clone) + estate.record(clone) + rows = rows_of(estate.sweep(LANE, "--porcelain").stdout) + assert rows[str(clone)][0] == "keep", rows[str(clone)] + assert "linked worktree(s) share" in rows[str(clone)][2] + + +def test_an_unreadable_inventory_record_refuses_the_sweep(estate): + """Copilot on #168: a record that cannot be read is a tree whose owner is + unknown, so the sweep refuses rather than report nothing to retire.""" + estate.worktree("fine", "feat/fine") + estate.inventory.append(US.join(["c9-odd", str(estate.lane_root / "odd"), "x", "y", "none", + "0", "0", ME, "2026-10-05T00:00:00Z", str(estate.checkout), + "1", "op-1", "9"])) + before = snapshot(estate.root, skip=("helper.log", "helper-spec.json", "prs.json")) + for args in (("--dry-run", "--porcelain"), ("--yes",)): + proc = estate.sweep(LANE, *args) + assert proc.returncode == 2, (args, proc.stdout, proc.stderr) + assert "c9-odd is unreadable or of schema 9" in proc.stdout + assert snapshot(estate.root, skip=("helper.log", "helper-spec.json", "prs.json")) == before + + +def test_the_lane_name_is_resolved_before_anything_is_derived_from_it(estate): + """Copilot on #168: Amendment 15's resolver first, as lane-start and + lane-end read it; an alias table that cannot be read refuses.""" + estate.worktree("w", "feat/w") + estate.extra_spec = {"canon-lane": {"rc": 0, "out": f"{LANE}\n"}} + proc = estate.sweep(LANE.upper(), "--porcelain") + assert proc.returncode == 3, proc.stdout + proc.stderr + assert proc.stdout.splitlines()[0].split("\t")[:2] == ["lane", LANE] + reconcile = [r for r in (json.loads(x) for x in estate.helper_log.read_text().splitlines()) + if r["argv"][:1] == ["lane-reconcile"]] + assert reconcile[-1]["argv"] == ["lane-reconcile", LANE] + estate.extra_spec = {"canon-lane": {"rc": 66, "err": "the alias table could not be read\n"}} + refused = estate.sweep(LANE, "--porcelain") + assert refused.returncode == 2 and "alias table could not be read" in refused.stdout + + def test_include_foreign_takes_a_word_and_another_lanes_tree_is_never_taken(estate): mine = estate.worktree("foreign-a", "feat/fa", where="claude", record=False) estate.git("push", "-q", "-u", "origin", "feat/fa", cwd=mine) @@ -745,6 +827,77 @@ def test_scratch_is_archived_then_removed_and_caches_are_removed(estate): assert str(scratch / "__pycache__") not in caches, "it went with its scratch" +def test_caches_are_never_taken_from_a_foreign_tree(estate): + """Copilot on #168: a FOREIGN tree under the lane's root is somebody's, + live or not, and so are its caches.""" + foreign = estate.worktree("theirs", "feat/theirs", record=False) + (foreign / "__pycache__").mkdir() + (foreign / "__pycache__" / "t.pyc").write_bytes(b"\0") + proc = estate.sweep(LANE, "--include-caches", "--yes", "--porcelain") + assert proc.returncode == 0, proc.stdout + proc.stderr + assert str(foreign / "__pycache__") not in items_of(proc.stdout, "cache") + assert (foreign / "__pycache__" / "t.pyc").exists() + + +def test_branch_deletes_wait_for_a_fetch_that_worked(estate): + """Copilot on #168: --yes deletes a merged branch only once origin was + read NOW - the lane's own checkout is fetched too, and a fetch that + failed deletes nothing.""" + estate.git("checkout", "-q", "-b", "feat/gone-merged", "origin/main") + tip = estate.commit(estate.checkout, "g", {"g.txt": "g\n"}) + estate.git("push", "-q", "-u", "origin", "feat/gone-merged") + estate.git("checkout", "-q", "main") + estate.pr(51, "feat/gone-merged", "MERGED", tip) + estate.origin.rename(estate.origin.with_name("moved.git")) + try: + proc = estate.sweep(LANE, "--branches", "--yes", "--porcelain") + finally: + estate.origin.with_name("moved.git").rename(estate.origin) + items = items_of(proc.stdout, "branch") + assert items["feat/gone-merged"][0] == "keep", items + assert "origin could not be fetched" in items["feat/gone-merged"][3] + assert "feat/gone-merged" in estate.git("branch", "--format=%(refname:short)").splitlines() + + +def test_a_clean_tree_without_origin_is_bundled_and_removed_under_bundle_yes(estate): + """Copilot on #168: `--yes` fetches, and a repository with no origin is + no failed fetch - the dry run's `bundle+remove` is what `--yes` does.""" + solo = estate.projects / "solo" + estate.git("init", "-q", "-b", "main", solo, cwd=estate.root) + estate.commit(solo, "solo", {"s.txt": "s\n"}) + lone = estate.lane_root / "lone" + estate.lane_root.mkdir(parents=True, exist_ok=True) + estate.git("worktree", "add", "-q", "-b", "feat/lone", lone, "main", cwd=solo) + estate.commit(lone, "lone", {"l.txt": "l\n"}) + estate.record(lone) + dry = rows_of(estate.sweep(LANE, "--bundle", "--porcelain").stdout) + assert dry[str(lone)][0] == "bundle+remove", dry + yes = estate.sweep(LANE, "--bundle", "--yes", "--porcelain") + assert yes.returncode == 0, yes.stdout + yes.stderr + assert not lone.exists() + (archive,) = estate.archives() + assert (archive / "lone.bundle").is_file() + + +def test_a_distance_to_origin_that_cannot_be_read_keeps_the_tree(estate): + """Copilot on #168: a failed `rev-list` is a read error, never "0 ahead" + - which is what lets a tree go without a rescue.""" + tree = estate.worktree("broken", "feat/broken", record=False) + pushed = estate.commit(tree, "c", {"c.txt": "c\n"}) + estate.git("push", "-q", "-u", "origin", "feat/broken", cwd=tree) + estate.commit(tree, "d", {"d.txt": "d\n"}) + estate.record(tree) + # origin's tip goes missing from the clone's object store: the distance + # from it to HEAD can no longer be walked + obj = estate.checkout / ".git" / "objects" / pushed[:2] / pushed[2:] + assert obj.is_file() + obj.unlink() + proc = estate.sweep(LANE, "--porcelain") + rows = rows_of(proc.stdout) + assert rows[str(tree)][0] == "keep", rows[str(tree)] + assert "could not be read" in rows[str(tree)][2] + + def test_killed_suite_sandboxes_whose_owner_is_gone_are_removed(estate): user = os.environ.get("USER") or __import__("getpass").getuser() base = estate.sandboxes @@ -814,33 +967,53 @@ def test_a_clone_something_leans_on_is_load_bearing_and_never_removed(estate): # =================================================================== --expire -def _archive(estate, days: int, branch: str | None) -> Path: +def _archive(estate, days: int, branch: str | None, sha: str = "0" * 40, + manifest: bool = True) -> Path: + """A sweep archive `days` old, as `--yes` leaves one: DISPOSITION.md, + rescues.tsv naming `branch` at `sha`, and a MANIFEST.sha256 of both.""" when = _dt.datetime.now(_dt.timezone.utc) - _dt.timedelta(days=days, hours=1) d = estate.state / "openRepoTools" / "sweeps" / LANE / when.strftime("%Y%m%dT%H%M%SZ") d.mkdir(parents=True) (d / "DISPOSITION.md").write_text("# x\n") rows = "# origin\tbranch\tsha\tcheckout\ttree\n" if branch: - rows += f"{GH_URL}\t{branch}\t{'0' * 40}\t-\t-\n" + rows += f"{GH_URL}\t{branch}\t{sha}\t-\t-\n" (d / "rescues.tsv").write_text(rows) + if manifest: + (d / "MANIFEST.sha256").write_text("".join( + f"{hashlib.sha256((d / n).read_bytes()).hexdigest()} {n}\n" + for n in ("DISPOSITION.md", "rescues.tsv"))) return d def test_expire_keeps_young_archives_and_any_whose_rescue_left_origin(estate): estate.git("push", "-q", "origin", "main:refs/heads/rescue/repoA-1/kept-1") - young = _archive(estate, 89, "rescue/repoA-1/kept-1") - due = _archive(estate, 90, "rescue/repoA-1/kept-1") - orphan = _archive(estate, 120, "rescue/repoA-1/gone-1") + sha = estate.git("rev-parse", "main") + young = _archive(estate, 89, "rescue/repoA-1/kept-1", sha) + due = _archive(estate, 90, "rescue/repoA-1/kept-1", sha) + orphan = _archive(estate, 120, "rescue/repoA-1/gone-1", sha) plain = _archive(estate, 91, None) + # THE RESCUED COMMIT, NOT THE NAME: the branch is on origin at another sha. + moved = _archive(estate, 92, "rescue/repoA-1/kept-1", "1" * 40) + # AN ARCHIVE A SWEEP NEVER FINISHED (no manifest), and one holding a file + # its manifest does not list (a bundle the ledger never recorded). + unfinished = _archive(estate, 93, None, manifest=False) + stray = _archive(estate, 94, None) + (stray / "x.bundle").write_bytes(b"bundle") dry = estate.sweep("--expire", "--porcelain") assert dry.returncode == 3, dry.stdout + dry.stderr rows = {line.split("\t")[2]: line.split("\t")[1] for line in dry.stdout.splitlines()} + why = {line.split("\t")[2]: line.split("\t")[4] for line in dry.stdout.splitlines()} assert rows[str(young)] == "keep" and rows[str(due)] == "expire" assert rows[str(orphan)] == "keep" and rows[str(plain)] == "expire" + assert rows[str(moved)] == "keep" and "not the rescued 1111111" in why[str(moved)] + assert rows[str(unfinished)] == "keep" and "MANIFEST" in why[str(unfinished)] + assert rows[str(stray)] == "keep" and "x.bundle" in why[str(stray)] assert young.is_dir() and due.is_dir() yes = estate.sweep("--expire", "--yes") assert yes.returncode == 0, yes.stdout + yes.stderr assert young.is_dir() and orphan.is_dir() + assert moved.is_dir() and unfinished.is_dir() and stray.is_dir() assert not due.exists() and not plain.exists() log = (estate.state / "openRepoTools" / "sweeps" / "EXPIRED.log").read_text() assert str(due) in log and str(orphan) not in log From ae17742a2207e61429ef3f6228798f53fd28d66b Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:22:52 +0000 Subject: [PATCH 14/29] Document round 1's rules in the lanes manual The manual now states what changed: the canonical lane name, refusal on an unreadable record, contested trees, the single-registration prune, the linked-worktree clone guard, merged-into-default with open PRs first, SHA-fenced branch deletes, liveness asked again before removal, scratch and cache rechecks, the exclusive archive directory, and expiry of whole archives only. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- docs/README-lanes.md | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/docs/README-lanes.md b/docs/README-lanes.md index 6beff9c..576a7c3 100644 --- a/docs/README-lanes.md +++ b/docs/README-lanes.md @@ -3327,7 +3327,12 @@ its two roots (`/.claude/worktrees/*`, `.lane-worktrees//*`). A tree the inventory does not name is **FOREIGN**: reported and left, unless `--include-foreign` and a `--word ""` (recorded in every register line it causes). A tree ANOTHER lane's inventory names is that lane's and is never -taken from here, word or no word. The lane's own checkout is never a candidate. +taken from here, word or no word — and one BOTH inventories name is kept, since +records are history and a path can be reused. An inventory record that cannot be +read (no path, an unknown schema) refuses the sweep, exit **2**: its tree's owner +is unknown. The lane's own checkout is never a candidate. The lane name is +resolved first (`lanes-edit.sh canon-lane`, Amendment 15), as `lane-start` and +`lane-end` resolve it; an alias table that cannot be read refuses. **Who may act** is #97's reconciliation (`lanes-edit.sh lane-reconcile`), read before anything is written — the fetch included: @@ -3351,16 +3356,17 @@ Each tree is classified in this order; the first row that matches decides. | not in the lane's inventory | `foreign` | nothing (`--include-foreign --word …` applies the rows below) | | a process stands in it or holds a file open there (`/proc`, else `lsof`), a tmux pane's current path is in it, it is named by `--live`, the sweep was started inside it, or the session that recorded it is live | `live` | nothing, ever — asked again at the moment of the act | | liveness could not be read | `keep` | nothing | -| its directory is gone and git still registers it | `prune` | `git worktree remove ` (the one registration; `git worktree prune` only if that is refused) | +| its directory is gone and git still registers it | `prune` | `git worktree remove ` — that one registration; a repository-wide `git worktree prune` is never run (it would unregister other lanes' and FOREIGN trees too) | | its directory is gone and nothing registers it | `gone` | nothing; the inventory record is history | -| a registration someone LOCKED; a submodule with uncommitted work or a commit no remote holds; a repository nested inside it; a CLONE with a branch or a stash origin lacks | `keep` | nothing, and the line says which | +| another lane's inventory names it too | `keep` | nothing: which lane owns it now is a person's to say | +| a registration someone LOCKED; a submodule with uncommitted work or a commit no remote holds; a repository nested inside it; a CLONE with a branch or a stash origin lacks, or whose git directory linked worktrees share | `keep` | nothing, and the line says which | | something leans on it — another repository's `objects/info/alternates`, or a remote whose URL is its path | `load-bearing` | nothing; the line names every dependent and the remedy (`git repack -a -d`, then drop the alternates or re-point the remote) | | no `origin` | `keep` (`bundle+remove` with `--bundle`) | a bundle is its only rescue | | dirty or untracked work | `wip-rescue+remove` | `git add -A` into a COPY of its index, `commit-tree` on its head, `rescue//-` pushed and seen on origin, a bundle, then removed | | an unborn branch, clean | `remove` | removed | | detached at a commit origin holds | `remove` | removed | | detached with commits of its own | `rescue+remove` | `rescue//-` pushed, a bundle, then removed | -| MERGED — its PR LANDED in the register, or `gh` says MERGED, and that PR's head holds this tip | `remove+delete-branch` | removed; local branch deleted; the remote branch too where its `Lane:` trailer is this lane's and it holds nothing the PR did not | +| MERGED — its PR LANDED in the register, or `gh` says MERGED, into the default branch, and that PR's head holds this tip | `remove+delete-branch` | removed; local branch deleted at the SHA judged merged (`update-ref -d `), and the remote branch too — under a lease on the judged SHA — where its `Lane:` trailer is this lane's and it holds nothing the PR did not | | clean, every commit on its own remote branch (an open PR is untouched) | `remove` | removed; the branch stays | | clean, its head on some origin branch | `remove` | removed; the branch stays | | unpublished commits on `main`/`master`, or on a branch origin has DIVERGED from | `rescue+remove` | the tip to `rescue//-`, then removed; the branch itself is never force-pushed | @@ -3368,13 +3374,17 @@ Each tree is classified in this order; the first row that matches decides. **"Merged" is never `git branch --merged` alone**: a squash merge leaves no ancestry, and a branch whose tip IS on `main` with no pull request naming it is -listed as unmerged and its branch kept. The branch's pull request comes from one -`gh pr list --state all` per repository; with `LANES_NO_GITHUB=1` or no `gh` -nothing is called merged. The register's `LANDED` lines are read from +listed as unmerged and its branch kept. A PR merged into another base (a release +or feature branch) is no landing, and an OPEN PR protects its branch however an +older PR of the same branch was merged — unless the register has LANDED that +very PR. The branch's pull request comes from one `gh pr list --state all` per +repository; with `LANES_NO_GITHUB=1` or no `gh` nothing is called merged. The register's `LANDED` lines are read from `origin/` of the workspace repository. -**At the moment of removal** the head is re-read, and the tree must still be -clean — or, after a WIP rescue, must still be byte for byte the tree the rescue +**Liveness is asked again before the rescue and again at the moment of +removal** — processes, open files, tmux panes and the recording session, afresh; +a scan that cannot be made leaves the tree. **At the moment of removal** the +head is re-read, and the tree must still be clean — or, after a WIP rescue, must still be byte for byte the tree the rescue commit carries. A push that origin refuses leaves the tree exactly as it was: the WIP commit is made BESIDE the tree, never into it, so its branch, index and files never move. @@ -3390,9 +3400,9 @@ the tree in place for a person. | flag | what | disposition | |---|---|---| -| `--branches` | local branches no worktree holds, in the lane's checkout | merged by PR evidence: `delete` (`delete+remote` where the `Lane:` trailer is this lane's and origin's tip is inside the PR); an open PR's branch: `keep`; unmerged with a missing, diverged or unpushed upstream: `list` with its tip, distance from `origin/main`, last commit date and owner — never deleted; `main`, `master` and `rescue/*`: never touched | -| `--include-scratch` | `.lane-worktrees//*-scratch`, `briefs/`, `bin/` that are no checkout | `archive+remove`: tar (caches left out) and sha256, then removed | -| `--include-caches` | `__pycache__`, `.pytest_cache`, `.mypy_cache`, `.ruff_cache`, `node_modules`, any directory with `pyvenv.cfg` (and `venv/`, `.venv/` with an activate script), under `.lane-worktrees//**` and the lane's inventory trees | `remove`, without archiving — never one git tracks, one in a live tree, or one inside scratch being archived | +| `--branches` | local branches no worktree holds, in the lane's checkout | merged by PR evidence: `delete` (`delete+remote` where the `Lane:` trailer is this lane's and origin's tip is inside the PR), each at the SHA judged and only after a fetch of that checkout that worked under `--yes`; an open PR's branch: `keep`; unmerged with a missing, diverged, unpushed or unreadable upstream: `list` with its tip, distance from `origin/main`, last commit date and owner — never deleted; `main`, `master` and `rescue/*`: never touched | +| `--include-scratch` | `.lane-worktrees//*-scratch`, `briefs/`, `bin/` that are no checkout | `archive+remove`: tar (caches left out) and sha256, then removed — only if no writer arrived and nothing outside the caches changed while the tar ran | +| `--include-caches` | `__pycache__`, `.pytest_cache`, `.mypy_cache`, `.ruff_cache`, `node_modules`, any directory with `pyvenv.cfg` (and `venv/`, `.venv/` with an activate script), under `.lane-worktrees//**` and the lane's inventory trees | `remove`, without archiving — never one git tracks, one inside scratch being archived, or one in a FOREIGN tree, a live one, one whose liveness is unknown or one left by its own act; the owning tree's liveness is asked again just before | | `--include-sandboxes` | `tmp.*` and `pytest-of-$USER/pytest-*` in `/tmp` and `$TMPDIR` (or `LANE_WORKTREES_SANDBOX_ROOTS`), owned by this account | `remove` where the owning process is gone: a pytest `.lock` naming a dead pid, and no live process standing in it, holding it open or naming it in its environment; a `tmp.*` younger than `sandbox_min_age_minutes` is left | | `--links` | every symlink under the estate and every worktree `.git` gitdir pointer | `broken`, listed with target and age; nothing changes | | `--bundle` | every tree acted on | a `git bundle` beside the archive (always, for a rescue) | @@ -3400,7 +3410,8 @@ the tree in place for a person. ### The sweeps directory, the register line, retention `${XDG_STATE_HOME:-$HOME/.local/state}/openRepoTools/sweeps///`, -mode 0700, created by the first act of a `--yes` and never by a dry run: +mode 0700, created EXCLUSIVELY by the first act of a `--yes` (a second sweep +started in the same second gets `-2`) and never by a dry run: * `DISPOSITION.md` — every tree and item acted on: what it was, why, what was done, where its rescue is, and the register line that records it; @@ -3418,8 +3429,11 @@ outlives the archive. **Retention: 90 days** (`sweep.conf`). `sweep --expire` lists archives older than that, and with `--yes` removes those whose every rescue branch is still on -origin (`git ls-remote`); an archive whose rescue branch is gone from origin is -the only copy and is NEVER expired. What expired is appended to +origin AT THE RESCUED SHA (`git ls-remote`); an archive whose rescue branch is +gone from origin, or was remade at another commit, is the only copy and is NEVER +expired. Nor is an archive that is not WHOLE — no `MANIFEST.sha256`, a file it +does not list or whose digest differs (a sweep interrupted after a bundle), or a +`rescues.tsv` that is missing or has a row that is not origin, branch and SHA. What expired is appended to `sweeps/EXPIRED.log`. `${XDG_CONFIG_HOME:-$HOME/.config}/openRepoTools/sweep.conf` (or From 23d84da0399c4b705c55776a357dad5a8008adab Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:26:28 +0000 Subject: [PATCH 15/29] Say what expiry_rows keeps now that #168 checks whole archives The merge from #168 brought in archive_ledger and the exact-SHA check. The report's archive section reads them through expiry_rows, so its docstring now says it keeps archives that are not whole and those whose rescue branch moved. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lane-worktrees | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lane-worktrees b/lane-worktrees index a3e7b5b..219c51b 100755 --- a/lane-worktrees +++ b/lane-worktrees @@ -2664,8 +2664,9 @@ def _gitdir_pointer(path: str, base: str, now: float, out: list) -> None: def expiry_rows(conf: dict) -> list: """(archive, age in days, keep|expire, why) for every archive under the sweeps directory - `sweep --expire`'s table, and the report's. An archive - past retention is `expire` only while every rescue branch it records is on - origin; one whose rescue left origin is the only copy and is kept.""" + past retention is `expire` only while it is whole (`archive_ledger`) and + every rescue branch it records is on origin at the rescued SHA; one whose + rescue left or moved is the only copy and is kept.""" root = sweeps_dir() keep_days = conf["retention_days"] now = _dt.datetime.now(_dt.timezone.utc) From da7bc5937e555327723c8ff40ef18dda4206551d Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:34:05 +0000 Subject: [PATCH 16/29] Stop the suite's whole process group when a run ends Copilot on #169: a TERM to the pytest pid alone leaves whatever it is waiting on (a shell suite, a git, a fixture's sleep) running, while the EXIT trap deletes its temp root and releases the mkdir lock around it. The suite now starts in a process group of its own, with `set -m` on for that one line only. The INT/TERM handlers and the EXIT trap TERM the whole group, reap the leader, wait up to ten seconds for running members and then KILL the group. Running members are counted from `ps`, never `kill -0`, because an orphan's zombie answers `kill -0` until whoever adopted it reaps it. The kill cases now give the stub suite a child of its own and require it to be gone. Against the previous run.sh that case fails with "the suite's own child outlived its wrapper". Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 3 ++- tests/run.sh | 48 ++++++++++++++++++++++++++++++++------- tests/test_run_wrapper.py | 35 ++++++++++++++++++---------- 3 files changed, 65 insertions(+), 21 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2ee003b..d01e63c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -268,7 +268,8 @@ goes to `${XDG_CACHE_HOME:-~/.cache}/openRepoTools/pycache` `--basetemp` and the `TMPDIR` the suite's own `mktemp`s use — is under one run root, `${XDG_STATE_HOME:-~/.local/state}/openRepoTools/tmp/-/`, which the wrapper's EXIT trap removes however the run ends (a TERM or an INT stops the -suite first). The lock is computed BEFORE `TMPDIR` moves, so it is still the +suite's whole process group first, so no `git` or shell suite it was waiting on +outlives the run root or the lock). The lock is computed BEFORE `TMPDIR` moves, so it is still the workstation's. Where the repository's venv (`${XDG_CACHE_HOME:-~/.cache}/openRepoTools/venvs/openRepoTools`) has `pytest`, the suite runs from it, first on `PATH`, so the command line still reads diff --git a/tests/run.sh b/tests/run.sh index be19e7f..f273d66 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -102,23 +102,47 @@ wait_for_pytest() { # — block while any other suite is running return 0 } -# THE RUN ROOT AND ITS CLEANUP. `cleanup` is the EXIT trap and removes the run -# root and, on the `mkdir` path, the lock it holds; the INT and TERM handlers -# stop the suite and EXIT, so the EXIT trap runs - a handler that only cleaned -# up and returned would release the lock while the suite carried on. +# THE RUN ROOT AND ITS CLEANUP. `cleanup` is the EXIT trap: it stops whatever +# is left of the suite's PROCESS GROUP, then removes the run root and, on the +# `mkdir` path, the lock it holds. The INT and TERM handlers stop the suite and +# EXIT, so the EXIT trap runs - a handler that only cleaned up and returned +# would release the lock while the suite carried on. +# +# THE GROUP, NOT THE PID: pytest is waiting on a shell suite, a `git`, a +# fixture's `sleep` when the signal comes, and a TERM to pytest alone leaves +# them running with their temp root deleted under them and the lock released +# around them. The suite is started in a process group of its own (below), and +# the whole group is stopped and waited for, KILLed after ten seconds. RUN_ROOT="" LOCKDIR_HELD="" SUITE_PID="" +SUITE_PGID="" +# A member still RUNNING, read from `ps` and never from `kill -0`: a zombie +# answers `kill -0` until whoever adopted it reaps it, which may be never. +group_alive() { + ps -A -o pgid= -o stat= 2>/dev/null | + awk -v g="$SUITE_PGID" '$1 == g && $2 !~ /^Z/ { n++ } END { exit n ? 0 : 1 }' +} +stop_group() { + [ -n "$SUITE_PGID" ] || return 0 + kill -TERM -- "-$SUITE_PGID" 2>/dev/null || return 0 + if [ -n "$SUITE_PID" ]; then wait "$SUITE_PID" 2>/dev/null || :; fi + sg_n=0 + while group_alive && [ "$sg_n" -lt 50 ]; do + sleep 0.2 + sg_n=$((sg_n + 1)) + done + kill -KILL -- "-$SUITE_PGID" 2>/dev/null || : + return 0 +} cleanup() { + stop_group [ -n "$RUN_ROOT" ] && rm -rf -- "$RUN_ROOT" [ -n "$LOCKDIR_HELD" ] && rm -rf -- "$LOCKDIR_HELD" return 0 } on_signal() { # - if [ -n "$SUITE_PID" ]; then - kill -TERM "$SUITE_PID" 2>/dev/null || : - wait "$SUITE_PID" 2>/dev/null || : - fi + stop_group exit "$1" } @@ -150,10 +174,18 @@ make_run_root() { # interrupted by a trapped signal; the handler then stops the suite itself. # `<&0` keeps the suite's stdin this shell's: a background command's default # stdin, with job control off, is /dev/null. +# +# `set -m` FOR THE ONE LINE THAT STARTS IT puts the suite in a process group of +# its own, whose id is its pid, so `stop_group` reaches every process it +# started (one that made a session or group of its own is beyond any wrapper). +# Monitor mode is off again before the `wait`, so no job notice is printed. run_suite() { printf '%s: python3 -m pytest tests -q -p no:cacheprovider %s\n' "$prog" "${*:-}" >&2 + set -m python3 -m pytest tests -q -p no:cacheprovider --basetemp="$RUN_ROOT/basetemp" ${1+"$@"} <&0 & SUITE_PID=$! + SUITE_PGID=$SUITE_PID + set +m wait "$SUITE_PID" rs_rc=$? SUITE_PID="" diff --git a/tests/test_run_wrapper.py b/tests/test_run_wrapper.py index a17506f..c8b5ce7 100644 --- a/tests/test_run_wrapper.py +++ b/tests/test_run_wrapper.py @@ -51,8 +51,13 @@ } > "$FAKE_SUITE_LOG" : > "$TMPDIR/the-suite-wrote-here" if [ -n "${FAKE_SUITE_SLEEP:-}" ]; then + # A CHILD OF ITS OWN, as pytest waits on a shell suite or a `git`: the + # wrapper must stop it too, not only the process it started. + sleep "$FAKE_SUITE_SLEEP" & + printf '%s\n' "$!" > "$FAKE_SUITE_LOG.child" : > "$FAKE_SUITE_LOG.started" - exec sleep "$FAKE_SUITE_SLEEP" + wait + exit 0 fi exit "${FAKE_SUITE_RC:-0}" """ @@ -79,7 +84,7 @@ def __init__(self, root: Path, flock: bool = True): # NO `flock` ON THIS PATH: the wrapper takes its `mkdir` lock, the # one macOS uses. Every other tool it calls is linked in by name. for tool in ("dirname", "awk", "mkdir", "cat", "mv", "rm", "sleep", "date", - "chmod"): + "chmod", "ps"): found = shutil.which(tool) assert found, tool (self.fakebin / tool).symlink_to(found) @@ -179,20 +184,26 @@ def test_a_killed_run_removes_its_run_root_and_stops_the_suite(tmp_path, sig, rc assert proc.returncode == rc assert time.time() - started < 20, "the trap waited for the suite" assert not run_root.exists(), "the run root outlived a killed run" - suite_pid = int(rec["pid"]) - deadline = time.time() + 10 - while time.time() < deadline: - try: - os.kill(suite_pid, 0) - except ProcessLookupError: - break - time.sleep(0.1) - else: - raise AssertionError("the suite outlived its wrapper") + child = Path(str(box.log) + ".child").read_text().strip() + for pid, what in ((int(rec["pid"]), "the suite"), (int(child), "the suite's own child")): + deadline = time.time() + 10 + while time.time() < deadline and _running(pid): + time.sleep(0.1) + if _running(pid): + os.kill(pid, signal.SIGKILL) + raise AssertionError(f"{what} outlived its wrapper") if not flock: assert not (box.lockdir / "openrepotools-pytest.lock.d").exists(), "the lock was kept" +def _running(pid: int) -> bool: + """Alive and not a zombie: an orphan's zombie answers `kill -0` until + whoever adopted it reaps it.""" + proc = subprocess.run(["ps", "-o", "stat=", "-p", str(pid)], capture_output=True, text=True) + state = proc.stdout.strip() + return bool(state) and not state.startswith("Z") + + def test_the_repository_venv_is_used_when_it_has_pytest(tmp_path): box = Box(tmp_path) venv_bin = box.cache / "openRepoTools" / "venvs" / "openRepoTools" / "bin" From b0ea54ef0d6c6ec6d567f25af9eb5113a9c4aace Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:34:05 +0000 Subject: [PATCH 17/29] Offer and seed site-packages/ with the other bytecode rules Copilot on #169: the refusal classifies any path under a site-packages directory, since that is a virtual environment under any name, but neither the offered lines nor the seeded ones covered it. Following the offer therefore left the refusal standing. `site-packages/` now joins BYTECODE_IGNORES and WIP_HYGIENE_LINES. The shell case adds a venv named env-x: it is refused, `site-packages/` is among the lines offered, and it is still refused until that line is added. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lanes-edit.sh | 3 ++- openRepoTools | 1 + tests/test_lane_helpers.sh | 11 +++++++++-- tests/test_wip_init_command.py | 4 ++-- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/lanes-edit.sh b/lanes-edit.sh index 27f5839..073d678 100755 --- a/lanes-edit.sh +++ b/lanes-edit.sh @@ -1824,7 +1824,8 @@ BYTECODE_IGNORES='__pycache__/ .ruff_cache/ node_modules/ .venv/ -venv/' +venv/ +site-packages/' bytecode_in_pathspec() { # ... bip_repo="${1-}"; shift || : [ "$#" -gt 0 ] || return 0 diff --git a/openRepoTools b/openRepoTools index de1f7dd..f7da407 100755 --- a/openRepoTools +++ b/openRepoTools @@ -2332,6 +2332,7 @@ WIP_HYGIENE_LINES=( "node_modules/" ".venv/" "venv/" + "site-packages/" ) # The hygiene block, less every line `$1` already carries; nothing at all when diff --git a/tests/test_lane_helpers.sh b/tests/test_lane_helpers.sh index 6f0803b..70a6f61 100755 --- a/tests/test_lane_helpers.sh +++ b/tests/test_lane_helpers.sh @@ -14757,10 +14757,12 @@ echo "== opensoft/openRepoTools#162: bytecode never rides a workspace commit ==" # them, and once they are there the same pathspec stages nothing of the kind. B162_REL="handoffs/sandbox/attachments/b162" B162="$WIP/$B162_REL" -mkdir -p "$B162/r1/__pycache__" "$B162/r1/venv/lib/python3.12/site-packages" +mkdir -p "$B162/r1/__pycache__" "$B162/r1/venv/lib/python3.12/site-packages" \ + "$B162/r1/env-x/lib/python3.12/site-packages" printf 'findings\n' > "$B162/r1/notes.md" printf 'bytecode\n' > "$B162/r1/__pycache__/m.cpython-312.pyc" printf 'pkg\n' > "$B162/r1/venv/lib/python3.12/site-packages/pkg.py" +printf 'pkg\n' > "$B162/r1/env-x/lib/python3.12/site-packages/other.py" B162_IGN="" [ -f "$WIP/.gitignore" ] && B162_IGN="$(cat "$WIP/.gitignore")" run "$E" pathspec-check "$B162_REL" @@ -14769,6 +14771,8 @@ has "…naming the bytecode" "$out" "$B162_REL/r1/__pycache__/m.cpython-312.pyc has "…and the virtual environment" "$out" "$B162_REL/r1/venv/lib/python3.12/site-packages/pkg.py" hasnt "…and not the attachment's own notes" "$out" "notes.md" has "…and offers the ignore lines the workspace lacks" "$err" "'__pycache__/'" +has "…a venv under any name included, by its site-packages" "$out" "$B162_REL/r1/env-x/lib/python3.12/site-packages/other.py" +has "…and the line that covers it is among those offered" "$err" "'site-packages/'" run "$E" pathspec-check "$B162_REL/r1/notes.md" is "a pathspec with nothing of the kind passes" "$rc" 0 b162_head="$(git -C "$WIP" rev-parse HEAD)" @@ -14779,7 +14783,10 @@ is "…and commits nothing" "$(git -C "$WIP" rev-parse HEAD)" "$b162_head" is "…and stages nothing" "$(git -C "$WIP" diff --cached --name-only | grep -c . || :)" 0 printf '%s\n' '__pycache__/' '*.pyc' 'venv/' >> "$WIP/.gitignore" run "$E" pathspec-check "$B162_REL" -is "with the ignore lines in place it stages nothing of the kind" "$rc" 0 +is "without the site-packages line a venv under another name is still refused" "$rc" 2 +printf '%s\n' 'site-packages/' >> "$WIP/.gitignore" +run "$E" pathspec-check "$B162_REL" +is "with the offered lines in place it stages nothing of the kind" "$rc" 0 rm -rf "$WIP/handoffs/sandbox" if [ -n "$B162_IGN" ]; then printf '%s\n' "$B162_IGN" > "$WIP/.gitignore"; else rm -f "$WIP/.gitignore"; fi git -C "$WIP" checkout -q -- .gitignore 2>/dev/null || : diff --git a/tests/test_wip_init_command.py b/tests/test_wip_init_command.py index 595edcb..933e4c4 100644 --- a/tests/test_wip_init_command.py +++ b/tests/test_wip_init_command.py @@ -2077,13 +2077,13 @@ def test_a_commit_that_fails_prints_what_git_said(tmp_path): #: What `openRepoTools`' `WIP_HYGIENE_LINES` adds to the seeded `.gitignore`. HYGIENE_LINES = ("__pycache__/", "*.pyc", "*.pyo", ".pytest_cache/", ".mypy_cache/", - ".ruff_cache/", "node_modules/", ".venv/", "venv/") + ".ruff_cache/", "node_modules/", ".venv/", "venv/", "site-packages/") def test_the_seeded_gitignore_ignores_bytecode_caches_and_venvs(tmp_path): """Cause 19 of the cleanup analysis: generated bytecode was committed into handoff attachments. The seed's `.gitignore` is the template's AND the - nine hygiene lines, so `git add` of an attachment folder never picks a + ten hygiene lines, so `git add` of an attachment folder never picks a `__pycache__` up; the re-run compares the same bytes and has nothing to do.""" home = tmp_path / "home" From b422d8f7e50910f71e9d4f347e318e6787928e8e Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:34:05 +0000 Subject: [PATCH 18/29] Report duplicate clones with no named checkout, over one estate Copilot on #169, two report defects: - Two clones of one origin, neither named for the repository, were both treated as canonical, so the duplicate finding vanished. The checkout is now chosen deterministically (the one named for the repository, else the shallowest), and every other clone is reported, with a note when the choice was ambiguous. - `status --all` was pointed at the estate only when --estate was given. It now always gets the root the report resolved, and lane-start passes its own $PROJECTS_ROOT to the daily run. The report's hygiene check wants site-packages/ too. Each new assertion fails against the previous head. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- docs/README-lanes.md | 10 ++++++---- lane-start | 6 +++++- lane-worktrees | 30 ++++++++++++++++++----------- tests/test_lane_worktrees_report.py | 30 +++++++++++++++++++++++++++-- 4 files changed, 58 insertions(+), 18 deletions(-) diff --git a/docs/README-lanes.md b/docs/README-lanes.md index 96518ff..1c886ad 100644 --- a/docs/README-lanes.md +++ b/docs/README-lanes.md @@ -3521,8 +3521,8 @@ and before the launch: the first start of a UTC day takes the stamp `${XDG_STATE_HOME:-~/.local/state}/openRepoTools/report-.stamp` (an atomic `set -C` create, so two starts at once run one report), removes older stamps, and starts the report DETACHED — stdin, stdout and stderr closed, -in a subshell that exits at once — writing -`…/openRepoTools/reports/.md`. Every step either works or is skipped in +in a subshell that exits at once — over the start's own `$PROJECTS_ROOT`, +writing `…/openRepoTools/reports/.md`. Every step either works or is skipped in silence: the report never delays a start and never fails one. `--dry-run` starts none; `LANE_WORKTREES_REPORT=off` is the switch (both suites set it). @@ -3534,7 +3534,8 @@ The sweep retires what is already there; these keep more from arriving. `${XDG_CACHE_HOME:-~/.cache}/openRepoTools/pycache`, turns pytest's cache off, and roots every temporary directory of a run under `${XDG_STATE_HOME:-~/.local/state}/openRepoTools/tmp/-/`, which its - EXIT trap removes however the run ends. A SIGKILL is the one end no trap sees; + EXIT trap removes however the run ends — after stopping the suite's whole + process group, so nothing it started outlives the run root or the lock. A SIGKILL is the one end no trap sees; the pid in the name is what lets `--include-sandboxes` tell its owner is gone. * **One virtual environment per repository, outside the estate:** `${XDG_CACHE_HOME:-~/.cache}/openRepoTools/venvs//`. `openRepoTools @@ -3548,7 +3549,8 @@ The sweep retires what is already there; these keep more from arriving. still found inside the estate. * **The workspace repository never carries bytecode.** `openRepoTools wip init` seeds `__pycache__/`, `*.pyc`, `*.pyo`, `.pytest_cache/`, `.mypy_cache/`, - `.ruff_cache/`, `node_modules/`, `.venv/` and `venv/` into the new + `.ruff_cache/`, `node_modules/`, `.venv/`, `venv/` and `site-packages/` (a + virtual environment under any other name) into the new workspace's `.gitignore` (a line the template already carries is not repeated). Every commit `lanes-edit.sh` makes asks git what its pathspec would stage (`git add --dry-run`) and REFUSES, exit **2**, nothing staged, when any diff --git a/lane-start b/lane-start index 09a8aa6..d152aff 100755 --- a/lane-start +++ b/lane-start @@ -3932,8 +3932,12 @@ lane_daily_report() { [ "$ldr_old" = "$ldr_state/report-$ldr_day.stamp" ] || rm -f -- "$ldr_old" 2>/dev/null || : done ldr_out="$ldr_state/reports/$(date -u +%Y%m%dT%H%M%SZ).md" + # THE ESTATE THIS START READS: `$PROJECTS_ROOT` as resolved above, named to + # the report rather than left for it to find again. + ldr_estate="" + [ -d "$PROJECTS_ROOT" ] && ldr_estate="$PROJECTS_ROOT" ( "$ldr_bin" sweep --all --dry-run --report --post "$ldr_out" \ - /dev/null 2>&1 & ) 2>/dev/null || : + ${ldr_estate:+--estate "$ldr_estate"} /dev/null 2>&1 & ) 2>/dev/null || : step "the estate's daily report is running, detached: $ldr_out" return 0 } diff --git a/lane-worktrees b/lane-worktrees index 219c51b..4b91cf2 100755 --- a/lane-worktrees +++ b/lane-worktrees @@ -2960,13 +2960,16 @@ class Report: continue name = key.rsplit("/", 1)[-1].replace(".git", "") named = [g for g in group if os.path.basename(g).lower() == name] - if len(named) == 1: - out.append(named[0]) - for g in group: - if g != named[0]: - self.duplicates[g] = named[0] - else: - out.extend(group) + # ONE CHECKOUT PER ORIGIN, CHOSEN DETERMINISTICALLY: the one named + # for the repository, the shallowest of several so named, and the + # shallowest of all where none is. The rest are duplicates - never + # all of them canonical, which would hide the finding. + pool = named or group + keep = sorted(pool, key=lambda g: (g.count(os.sep), g))[0] + out.append(keep) + for g in group: + if g != keep: + self.duplicates[g] = (keep, len(named) != 1) return sorted(out) def _foreign(self, clones: list, canonical: list) -> None: @@ -2977,7 +2980,10 @@ class Report: if container: why = f"a CLONE inside {container}, where a lane makes worktrees" elif c in self.duplicates: - why = f"a second clone of {self.duplicates[c]}'s origin" + first, ambiguous = self.duplicates[c] + why = f"a second clone of {first}'s origin" + ( + " (no single clone is named for the repository; the shallowest is " + "taken as its checkout)" if ambiguous else "") if not why: continue last = git_out(c, "log", "-1", "--format=%cs") or "?" @@ -3233,7 +3239,7 @@ class Report: except OSError: have = set() want = ["__pycache__/", "*.pyc", "*.pyo", ".pytest_cache/", ".mypy_cache/", - ".ruff_cache/", "node_modules/", ".venv/", "venv/"] + ".ruff_cache/", "node_modules/", ".venv/", "venv/", "site-packages/"] missing = [w for w in want if w not in have] if missing: rows.append([os.path.join(wroot, ".gitignore"), "lacks " + " ".join(missing), @@ -3261,8 +3267,10 @@ class Report: why = "no `status` beside this command or on PATH" else: env = dict(os.environ) - if self.opts.estate: - env["PROJECTS_DIR"] = self.opts.estate + if self.root: + # THE ESTATE THIS REPORT READ, however it was resolved: from + # `--estate`, `$PROJECTS_ROOT` or the home default. + env["PROJECTS_DIR"] = self.root proc = run([exe, "--all"], env=env, timeout=600) if proc.returncode not in (0, 1): why = f"`status --all` exited {proc.returncode}: " + _first_line(proc.stderr) diff --git a/tests/test_lane_worktrees_report.py b/tests/test_lane_worktrees_report.py index bb0e43d..f07f54b 100644 --- a/tests/test_lane_worktrees_report.py +++ b/tests/test_lane_worktrees_report.py @@ -50,6 +50,7 @@ root main /x/repoA - shape pin 71cf5de is 1 commit(s) behind opensoft/openRepoShape main OUT +printf ' - estate read at %s\n' "${PROJECTS_DIR:-unset}" exit 1 """ @@ -75,6 +76,13 @@ def build_estate(e) -> dict: e.git("clone", "-q", LW.GH_URL, f["clone"], cwd=e.root) f["dup"] = e.projects / "repoA-copy" e.git("clone", "-q", LW.GH_URL, f["dup"], cwd=e.root) + # ... and two clones of one origin NEITHER of which is named for it: one + # is still the checkout, the other still a finding. + c_origin = e.root / "remotes" / "repoC.git" + e.git("init", "-q", "--bare", "-b", "main", c_origin, cwd=e.root) + for name in ("c-one", "c-two"): + e.git("clone", "-q", c_origin, e.projects / name, cwd=e.root) + f["c-two"] = e.projects / "c-two" # ORPHANED: a tree in the inventory of a lane whose snapshot is CLOSED. f["orphan"] = e.projects / ".lane-worktrees" / "repoA-9" / "left" e.git("worktree", "add", "-q", "-b", "feat/left", f["orphan"], "origin/main") @@ -168,12 +176,13 @@ def build_estate(e) -> dict: return f -def report(e, *args, env=None): +def report(e, *args, env=None, estate=True): e.write_spec() child = dict(e.env) child.update(env or {}) + where = ["--estate", str(e.projects)] if estate else [] return subprocess.run([sys.executable, str(LW.LW), "sweep", "--all", "--dry-run", "--report", - "--estate", str(e.projects), *args], + *where, *args], capture_output=True, text=True, env=child, timeout=300, cwd=str(e.root), stdin=subprocess.DEVNULL) @@ -197,6 +206,9 @@ def test_the_report_finds_every_kind_of_leftover_and_changes_nothing(tmp_path): assert str(f["clone"]) in foreign and "a CLONE inside" in foreign assert str(f["dup"]) in foreign and "a second clone of" in foreign assert f"{e.checkout} ·" not in foreign, "the estate's own checkout is not foreign" + assert f"{f['c-two']} · a second clone of {e.projects / 'c-one'}'s origin (no single" \ + in foreign + assert f"{e.projects / 'c-one'} ·" not in foreign orphans = section(out, "Orphaned worktrees of ENDED lanes") assert str(f["orphan"]) in orphans and "repoA-9" in orphans assert f"{f['nested-orphan']} \u00b7 repoB-2 \u00b7 its snapshot is CLOSED" in orphans @@ -223,10 +235,23 @@ def test_the_report_finds_every_kind_of_leftover_and_changes_nothing(tmp_path): assert "lacks __pycache__/" in hygiene and "bytecode path(s) in history" in hygiene status = section(out, "`status --all` findings") assert "shape pin 71cf5de is 1 commit(s) behind" in status + assert f"estate read at {e.projects}" in status assert "| rescue branches (all ages) | 1 |" in out assert before == after, sorted(set(before.items()) ^ set(after.items()))[:10] +def test_status_reads_the_estate_the_report_resolved(tmp_path): + """Copilot on #169: with no `--estate` - lane-start's daily run - the + report resolves `$PROJECTS_ROOT`, and `status --all` is pointed at that + same estate, never at its own default.""" + e = LW.Estate(tmp_path) + build_estate(e) + proc = report(e, estate=False) + assert proc.returncode == 0, proc.stderr + assert f"over `{e.projects}`" in proc.stdout + assert f"estate read at {e.projects}" in section(proc.stdout, "`status --all` findings") + + def test_post_writes_a_file_or_comments_on_an_issue(tmp_path): e = LW.Estate(tmp_path) build_estate(e) @@ -314,6 +339,7 @@ def test_lane_start_runs_the_report_detached_once_a_day(tmp_path): pass assert calls.startswith("sweep --all --dry-run --report --post "), calls assert f"{state}/openRepoTools/reports/" in calls + assert calls.rstrip().endswith(f"--estate {box.home / 'projects'}"), calls day = _dt.datetime.now(_dt.timezone.utc).strftime("%Y%m%d") assert (state / "openRepoTools" / f"report-{day}.stamp").exists() assert "the estate's daily report is running, detached" in result.stderr From 3c0b1881d53b745fa69a622d9d3bddc9a12e745e Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:39:40 +0000 Subject: [PATCH 19/29] Take Copilot round 2's defects in the sweep Seven of round 2's eight. Each one let an act proceed on an unknown: - Another lane's inventory record or directory that cannot be read, or that names no path, now refuses the sweep. Who else claims a tree is read before the fetch, so the refusal writes nothing. - A submodule's ignored files (an .env) keep the tree. The tree's ignored archive covers the superproject only. - Ignored files are printed before the archive and listed again at removal. A change, an addition or a failed listing leaves the tree. - A prune candidate whose directory reappeared is left for the next sweep to classify. - A pytest sandbox's .lock is read again at the act, and a live pid leaves it. - `tracked()` returns None when a repository's index cannot be read, and such a cache is left. - A manifest entry whose file is gone makes the archive not whole. The eighth, unreadable /proc entries of same-account processes, is a design trade and is filed in #170. On Eagle, 9 such processes exist at any time (6 `timeout`, 3 `ssh-agent`), so reading them as "unknown" would keep every tree on every sweep. Four new or extended cases. Each fails against ae17742 and passes here. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- docs/README-lanes.md | 16 +++-- lane-worktrees | 110 +++++++++++++++++++++++++++++++---- tests/test_lane_worktrees.py | 56 +++++++++++++++++- 3 files changed, 165 insertions(+), 17 deletions(-) diff --git a/docs/README-lanes.md b/docs/README-lanes.md index 576a7c3..250d43c 100644 --- a/docs/README-lanes.md +++ b/docs/README-lanes.md @@ -3330,7 +3330,8 @@ it causes). A tree ANOTHER lane's inventory names is that lane's and is never taken from here, word or no word — and one BOTH inventories name is kept, since records are history and a path can be reused. An inventory record that cannot be read (no path, an unknown schema) refuses the sweep, exit **2**: its tree's owner -is unknown. The lane's own checkout is never a candidate. The lane name is +is unknown — and so does ANOTHER lane's record or inventory directory that cannot +be read, before anything is fetched. The lane's own checkout is never a candidate. The lane name is resolved first (`lanes-edit.sh canon-lane`, Amendment 15), as `lane-start` and `lane-end` resolve it; an alias table that cannot be read refuses. @@ -3356,10 +3357,10 @@ Each tree is classified in this order; the first row that matches decides. | not in the lane's inventory | `foreign` | nothing (`--include-foreign --word …` applies the rows below) | | a process stands in it or holds a file open there (`/proc`, else `lsof`), a tmux pane's current path is in it, it is named by `--live`, the sweep was started inside it, or the session that recorded it is live | `live` | nothing, ever — asked again at the moment of the act | | liveness could not be read | `keep` | nothing | -| its directory is gone and git still registers it | `prune` | `git worktree remove ` — that one registration; a repository-wide `git worktree prune` is never run (it would unregister other lanes' and FOREIGN trees too) | +| its directory is gone and git still registers it | `prune` | `git worktree remove ` — that one registration; a repository-wide `git worktree prune` is never run (it would unregister other lanes' and FOREIGN trees too); a directory that reappeared meanwhile is left for the next sweep to classify | | its directory is gone and nothing registers it | `gone` | nothing; the inventory record is history | | another lane's inventory names it too | `keep` | nothing: which lane owns it now is a person's to say | -| a registration someone LOCKED; a submodule with uncommitted work or a commit no remote holds; a repository nested inside it; a CLONE with a branch or a stash origin lacks, or whose git directory linked worktrees share | `keep` | nothing, and the line says which | +| a registration someone LOCKED; a submodule with uncommitted work, ignored files no commit carries (an `.env`), or a commit no remote holds; a repository nested inside it; a CLONE with a branch or a stash origin lacks, or whose git directory linked worktrees share | `keep` | nothing, and the line says which | | something leans on it — another repository's `objects/info/alternates`, or a remote whose URL is its path | `load-bearing` | nothing; the line names every dependent and the remedy (`git repack -a -d`, then drop the alternates or re-point the remote) | | no `origin` | `keep` (`bundle+remove` with `--bundle`) | a bundle is its only rescue | | dirty or untracked work | `wip-rescue+remove` | `git add -A` into a COPY of its index, `commit-tree` on its head, `rescue//-` pushed and seen on origin, a bundle, then removed | @@ -3392,6 +3393,8 @@ files never move. **Ignored files that are not caches or build output** (an `.env`, a local config) are archived to `-ignored.tar.gz` before a tree goes: a commit cannot carry them, and the rule is that nothing goes that is not first saved. +They are listed again at the moment of removal, and a tree whose ignored files +changed, appeared or went since the archive was taken is left. Build output (`target`, `build`, `dist`, `.tox`, `.next`, `*.egg-info`, …) is generated and is not archived; more than `ignored_archive_mb` of the rest leaves the tree in place for a person. @@ -3402,8 +3405,8 @@ the tree in place for a person. |---|---|---| | `--branches` | local branches no worktree holds, in the lane's checkout | merged by PR evidence: `delete` (`delete+remote` where the `Lane:` trailer is this lane's and origin's tip is inside the PR), each at the SHA judged and only after a fetch of that checkout that worked under `--yes`; an open PR's branch: `keep`; unmerged with a missing, diverged, unpushed or unreadable upstream: `list` with its tip, distance from `origin/main`, last commit date and owner — never deleted; `main`, `master` and `rescue/*`: never touched | | `--include-scratch` | `.lane-worktrees//*-scratch`, `briefs/`, `bin/` that are no checkout | `archive+remove`: tar (caches left out) and sha256, then removed — only if no writer arrived and nothing outside the caches changed while the tar ran | -| `--include-caches` | `__pycache__`, `.pytest_cache`, `.mypy_cache`, `.ruff_cache`, `node_modules`, any directory with `pyvenv.cfg` (and `venv/`, `.venv/` with an activate script), under `.lane-worktrees//**` and the lane's inventory trees | `remove`, without archiving — never one git tracks, one inside scratch being archived, or one in a FOREIGN tree, a live one, one whose liveness is unknown or one left by its own act; the owning tree's liveness is asked again just before | -| `--include-sandboxes` | `tmp.*` and `pytest-of-$USER/pytest-*` in `/tmp` and `$TMPDIR` (or `LANE_WORKTREES_SANDBOX_ROOTS`), owned by this account | `remove` where the owning process is gone: a pytest `.lock` naming a dead pid, and no live process standing in it, holding it open or naming it in its environment; a `tmp.*` younger than `sandbox_min_age_minutes` is left | +| `--include-caches` | `__pycache__`, `.pytest_cache`, `.mypy_cache`, `.ruff_cache`, `node_modules`, any directory with `pyvenv.cfg` (and `venv/`, `.venv/` with an activate script), under `.lane-worktrees//**` and the lane's inventory trees | `remove`, without archiving — never one git tracks or whose tracking cannot be read, one inside scratch being archived, or one in a FOREIGN tree, a live one, one whose liveness is unknown or one left by its own act; the owning tree's liveness is asked again just before | +| `--include-sandboxes` | `tmp.*` and `pytest-of-$USER/pytest-*` in `/tmp` and `$TMPDIR` (or `LANE_WORKTREES_SANDBOX_ROOTS`), owned by this account | `remove` where the owning process is gone: a pytest `.lock` naming a dead pid (read again at the act), and no live process standing in it, holding it open or naming it in its environment; a `tmp.*` younger than `sandbox_min_age_minutes` is left | | `--links` | every symlink under the estate and every worktree `.git` gitdir pointer | `broken`, listed with target and age; nothing changes | | `--bundle` | every tree acted on | a `git bundle` beside the archive (always, for a rescue) | @@ -3433,7 +3436,8 @@ origin AT THE RESCUED SHA (`git ls-remote`); an archive whose rescue branch is gone from origin, or was remade at another commit, is the only copy and is NEVER expired. Nor is an archive that is not WHOLE — no `MANIFEST.sha256`, a file it does not list or whose digest differs (a sweep interrupted after a bundle), or a -`rescues.tsv` that is missing or has a row that is not origin, branch and SHA. What expired is appended to +`rescues.tsv` that is missing or has a row that is not origin, branch and SHA, +or a manifest entry whose file is gone. What expired is appended to `sweeps/EXPIRED.log`. `${XDG_CONFIG_HOME:-$HOME/.config}/openRepoTools/sweep.conf` (or diff --git a/lane-worktrees b/lane-worktrees index 5036229..0a189e7 100755 --- a/lane-worktrees +++ b/lane-worktrees @@ -1175,6 +1175,14 @@ def tree_guards(tree: Tree) -> str: if dirt is None or dirt: return (f"submodule {parts[1]} holds uncommitted work (a commit in the tree " "cannot carry it) - commit it there first") + # ITS IGNORED WORK TOO: an `.env` in a submodule is archived by + # nobody - the tree's own archive lists the superproject's only. + extra = ignored_paths(spath) + if extra is None or extra: + return (f"submodule {parts[1]} holds ignored files no commit carries " + + (f"({extra[0]}{' and more' if len(extra) > 1 else ''})" if extra + else "(they could not be listed)") + + " - move them out first") on = git(spath, "for-each-ref", "--count=1", "--contains", "HEAD", "--format=%(refname)", "refs/remotes/") if on.returncode != 0 or not on.stdout.strip(): @@ -1202,7 +1210,12 @@ def ignored_work(tree: Tree): """Ignored paths that are NOT caches: a commit cannot carry them, so they are archived beside the bundle before the tree goes. None where git could not list them - which is never read as "there are none".""" - proc = git(tree.path, "ls-files", "--others", "--ignored", "--exclude-standard", + return ignored_paths(tree.path) + + +def ignored_paths(path: str): + """`ignored_work` for any checkout path - a submodule's included.""" + proc = git(path, "ls-files", "--others", "--ignored", "--exclude-standard", "--directory", "-z") if proc.returncode != 0: return None @@ -1214,7 +1227,7 @@ def ignored_work(tree: Tree): parts = rel.split("/") if any(p in CACHE_NAMES for p in parts): continue - full = os.path.join(tree.path, rel) + full = os.path.join(path, rel) if os.path.isdir(full) and (os.path.basename(rel) in VENV_NAMES or os.path.exists(os.path.join(full, "pyvenv.cfg"))): continue @@ -1325,12 +1338,14 @@ class Sweep: self._lane_gate() self.procs = ProcScan() self._discover() + # WHO ELSE CLAIMS THESE TREES is read before anything is written: a + # claim that cannot be read refuses, and a refusal never fetches. + self._other_lanes() if self.opts.fetch and not self.refusal: self._fetch_all() self.deps = dependents_map(estate_repos(self.estate_root)) if self.estate_root else {} for tree in self.trees: read_tree(tree) - self._other_lanes() for tree in self.trees: self._liveness(tree) if tree.is_git and tree.exists: @@ -1418,10 +1433,20 @@ class Sweep: if os.environ.get("LANES_LANE_STATE_ROOT"): parents.add(os.environ["LANES_LANE_STATE_ROOT"]) claimed: dict = {} - for parent in parents: + + def unreadable(what: str) -> None: + # A CLAIM THAT CANNOT BE READ IS NOT "NO CLAIM" (Amendment 7(d)). + self.refusal = self.refusal or ( + f"{what} could not be read, so whether another lane claims these trees is " + "unknown; read it, or move it aside if it is debris") + + for parent in sorted(parents): try: lanes = os.listdir(parent) + except FileNotFoundError: + continue except OSError: + unreadable(f"the lane-state directory {parent}") continue for other in lanes: if other == self.lane: @@ -1429,17 +1454,28 @@ class Sweep: tdir = os.path.join(parent, other, "trees") try: files = os.listdir(tdir) + except (FileNotFoundError, NotADirectoryError): + continue # that lane has no inventory: no claim except OSError: + unreadable(f"lane {other}'s inventory {tdir}") continue for fname in files: + if fname.startswith(".") or not fname.endswith(".yaml"): + continue + path = "" try: with open(os.path.join(tdir, fname), encoding="utf-8") as fh: for line in fh: if line.startswith("path: "): - claimed[real(line[6:].strip())] = other + path = line[6:].strip() break - except OSError: + except (OSError, UnicodeDecodeError): + unreadable(f"lane {other}'s inventory record {fname}") + continue + if not path: + unreadable(f"lane {other}'s inventory record {fname} (it names no path)") continue + claimed[real(path)] = other for tree in self.trees: if tree.real not in claimed: continue @@ -1766,8 +1802,8 @@ class Sweep: # the tree, not in its __pycache__. if self.procs is None or not self.procs.available or self.procs.holders(owner): continue - if tracked(path): - continue + if tracked(path) is not False: + continue # tracked, or its tracking is unknown it = Item("cache", path, "remove", "a dependency cache or bytecode: it holds no " "work and is rebuilt", False) it.size = size @@ -1944,6 +1980,13 @@ class Sweep: t.disposition = "live" if not why.startswith("liveness") else "keep" t.result = f"left: {why}" return + if t.disposition == "prune" and os.path.lexists(t.path): + # IT CAME BACK while the sweep ran (a remount, a restore): it is a + # tree again, and it is classified afresh by the next sweep, never + # removed on a reading taken while it was gone. + t.disposition = "keep" + t.result = "left: the directory reappeared after it was read as gone; sweep again" + return if t.disposition == "prune": # THAT ONE REGISTRATION: `git worktree prune` would unregister every # missing tree of the repository - another lane's, a FOREIGN one, @@ -2011,6 +2054,9 @@ class Sweep: raise _ActFailed("its ignored files could not be listed, so whether it holds an " ".env or local configuration no commit carries is unknown; left " "in place" + (f"; its rescue is at {t.rescue}" if t.rescue else "")) + # TAKEN BEFORE THE ARCHIVE and compared at removal, so a change made + # while the tar ran leaves the tree too. + ignored_before = ignored_print(t.path, extra) if extra: size = sum(dir_size(os.path.join(t.path, r)) if os.path.isdir(os.path.join(t.path, r)) else os.lstat(os.path.join(t.path, r)).st_size for r in extra @@ -2031,6 +2077,14 @@ class Sweep: if why: raise _ActFailed(f"{why} - it arrived while the rescue was made; left in place" + (f"; its rescue is at {t.rescue}" if t.rescue else "")) + # THE IGNORED FILES ARE THE ONES ARCHIVED: none changed, appeared or + # went since the list was taken - neither `git status` nor the WIP + # snapshot sees an `.env`. + again_ignored = ignored_work(t) + if again_ignored is None or ignored_print(t.path, again_ignored) != ignored_before: + raise _ActFailed("its ignored files changed after they were archived (or could not " + "be listed again); left in place" + + (f"; its rescue is at {t.rescue}" if t.rescue else "")) head_now = git_out(t.path, "rev-parse", "-q", "--verify", "HEAD") or "" if head_now != t.head: raise _ActFailed(f"HEAD moved from {short(t.head) or 'none'} to " @@ -2201,6 +2255,13 @@ class Sweep: rmtree(i.path) i.result = "removed" elif i.kind == "sandbox" and i.disposition == "remove": + # ITS OWNER IS ASKED AGAIN: a pytest `.lock` may name a live pid + # now - a run that works outside the directory holds nothing in it. + if i.extra.get("kind") == "pytest": + owner = _lock_pid(os.path.join(i.path, ".lock")) + if owner and pid_alive(owner): + i.disposition, i.result = "live", f"left: its .lock names live pid {owner}" + return fresh = ProcScan() fresh.load_environ() if not fresh.available or fresh.holders(i.path): @@ -2354,6 +2415,23 @@ def _gitdir_activity(path: str) -> float: return newest +def ignored_print(base: str, rels: list) -> list: + """The ignored paths `rels` under `base`, every file in them with its size + and mtime: what the ignored archive carries, compared again at removal.""" + out = [] + for rel in sorted(rels): + full = os.path.join(base, rel) + if os.path.isdir(full) and not os.path.islink(full): + out += [(rel, *row) for row in content_print(full)] + else: + try: + st = os.lstat(full) + out.append((rel, "", st.st_size, st.st_mtime_ns)) + except OSError: + out.append((rel, "", -1, -1)) + return out + + def content_print(root: str) -> list: """Every non-cache file under `root` with its size and mtime - what an archive of it carries, so a change made while it was archived shows.""" @@ -2397,10 +2475,18 @@ def write_manifest(d: str) -> None: fh.write("\n".join(sorted(rows, key=lambda r: r.split(" ", 1)[1])) + "\n") -def tracked(path: str) -> bool: +def tracked(path: str): + """True when git tracks something under `path`, False when it tracks + nothing there or `path` is in no repository, None when a repository's + index could not be read - which is never "untracked".""" parent = os.path.dirname(path.rstrip("/")) + inside = git(parent, "rev-parse", "--is-inside-work-tree") + if inside.returncode != 0 or inside.stdout.strip() != "true": + return False proc = git(parent, "ls-files", "-z", "--", os.path.basename(path.rstrip("/"))) - return proc.returncode == 0 and bool(proc.stdout.strip("\0")) + if proc.returncode != 0: + return None + return bool(proc.stdout.strip("\0")) def find_caches(root: str) -> list: @@ -2573,6 +2659,7 @@ def sandbox_items(procs: ProcScan, conf: dict) -> list: it = Item("sandbox", path, "remove", f"a {kind} sandbox whose owning process is gone" + (f" (pid {pid})" if pid else ""), False) it.size = dir_size(path) + it.extra["kind"] = kind out.append(it) return out @@ -2753,6 +2840,9 @@ def archive_ledger(adir: str): listed[rel] = digest except OSError: return None, "it has no readable MANIFEST.sha256 (the sweep that made it did not finish)" + for rel in listed: + if not os.path.isfile(os.path.join(adir, rel)): + return None, f"{rel} is in its MANIFEST.sha256 and no longer in it" for base, _dirs, files in os.walk(adir): for name in files: full = os.path.join(base, name) diff --git a/tests/test_lane_worktrees.py b/tests/test_lane_worktrees.py index 587951f..e9064a9 100644 --- a/tests/test_lane_worktrees.py +++ b/tests/test_lane_worktrees.py @@ -673,6 +673,55 @@ def test_a_tree_another_lanes_inventory_also_names_is_kept(estate): assert tree.is_dir() +def test_an_unreadable_claim_of_another_lane_refuses_before_any_write(estate): + """Copilot round 2 on #168: a sibling inventory record that cannot be + read is an unknown claim, never "no claim" - refused, and refused before + the fetch.""" + estate.worktree("w", "feat/w") + other = estate.root / "lane-state" / "repoA-7" / "trees" + other.mkdir(parents=True) + (other / "c1.yaml").write_text("schema: 1\n") + before = snapshot(estate.root, skip=("helper.log", "helper-spec.json", "prs.json")) + proc = estate.sweep(LANE, "--yes", "--porcelain") + assert proc.returncode == 2, proc.stdout + proc.stderr + assert "lane repoA-7's inventory record c1.yaml (it names no path)" in proc.stdout + assert snapshot(estate.root, skip=("helper.log", "helper-spec.json", "prs.json")) == before + + +def test_a_submodules_ignored_files_keep_its_tree(estate): + """Copilot round 2 on #168: an `.env` in a submodule is in no archive - + the tree's ignored-file archive lists the superproject's only.""" + sub_origin = estate.remotes / "sub.git" + estate.git("init", "-q", "--bare", "-b", "main", sub_origin, cwd=estate.root) + seed = estate.root / "sub-seed" + estate.git("clone", "-q", sub_origin, seed, cwd=estate.root) + estate.commit(seed, "sub", {".gitignore": ".env\n", "s.txt": "s\n"}) + estate.git("push", "-q", "origin", "main", cwd=seed) + tree = estate.worktree("withsub", "feat/withsub", record=False) + estate.git("submodule", "add", "-q", str(sub_origin), "sub", cwd=tree) + estate.commit(tree, "add sub") + estate.git("push", "-q", "-u", "origin", "feat/withsub", cwd=tree) + (tree / "sub" / ".env").write_text("SECRET=1\n") + estate.record(tree) + rows = rows_of(estate.sweep(LANE, "--porcelain").stdout) + assert rows[str(tree)][0] == "keep", rows[str(tree)] + assert "submodule sub holds ignored files no commit carries (.env)" in rows[str(tree)][2] + + +def test_a_cache_whose_tracking_cannot_be_read_is_left(estate): + """Copilot round 2 on #168: an index git cannot read is no proof a cache + is untracked.""" + tree = estate.worktree("badindex", "feat/badindex") + estate.git("push", "-q", "-u", "origin", "feat/badindex", cwd=tree) + (tree / "__pycache__").mkdir() + (tree / "__pycache__" / "b.pyc").write_bytes(b"\0") + index = Path(estate.git("rev-parse", "--git-path", "index", cwd=tree)) + index = index if index.is_absolute() else tree / index + index.write_bytes(b"not an index") + proc = estate.sweep(LANE, "--include-caches", "--porcelain") + assert str(tree / "__pycache__") not in items_of(proc.stdout, "cache"), proc.stdout + + def test_a_clone_its_linked_worktrees_share_is_kept(estate): """Copilot on #168: removing a clone takes the git directory every worktree linked to it shares, a live writer's included.""" @@ -1000,6 +1049,10 @@ def test_expire_keeps_young_archives_and_any_whose_rescue_left_origin(estate): unfinished = _archive(estate, 93, None, manifest=False) stray = _archive(estate, 94, None) (stray / "x.bundle").write_bytes(b"bundle") + # ... and one whose manifest lists a file that is gone (round 2) + lost = _archive(estate, 95, None) + with open(lost / "MANIFEST.sha256", "a") as fh: + fh.write(f"{'0' * 64} lost.bundle\n") dry = estate.sweep("--expire", "--porcelain") assert dry.returncode == 3, dry.stdout + dry.stderr rows = {line.split("\t")[2]: line.split("\t")[1] for line in dry.stdout.splitlines()} @@ -1009,11 +1062,12 @@ def test_expire_keeps_young_archives_and_any_whose_rescue_left_origin(estate): assert rows[str(moved)] == "keep" and "not the rescued 1111111" in why[str(moved)] assert rows[str(unfinished)] == "keep" and "MANIFEST" in why[str(unfinished)] assert rows[str(stray)] == "keep" and "x.bundle" in why[str(stray)] + assert rows[str(lost)] == "keep" and "lost.bundle" in why[str(lost)] assert young.is_dir() and due.is_dir() yes = estate.sweep("--expire", "--yes") assert yes.returncode == 0, yes.stdout + yes.stderr assert young.is_dir() and orphan.is_dir() - assert moved.is_dir() and unfinished.is_dir() and stray.is_dir() + assert moved.is_dir() and unfinished.is_dir() and stray.is_dir() and lost.is_dir() assert not due.exists() and not plain.exists() log = (estate.state / "openRepoTools" / "sweeps" / "EXPIRED.log").read_text() assert str(due) in log and str(orphan) not in log From 65f73ed1762519fa516ae224f2380e5d8ead3449 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:43:16 +0000 Subject: [PATCH 20/29] Refuse bytecode that is already staged, not only what add would stage Copilot round 2 on #169: `git add --dry-run` says nothing about a path the index already holds at those bytes, yet the path-limited commit still takes it. bytecode_in_pathspec now also reads `git diff --cached --name-only --diff-filter=d` for the same pathspec. A staged deletion of bytecode is a cleanup and is let through. The shell case force-stages a .pyc after the ignore lines are in place: pathspec-check and the commit path both refuse it, and nothing is committed. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lanes-edit.sh | 13 ++++++++++--- tests/test_lane_helpers.sh | 11 +++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/lanes-edit.sh b/lanes-edit.sh index 073d678..910a814 100755 --- a/lanes-edit.sh +++ b/lanes-edit.sh @@ -1829,9 +1829,16 @@ site-packages/' bytecode_in_pathspec() { # ... bip_repo="${1-}"; shift || : [ "$#" -gt 0 ] || return 0 - git -C "$bip_repo" add --dry-run --ignore-missing -- "$@" 2>/dev/null | - sed -n "s/^add '\(.*\)'\$/\1/p" | - awk '{ + # WHAT THE ADD WOULD STAGE, AND WHAT IS STAGED ALREADY: `add --dry-run` says + # nothing of a path the index already holds at these bytes, and the commit + # takes it all the same. A staged DELETION of bytecode is a cleanup and is + # let through (`--diff-filter=d`). + { + git -C "$bip_repo" add --dry-run --ignore-missing -- "$@" 2>/dev/null | + sed -n "s/^add '\(.*\)'\$/\1/p" + git -C "$bip_repo" diff --cached --name-only --diff-filter=d -- "$@" 2>/dev/null + } | + awk '!seen[$0]++ { n = split($0, part, "/"); hit = 0 for (i = 1; i <= n; i++) if (part[i] == "__pycache__" || part[i] == ".pytest_cache" || diff --git a/tests/test_lane_helpers.sh b/tests/test_lane_helpers.sh index 70a6f61..7c7cd30 100755 --- a/tests/test_lane_helpers.sh +++ b/tests/test_lane_helpers.sh @@ -14787,6 +14787,17 @@ is "without the site-packages line a venv under another name is still refused" printf '%s\n' 'site-packages/' >> "$WIP/.gitignore" run "$E" pathspec-check "$B162_REL" is "with the offered lines in place it stages nothing of the kind" "$rc" 0 +# ALREADY STAGED (Copilot round 2 on #169): `add --dry-run` is silent about a +# path the index already holds, and the commit would carry it regardless. +git -C "$WIP" add -f -- "$B162_REL/r1/__pycache__/m.cpython-312.pyc" +run "$E" pathspec-check "$B162_REL" +is "bytecode already staged is refused too" "$rc" 2 +has "…and named" "$out" "$B162_REL/r1/__pycache__/m.cpython-312.pyc" +b162_head="$(git -C "$WIP" rev-parse HEAD)" +run env LANES_LANE=repoA-1 LANES_PATH="$B162_REL" "$E" commit "attach r1" +is "…by the commit path as well" "$rc" 2 +is "…which commits nothing" "$(git -C "$WIP" rev-parse HEAD)" "$b162_head" +git -C "$WIP" reset -q -- "$B162_REL" rm -rf "$WIP/handoffs/sandbox" if [ -n "$B162_IGN" ]; then printf '%s\n' "$B162_IGN" > "$WIP/.gitignore"; else rm -f "$WIP/.gitignore"; fi git -C "$WIP" checkout -q -- .gitignore 2>/dev/null || : From f5bccbdec978bf11607c36d456109ab83ef78b3b Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:43:16 +0000 Subject: [PATCH 21/29] Quote the venv paths in the command --install prints Copilot round 2 on #169: the command is meant to be pasted into a shell, and $XDG_CACHE_HOME is the person's to choose, spaces included. Both paths are now `printf %q`-quoted. The new case installs with a cache directory containing a space, and shlex reads each path back as one word. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- openRepoTools | 4 +++- tests/test_openrepotools_command.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/openRepoTools b/openRepoTools index f7da407..59f2063 100755 --- a/openRepoTools +++ b/openRepoTools @@ -1106,7 +1106,9 @@ venv_note() { say "openRepoTools: this repository's virtual environment is $venv (present; tests/run.sh runs from it when it has pytest)" else say "openRepoTools: this repository's virtual environment is $venv (not created; one per repository, outside the estate):" - say " python3 -m venv $venv && $venv/bin/python3 -m pip install pytest" + # QUOTED FOR THE SHELL IT IS PASTED INTO: `$XDG_CACHE_HOME` is the + # person's to choose, spaces and all. + say " python3 -m venv $(printf '%q' "$venv") && $(printf '%q' "$venv/bin/python3") -m pip install pytest" fi } diff --git a/tests/test_openrepotools_command.py b/tests/test_openrepotools_command.py index 250da45..1cb58f6 100644 --- a/tests/test_openrepotools_command.py +++ b/tests/test_openrepotools_command.py @@ -27,6 +27,7 @@ import hashlib import os import re +import shlex import shutil import stat import subprocess @@ -1903,3 +1904,17 @@ def test_install_names_the_repository_venv_and_whether_it_is_there(tmp_path): second = run_cmd("--install", home=tmp_path, env={"XDG_CACHE_HOME": str(cache)}) assert second.returncode == 0, second.stderr assert f"virtual environment is {venv} (present;" in second.stdout + + +def test_the_printed_venv_command_survives_a_path_with_a_space(tmp_path): + """Copilot round 2 on #169: the command is pasted into a shell, and the + cache directory is the person's to choose.""" + cache = tmp_path / "my cache" + venv = cache / "openRepoTools" / "venvs" / "openRepoTools" + result = run_cmd("--install", home=tmp_path, env={"XDG_CACHE_HOME": str(cache)}) + assert result.returncode == 0, result.stderr + (line,) = [ln.strip() for ln in result.stdout.splitlines() + if ln.strip().startswith("python3 -m venv ")] + words = shlex.split(line) + assert words[:4] == ["python3", "-m", "venv", str(venv)], words + assert words[5] == str(venv / "bin" / "python3"), words From dc7971cca5764ded1b0b7e81a1f385ebea2c6864 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:43:16 +0000 Subject: [PATCH 22/29] Report what the report could not read Copilot round 2 on #169: an unreadable lane snapshot or inventory record became an empty one, so a lane or tree dropped out of every section and the report could look clean. Reads now go through _ls and _record. A failure other than "not there", a tree record naming no path, or an inventory tree whose git status fails is a row in a new section, "Records the report could not read". The unused _sidecar reader is gone. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- docs/README-lanes.md | 5 +- lane-worktrees | 86 +++++++++++++++++------------ tests/test_lane_worktrees_report.py | 24 ++++++++ 3 files changed, 78 insertions(+), 37 deletions(-) diff --git a/docs/README-lanes.md b/docs/README-lanes.md index 7c1a2a6..833efa4 100644 --- a/docs/README-lanes.md +++ b/docs/README-lanes.md @@ -3512,6 +3512,7 @@ lane-worktrees sweep --all --dry-run --report [--estate ] [--post | | Evidence-shaped paths | untracked or ignored `junit*.xml`, `MANIFEST*`, `*-report.md`, `*REPORT*.md`, `canary-*`, `*-evidence`, `deployment-evidence` inside a repository, and a directory of reports beside the repositories — each a finding to move to the evidence root below | | Sweep archives past retention | `sweep --expire`'s own dry run: what WOULD expire, and what is past retention but kept because its rescue left origin. Expiring stays an act (`sweep --expire --yes`) | | Workspace repository hygiene | the workspace's `.gitignore` lines it lacks (with the one command that adds them), and bytecode already in its history (for a person's word: history is rewritten only on one) | +| Records the report could not read | a lane snapshot, inventory directory or record, or an inventory tree's `git status`, that could not be read — unknown is never absent, and the sections above may be missing what it holds | | `status --all` findings | the estate command's ahead/behind, fork, shape-pin and parked-record lines, reported as what they are and never counted as dirt (`LANE_WORKTREES_STATUS` names the command when it is not beside this one) | The summary table at its head also carries **the count of rescue branches** and @@ -3557,8 +3558,8 @@ The sweep retires what is already there; these keep more from arriving. virtual environment under any other name) into the new workspace's `.gitignore` (a line the template already carries is not repeated). Every commit `lanes-edit.sh` makes asks git what its pathspec would - stage (`git add --dry-run`) and REFUSES, exit **2**, nothing staged, when any - of it is bytecode, a cache, a dependency tree or a virtual environment — and + stage (`git add --dry-run`) or holds staged already, and REFUSES, exit **2**, + nothing staged, when any of it is bytecode, a cache, a dependency tree or a virtual environment — and offers the `.gitignore` lines a workspace without them lacks. Attachments are committed by hand, so the same question is a subcommand: diff --git a/lane-worktrees b/lane-worktrees index 1019754..af45515 100755 --- a/lane-worktrees +++ b/lane-worktrees @@ -2978,19 +2978,6 @@ def _lane_tree_roots(parents: list) -> list: return out -def _sidecar(path: str) -> dict: - out = {} - try: - with open(path, encoding="utf-8") as fh: - for line in fh: - key, sep, value = line.partition(": ") - if sep: - out[key.strip()] = value.strip() - except OSError: - pass - return out - - class Report: """`sweep --all --dry-run --report`: the estate's leftovers as one document. It reads every lane's #97 inventory, the workspace register and @@ -3004,6 +2991,7 @@ class Report: self.sections: list = [] self.counts: dict = {} self.now = time.time() + self.unreadable: dict = {} def add(self, title: str, rows: list, empty: str, note_text: str = "") -> None: self.sections.append((title, rows, empty, note_text)) @@ -3030,6 +3018,44 @@ class Report: self._archives() self._workspace_hygiene() self._status_all() + self.add("Records the report could not read", [[p, w] for p, w in + sorted(self.unreadable.items())], "none", + "UNKNOWN IS NEVER ABSENT: a lane snapshot, inventory or tree the report " + "could not read is one whose findings above may be missing.") + + # A READ THAT FAILED IS A FINDING, never an empty answer (Amendment 7(d)): + # an inventory the report could not read would otherwise make it clean. + def _ls(self, d: str) -> list: + try: + return sorted(os.listdir(d)) + except (FileNotFoundError, NotADirectoryError): + return [] + except OSError as exc: + self.unreadable.setdefault(d, f"could not be listed ({exc.strerror or exc})") + return [] + + def _record(self, path: str, need_path: bool = False) -> dict: + try: + with open(path, encoding="utf-8") as fh: + text = fh.read() + except FileNotFoundError: + return {} + except (OSError, UnicodeDecodeError) as exc: + self.unreadable.setdefault(path, "could not be read (" + + (getattr(exc, "strerror", "") or str(exc)) + ")") + return {} + out = {} + for line in text.splitlines(): + key, sep, value = line.partition(": ") + if sep: + out[key.strip()] = value.strip() + if need_path and not out.get("path"): + self.unreadable.setdefault(path, "an inventory record that names no path") + return out + + def _tree_records(self, tdir: str) -> list: + return [self._record(os.path.join(tdir, f), need_path=True) for f in self._ls(tdir) + if f.endswith(".yaml") and not f.startswith(".")] def _canonical(self, clones: list) -> list: """The estate's own checkouts: a clone outside every worktree @@ -3090,8 +3116,8 @@ class Report: last lane-kind line is ENDED or RETIRED.""" closed: dict = {} for d in self.state_dirs: - for lane in sorted(os.listdir(d)): - if _sidecar(os.path.join(d, lane, "lane-state.yaml")).get("state") == "CLOSED": + for lane in self._ls(d): + if self._record(os.path.join(d, lane, "lane-state.yaml")).get("state") == "CLOSED": closed[lane] = "its snapshot is CLOSED" wroot = self.helper.workspace_root() if self.helper.path else "" if wroot: @@ -3114,23 +3140,14 @@ class Report: seen = set() for d in self.state_dirs: for lane in sorted(closed): - tdir = os.path.join(d, lane, "trees") - try: - files = sorted(os.listdir(tdir)) - except OSError: - continue - for f in files: - path = _sidecar(os.path.join(tdir, f)).get("path", "") + for rec in self._tree_records(os.path.join(d, lane, "trees")): + path = rec.get("path", "") if path and os.path.isdir(path) and real(path) not in seen: seen.add(real(path)) rows.append([path, lane, closed[lane], "lane-worktrees sweep " + lane]) for lane, lroot in [(ln, r) for ln in sorted(closed) for r in self.tree_roots]: ldir = os.path.join(lroot, lane) - try: - names = sorted(os.listdir(ldir)) - except OSError: - continue - for n in names: + for n in self._ls(ldir): full = os.path.join(ldir, n) if os.path.isdir(full) and real(full) not in seen: seen.add(real(full)) @@ -3214,17 +3231,16 @@ class Report: rows.append([f"{c} rescue/{name}", f"rescue branch, {age} d", owner, "awaiting disposition"]) for d in self.state_dirs: - for lane in sorted(os.listdir(d)): - tdir = os.path.join(d, lane, "trees") - try: - files = sorted(os.listdir(tdir)) - except OSError: - continue - for f in files: - path = _sidecar(os.path.join(tdir, f)).get("path", "") + for lane in self._ls(d): + for rec in self._tree_records(os.path.join(d, lane, "trees")): + path = rec.get("path", "") if not path or not os.path.isdir(path): continue dirt = git_out(path, "status", "--porcelain") + if dirt is None: + self.unreadable.setdefault(path, f"lane {lane}'s inventory tree; git " + "status could not be read in it") + continue if not dirt: continue newest = _gitdir_activity(path) diff --git a/tests/test_lane_worktrees_report.py b/tests/test_lane_worktrees_report.py index f07f54b..60d80db 100644 --- a/tests/test_lane_worktrees_report.py +++ b/tests/test_lane_worktrees_report.py @@ -252,6 +252,30 @@ def test_status_reads_the_estate_the_report_resolved(tmp_path): assert f"estate read at {e.projects}" in section(proc.stdout, "`status --all` findings") +def test_a_record_the_report_cannot_read_is_a_finding_never_absence(tmp_path): + """Copilot round 2 on #169: an inventory or snapshot the report could not + read would otherwise make it clean.""" + e = LW.Estate(tmp_path) + build_estate(e) + trees = e.projects / ".lane-state" / "repoA-5" / "trees" + trees.mkdir(parents=True) + (trees / "c1.yaml").write_text("schema: 1\n") + locked = e.projects / ".lane-state" / "repoA-6" + locked.mkdir() + (locked / "lane-state.yaml").write_text("schema: 1\nstate: CLOSED\n") + (locked / "lane-state.yaml").chmod(0) + try: + proc = report(e) + finally: + (locked / "lane-state.yaml").chmod(0o644) + assert proc.returncode == 0, proc.stderr + found = section(proc.stdout, "Records the report could not read") + assert f"{trees / 'c1.yaml'} \u00b7 an inventory record that names no path" in found + if os.geteuid() != 0: + assert f"{locked / 'lane-state.yaml'} \u00b7 could not be read" in found + assert "| Records the report could not read | " in proc.stdout + + def test_post_writes_a_file_or_comments_on_an_issue(tmp_path): e = LW.Estate(tmp_path) build_estate(e) From 31db36f8b7aee3888a5a0a532d6e657b8c86dac1 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:48:40 +0000 Subject: [PATCH 23/29] Keep AGENTS.md and README.md within their line caps test_repo_hygiene caps AGENTS.md at 316 lines and README.md at 486, and both files were exactly at their caps on main. The run.sh paragraph had taken AGENTS.md to 330 and the venv sentence README.md to 491. The full text already lives in docs/README-lanes.md, so AGENTS.md now says the same thing in the three lines the old sentence took, and README.md says it on the line it extends. No cap is raised. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 20 +++----------------- README.md | 7 +------ 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d01e63c..a7f0e21 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -257,23 +257,9 @@ The run itself is minutes of bash and hundreds of `git` processes, and several lanes build in sibling worktrees of one checkout: the wrapper waits for any live run, takes `${TMPDIR:-/tmp}/openrepotools-pytest.lock` (`flock` where there is one, a `mkdir` lock on macOS, which has none), waits again -inside it, then runs `python3 -m pytest tests -q -p no:cacheprovider ---basetemp=/basetemp "$@"`. Every lane on one workstation must name -the SAME lock file or there is no lock, which is the whole reason the path is -written here as well as in the file. - -**A run leaves nothing beside the code** (opensoft/openRepoTools#162): bytecode -goes to `${XDG_CACHE_HOME:-~/.cache}/openRepoTools/pycache` -(`PYTHONPYCACHEPREFIX`), pytest's cache is off, and every temporary directory — -`--basetemp` and the `TMPDIR` the suite's own `mktemp`s use — is under one run -root, `${XDG_STATE_HOME:-~/.local/state}/openRepoTools/tmp/-/`, which -the wrapper's EXIT trap removes however the run ends (a TERM or an INT stops the -suite's whole process group first, so no `git` or shell suite it was waiting on -outlives the run root or the lock). The lock is computed BEFORE `TMPDIR` moves, so it is still the -workstation's. Where the repository's venv -(`${XDG_CACHE_HOME:-~/.cache}/openRepoTools/venvs/openRepoTools`) has `pytest`, -the suite runs from it, first on `PATH`, so the command line still reads -`python3 -m pytest`. +inside it, then runs `python3 -m pytest tests -q "$@"` (no cache or temp left in +the checkout: #162). Every lane on one workstation must name the SAME lock file +or there is no lock, which is the whole reason the path is written here too. Two measured defects on 2026-09-14 (opensoft/openRepoTools#51), both of them inside a guard that had been copied into four briefs: diff --git a/README.md b/README.md index 415c85d..18c8f7e 100644 --- a/README.md +++ b/README.md @@ -260,12 +260,7 @@ that would not take it, and a create needs a directory's search bit as well as its write bit. A refusal creates none of those directories either. `cp` follows a symlink, and an install through one leaves the command uninstalled and writes these bytes into whatever it points at. Then a `17 of 17 placed in ` line, -and the `export PATH=…` line if that directory is not on your `PATH`. Last, it -NAMES this repository's virtual environment — one per repository, outside the -estate, at `${XDG_CACHE_HOME:-~/.cache}/openRepoTools/venvs/openRepoTools/` -(opensoft/openRepoTools#162) — and says whether it is there; it never creates -one, since that needs `pip` and the network, and prints the two commands that -do. `tests/run.sh` runs the suite from it once it is there and has `pytest`. +and the `export PATH=…` line if that directory is not on your `PATH`. Last, it names this repository's virtual environment, `${XDG_CACHE_HOME:-~/.cache}/openRepoTools/venvs/openRepoTools/` (one per repository, outside the estate; #162), says whether it is there, and prints the two commands that make it: it never makes one, and `tests/run.sh` runs from it once it has `pytest`. It also places **fourteen things that are not files in that directory**: THREE SKILLS, `/handoff`, `/lane-swap` and `/restart`, each at From 30cc63ac0a3c6000c6bfb01bd0e069ba52927902 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:09:39 +0000 Subject: [PATCH 24/29] Assert the flock file only where there is flock Copilot on #169: macOS has no flock, so the wrapper takes the mkdir lock and never creates openrepotools-pytest.lock. The relocation case checked for that file unconditionally and would have failed the macOS job, which is the landing gate. Where there is no flock, the case now checks instead that the mkdir lock was released. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- tests/test_run_wrapper.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_run_wrapper.py b/tests/test_run_wrapper.py index c8b5ce7..64362ae 100644 --- a/tests/test_run_wrapper.py +++ b/tests/test_run_wrapper.py @@ -152,8 +152,14 @@ def test_a_run_relocates_bytecode_caches_and_temp_and_leaves_none_of_it(tmp_path assert rec["arg"][-2:] == ["-k", "lane_worktrees"] assert not run_root.exists(), "the run root outlived the run" assert box.runs() == [] - assert (box.lockdir / "openrepotools-pytest.lock").exists(), ( - "the lock is the workstation's, under the caller's TMPDIR, not the run's") + # THE LOCK IS THE WORKSTATION'S, under the caller's TMPDIR and not the + # run's: the `flock` file where there is `flock`, and on macOS, which has + # none, the `mkdir` lock beside it - released by the exit. + if shutil.which("flock"): + assert (box.lockdir / "openrepotools-pytest.lock").exists(), ( + "the lock is the workstation's, under the caller's TMPDIR, not the run's") + else: + assert not (box.lockdir / "openrepotools-pytest.lock.d").exists(), "the lock was kept" @pytest.mark.parametrize("sig,rc", [(signal.SIGTERM, 143), (signal.SIGINT, 130)]) From 5f4bd848e3876896c3b66d62b5ce618c02e08355 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:28:42 +0000 Subject: [PATCH 25/29] Switch --yes off until #170; keep the gate and exit contract honest The adversarial review of 3c0b188, reproduced with its probes against the suite's own Estate fixture: 1. --yes and --expire --yes are refused, exit 2, changing nothing, unless LANE_WORKTREES_ENABLE_YES=1 is set. #170 lists data-loss paths that are still open. The suite's Estate sets the variable; a real estate should not until #170 lands. The dry run, --porcelain and --report are untouched. 2. B1: a tree that #97's inventory does not name is still the lane's if it stands under .lane-worktrees//, or if it sits in /.claude/worktrees on a branch whose own commits carry this lane's Lane: trailer. Another lane's claim still wins. Such trees keep the quiet-hours liveness rule. A lane whose state is NONE but which has trees of its own is refused, exit 2. On Eagle, `sweep openRepoTools-3 --dry-run --porcelain` gave exit 0, `summary 0 15 0 15`. It now gives exit 2, `summary 9 13 4 4`: 9 trees are the lane's, and the snapshot is NONE. 3. B2: subprocess output and git metadata files are decoded with surrogateescape, and stdout and stderr write with it too, so a Latin-1 worktree name is reported rather than raising. An unreadable pytest-of-$USER is a kept sandbox row. A catch-all keeps errors that no read caught inside the contract: exit 2 on a dry run, and under --yes DISPOSITION.md is written first and the exit is 1. 4. B3: every porcelain field escapes backslash, TAB, newline and CR. Worktree registrations are read with `git worktree list -z` where git has it, so a newline in a path no longer becomes a phantom registration. Five new cases (switch off, ownership plus NONE, escaping, non-UTF-8 path, unreadable sandbox root), each failing against 3c0b188. Two cases were moved off the lane root to keep testing FOREIGN trees. 40 cases. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lane-worktrees | 218 +++++++++++++++++++++++++++++------ tests/test_lane_worktrees.py | 107 ++++++++++++++++- 2 files changed, 285 insertions(+), 40 deletions(-) diff --git a/lane-worktrees b/lane-worktrees index 0a189e7..ec94837 100755 --- a/lane-worktrees +++ b/lane-worktrees @@ -48,11 +48,22 @@ origin at the rescued SHA; an archive whose rescue branch is gone or moved, or that is not whole (its MANIFEST.sha256 missing or not matching, its rescues.tsv unreadable), is never expired. +`--yes` (and `--expire --yes`) is SWITCHED OFF until opensoft/openRepoTools +#170 lands: it is refused, exit 2, unless the environment carries +LANE_WORKTREES_ENABLE_YES=1 - which the test suite sets, and which is not for a +real estate until #170. The dry run, `--porcelain` and `--report` are whole. + +PORCELAIN: one row per line, fields separated by one TAB. In every field a +backslash is written `\\`, a TAB `\t`, a newline `\n` and a carriage +return `\r`, so a path holding any of them is still one field of one row. +Undecodable bytes in a path pass through as the bytes they are. + Exit codes: 0 dry run: nothing to retire · --yes: every act completed 3 dry run: something to retire (the `lane-end` gate's answer, #163) 2 refused: the lane is bound elsewhere, held by another live session, - managed-owned, or a read failed - nothing was changed + managed-owned, has trees but no #97 snapshot, or a read failed - or + `--yes` is switched off, or an error no read caught - nothing was changed 1 --yes: an act failed part-way; what was done is recorded and the rest is left exactly as it was 64 usage @@ -145,6 +156,18 @@ def say(text: str = "") -> None: sys.stdout.write(text + "\n") +def pesc(value) -> str: + """One porcelain field: a backslash, TAB, newline or carriage return in + it is written `\\`, `\\t`, `\\n`, `\\r`, so a row is always one line of + exactly its columns - a path with a TAB in it is still one field (B3).""" + return (str(value).replace("\\", "\\\\").replace("\t", "\\t") + .replace("\n", "\\n").replace("\r", "\\r")) + + +def prow(*fields) -> None: + say("\t".join(pesc(f) for f in fields)) + + def note(text: str) -> None: sys.stderr.write(f"{PROG}: {text}\n") @@ -156,9 +179,11 @@ def run(argv, cwd=None, env=None, timeout=GIT_TIMEOUT, input_text=None): stderr, and a timeout is 124 - the two shapes every caller here already reads as "the read failed", which is never an answer (Amendment 7(d)).""" try: + # SURROGATEESCAPE, NOT STRICT UTF-8: a path is bytes, and a worktree + # named in Latin-1 is a tree to report, never a traceback (B2). return subprocess.run( [str(a) for a in argv], cwd=cwd, env=env, capture_output=True, - text=True, timeout=timeout, input=input_text, + encoding="utf-8", errors="surrogateescape", timeout=timeout, input=input_text, stdin=None if input_text is not None else subprocess.DEVNULL) except subprocess.TimeoutExpired: return subprocess.CompletedProcess(argv, 124, "", f"timed out after {timeout}s") @@ -311,7 +336,7 @@ def load_conf() -> dict: conf = dict(DEFAULT_CONF) path = conf_path() try: - with open(path, encoding="utf-8") as fh: + with open(path, encoding="utf-8", errors="surrogateescape") as fh: lines = fh.read().splitlines() except OSError: return conf @@ -840,7 +865,7 @@ def git_common_dir(repo: str) -> str: if os.path.isdir(dotgit): return dotgit try: - with open(dotgit, encoding="utf-8") as fh: + with open(dotgit, encoding="utf-8", errors="surrogateescape") as fh: first = fh.readline().strip() except OSError: return "" @@ -850,7 +875,7 @@ def git_common_dir(repo: str) -> str: if not os.path.isabs(gitdir): gitdir = os.path.normpath(os.path.join(repo, gitdir)) try: - with open(os.path.join(gitdir, "commondir"), encoding="utf-8") as fh: + with open(os.path.join(gitdir, "commondir"), encoding="utf-8", errors="surrogateescape") as fh: common = fh.readline().strip() if not os.path.isabs(common): common = os.path.normpath(os.path.join(gitdir, common)) @@ -895,7 +920,7 @@ def dependents_map(repos: list) -> dict: continue try: with open(os.path.join(common, "objects", "info", "alternates"), - encoding="utf-8") as fh: + encoding="utf-8", errors="surrogateescape") as fh: for raw in fh.read().splitlines(): alt = raw.strip() if not alt or alt.startswith("#"): @@ -999,11 +1024,18 @@ class Tree: def worktree_registrations(checkout: str) -> list: - proc = git(checkout, "worktree", "list", "--porcelain") - if proc.returncode != 0: - return [] + # `-z` WHERE GIT HAS IT (2.36+): a path with a newline in it is one field, + # not a registration of half its name and a line of noise (B3). + proc = git(checkout, "worktree", "list", "--porcelain", "-z") + if proc.returncode == 0: + fields = proc.stdout.split("\0") + else: + proc = git(checkout, "worktree", "list", "--porcelain") + if proc.returncode != 0: + return [] + fields = proc.stdout.splitlines() out, cur = [], {} - for line in proc.stdout.splitlines() + [""]: + for line in fields + [""]: if not line: if cur: out.append(cur) @@ -1298,6 +1330,7 @@ class Sweep: self.fetched: dict = {} self.rescues: list = [] self.dispositions: list = [] + self.acting = False # ------------------------------------------------------------- reading def gather(self) -> None: @@ -1341,6 +1374,7 @@ class Sweep: # WHO ELSE CLAIMS THESE TREES is read before anything is written: a # claim that cannot be read refuses, and a refusal never fetches. self._other_lanes() + self._lane_claims() if self.opts.fetch and not self.refusal: self._fetch_all() self.deps = dependents_map(estate_repos(self.estate_root)) if self.estate_root else {} @@ -1355,6 +1389,38 @@ class Sweep: tree.merge = self.evidence.judge(tree.path, slug_of(tree.origin_url), tree.branch, tree.head) + def _lane_claims(self) -> None: + """A tree #97's inventory does not name is still THIS LANE'S when it + stands under the lane's own worktree root (`.lane-worktrees//`, + which only this lane's starts make), or sits in the checkout's + `.claude/worktrees` on a branch whose OWN commits - those its origin's + default branch lacks - carry this lane's `Lane:` trailer. Another + lane's claim still wins. Without this a lane whose trees were never + recorded read as "nothing to retire", and #163's gate passed it (B1). + + AND A LANE WITH NO SNAPSHOT AT ALL (#97's state NONE) BUT TREES OF ITS + OWN IS REFUSED: which of them are its is recorded nowhere, so the gate + does not pass it until a handoff records them or a person retires + them.""" + roots = self._roots() + lane_root = roots[-1] if roots else "" + claude_root = roots[0] if self.facts.dir and roots else "" + for t in self.trees: + if t.owner != "foreign" or t.foreign_kind == "other-lane": + continue + if lane_root and under(t.path, lane_root) and t.real != real(lane_root): + t.owner = "lane-root" + elif claude_root and t.exists and under(t.path, claude_root) \ + and self.lane.lower() in own_trailers(t.path): + t.owner = "trailer" + claimed = [t for t in self.trees if t.owner in ("lane-root", "trailer")] + if self.facts.state == "NONE" and claimed: + self.refusal = self.refusal or ( + f"lane {self.lane} has no #97 snapshot (state NONE), yet {len(claimed)} tree(s) " + f"are its by where they stand or what they carry (e.g. {claimed[0].path}); which " + "are its is recorded nowhere, so the gate does not pass - record them with a " + "handoff (lane-handoff takes the inventory), or retire them by hand") + def _roots(self) -> list: roots = [] if self.facts.dir: @@ -1464,7 +1530,7 @@ class Sweep: continue path = "" try: - with open(os.path.join(tdir, fname), encoding="utf-8") as fh: + with open(os.path.join(tdir, fname), encoding="utf-8", errors="surrogateescape") as fh: for line in fh: if line.startswith("path: "): path = line[6:].strip() @@ -1552,11 +1618,13 @@ class Sweep: if answer == "unknown": tree.live = (None, f"whether session {writer[:8]} is live could not be read") return - if tree.owner == "foreign" and tree.is_git: + if tree.owner != "inventory" and tree.is_git: quiet = self.conf["foreign_quiet_hours"] * 3600 newest = _gitdir_activity(tree.path) if newest and time.time() - newest < quiet: - tree.live = (True, f"FOREIGN and active {int((time.time() - newest) // 60)} min ago " + tree.live = (True, ("FOREIGN" if tree.owner == "foreign" else + "not in the inventory") + + f" and active {int((time.time() - newest) // 60)} min ago " f"(foreign trees quiet for {self.conf['foreign_quiet_hours']} h only)") def _lane_gate(self) -> None: @@ -1603,6 +1671,10 @@ class Sweep: def decide(self) -> None: for tree in self.trees: self._decide(tree) + if tree.owner in ("lane-root", "trailer") and tree.disposition != "foreign": + tree.reason += ("; not in #97's inventory - the lane's by " + + ("its root" if tree.owner == "lane-root" else + "its branch's Lane: trailer")) if self.opts.branches: self._branches() if self.opts.scratch: @@ -1615,7 +1687,7 @@ class Sweep: self.items.extend(link_items(self.estate_root)) def _decide(self, t: Tree) -> None: - mine = t.owner == "inventory" or (self.opts.foreign and t.foreign_kind != "other-lane") + mine = t.owner != "foreign" or (self.opts.foreign and t.foreign_kind != "other-lane") def put(disp: str, why: str, retire: bool = True) -> None: t.disposition, t.reason, t.retire = disp, why, retire @@ -1886,24 +1958,24 @@ class Sweep: def porcelain(self) -> None: f = self.facts - say("\t".join(["lane", self.lane, f.state or "-", f.holder or "-", - f.holder_uuid or "-", f.binding or "-", f.verdict or "-"])) + prow("lane", self.lane, f.state or "-", f.holder or "-", + f.holder_uuid or "-", f.binding or "-", f.verdict or "-") if self.refusal: - say("refused\t" + self.refusal.replace("\t", " ")) + prow("refused", self.refusal) for t in self.trees: - say("\t".join(["tree", t.disposition, "retire" if t.retire else "-", t.path, - t.branch or ("(detached)" if t.detached else "-"), t.head or "-", - t.reason.replace("\t", " ") + (f" -> {t.result}" if t.result else "")])) + prow("tree", t.disposition, "retire" if t.retire else "-", t.path, + t.branch or ("(detached)" if t.detached else "-"), t.head or "-", + t.reason + (f" -> {t.result}" if t.result else "")) for i in self.items: - say("\t".join([i.kind, i.disposition, "retire" if i.retire else "-", i.path, - i.detail or "-", str(i.size), - i.reason.replace("\t", " ") + (f" -> {i.result}" if i.result else "")])) - say(f"summary\t{self.retire_count()}\t{len(self.trees)}\t" - f"{sum(1 for t in self.trees if t.disposition == 'live')}\t" - f"{sum(1 for t in self.trees if t.disposition == 'foreign')}") + prow(i.kind, i.disposition, "retire" if i.retire else "-", i.path, + i.detail or "-", str(i.size), i.reason + (f" -> {i.result}" if i.result else "")) + prow("summary", self.retire_count(), len(self.trees), + sum(1 for t in self.trees if t.disposition == "live"), + sum(1 for t in self.trees if t.disposition == "foreign")) # -------------------------------------------------------------- acting def act(self) -> None: + self.acting = True os.umask(0o077) for t in self.trees: if t.disposition in ("live", "foreign", "keep", "gone", "load-bearing", ""): @@ -2356,9 +2428,9 @@ class Sweep: for path, text, line in self.dispositions: lines.append(f"- `{path}`: {line} - {text}") lines.append("") - with open(os.path.join(d, "DISPOSITION.md"), "w", encoding="utf-8") as fh: + with open(os.path.join(d, "DISPOSITION.md"), "w", encoding="utf-8", errors="surrogateescape") as fh: fh.write("\n".join(lines)) - with open(os.path.join(d, "rescues.tsv"), "w", encoding="utf-8") as fh: + with open(os.path.join(d, "rescues.tsv"), "w", encoding="utf-8", errors="surrogateescape") as fh: fh.write("# origin\tbranch\tsha\tcheckout\ttree\n") for row in self.rescues: fh.write("\t".join(row) + "\n") @@ -2402,6 +2474,20 @@ def _lean_text(lean: list) -> str: f"; and {len(lean) - 5} more" if len(lean) > 5 else "") +def own_trailers(path: str) -> set: + """The `Lane:` trailers of the commits HEAD has that origin's default + branch lacks, lower-cased - empty where there is no such base to tell the + branch's own commits by.""" + base = git_out(path, "symbolic-ref", "-q", "refs/remotes/origin/HEAD") or "" + if not base and git(path, "rev-parse", "-q", "--verify", + "refs/remotes/origin/main").returncode == 0: + base = "refs/remotes/origin/main" + if not base: + return set() + text = git_out(path, "log", "--format=%B", f"{base}..HEAD") or "" + return {v.lower() for v in re.findall(r"^Lane:\s*(\S+)\s*$", text, re.MULTILINE)} + + def _gitdir_activity(path: str) -> float: gitdir = git_out(path, "rev-parse", "--absolute-git-dir") newest = 0.0 @@ -2471,7 +2557,7 @@ def write_manifest(d: str) -> None: if rel == "MANIFEST.sha256": continue rows.append(f"{sha256_of(full)} {rel}") - with open(os.path.join(d, "MANIFEST.sha256"), "w", encoding="utf-8") as fh: + with open(os.path.join(d, "MANIFEST.sha256"), "w", encoding="utf-8", errors="surrogateescape") as fh: fh.write("\n".join(sorted(rows, key=lambda r: r.split(" ", 1)[1])) + "\n") @@ -2612,6 +2698,7 @@ def sandbox_items(procs: ProcScan, conf: dict) -> list: environment is never touched, and neither is one younger than the minimum age (`sandbox_min_age_minutes`).""" out = [] + unreadable: list = [] uid = os.getuid() procs.load_environ() min_age = conf["sandbox_min_age_minutes"] * 60 @@ -2627,7 +2714,16 @@ def sandbox_items(procs: ProcScan, conf: dict) -> list: if name.startswith("tmp.") and os.path.isdir(full) and not os.path.islink(full): candidates.append((full, "mktemp", 0)) if name == f"pytest-of-{user}" and os.path.isdir(full): - for sub in os.listdir(full): + try: + subs = os.listdir(full) + except OSError as exc: + # UNREADABLE IS A ROW, never a traceback and never "empty". + it = Item("sandbox", full, "keep", "its pytest sandboxes could not be listed (" + + (exc.strerror or str(exc)) + "), so none of them is touched", + False) + unreadable.append(it) + continue + for sub in subs: sp = os.path.join(full, sub) if sub.startswith("pytest-") and os.path.isdir(sp) and not os.path.islink(sp): candidates.append((sp, "pytest", _lock_pid(os.path.join(sp, ".lock")))) @@ -2661,12 +2757,12 @@ def sandbox_items(procs: ProcScan, conf: dict) -> list: it.size = dir_size(path) it.extra["kind"] = kind out.append(it) - return out + return out + unreadable def _lock_pid(path: str) -> int: try: - with open(path, encoding="utf-8") as fh: + with open(path, encoding="utf-8", errors="surrogateescape") as fh: text = fh.read().strip() return int(text) if text.isdigit() else 0 except (OSError, ValueError): @@ -2717,7 +2813,7 @@ def link_items(root: str) -> list: def _gitdir_pointer(path: str, base: str, now: float, out: list) -> None: try: - with open(path, encoding="utf-8") as fh: + with open(path, encoding="utf-8", errors="surrogateescape") as fh: first = fh.readline().strip() except OSError: return @@ -2795,14 +2891,14 @@ def expire(opts) -> int: rmtree(adir) rows[i] = (adir, age, "expired", why) os.makedirs(root, exist_ok=True) - with open(os.path.join(root, "EXPIRED.log"), "a", encoding="utf-8") as fh: + with open(os.path.join(root, "EXPIRED.log"), "a", encoding="utf-8", errors="surrogateescape") as fh: fh.write(f"{utc_now()}\t{adir}\t{why}\n") except OSError as exc: rows[i] = (adir, age, "FAILED", str(exc)) failed += 1 if opts.porcelain: for adir, age, disp, why in rows: - say("\t".join(["archive", disp, adir, str(age), why])) + prow("archive", disp, adir, age, why) else: say(f"{PROG} sweep --expire - " + ("YES" if opts.yes else "DRY RUN") + f" - retention {keep_days} d ({conf_path()})") @@ -2830,7 +2926,7 @@ def archive_ledger(adir: str): manifest = os.path.join(adir, "MANIFEST.sha256") listed: dict = {} try: - with open(manifest, encoding="utf-8") as fh: + with open(manifest, encoding="utf-8", errors="surrogateescape") as fh: for line in fh: if not line.strip(): continue @@ -2861,7 +2957,7 @@ def archive_ledger(adir: str): return None, "it has no rescues.tsv" out = [] try: - with open(os.path.join(adir, "rescues.tsv"), encoding="utf-8") as fh: + with open(os.path.join(adir, "rescues.tsv"), encoding="utf-8", errors="surrogateescape") as fh: for line in fh: if line.startswith("#") or not line.strip(): continue @@ -2906,7 +3002,43 @@ def parse(argv: list): return parser.parse_args(argv) +YES_SWITCH = "LANE_WORKTREES_ENABLE_YES" +_ACTIVE: dict = {} + + def main(argv: list) -> int: + """The exit contract, whatever happens inside: an error no read caught is + exit 2 on a dry run (nothing was changed), and under `--yes` the archive's + DISPOSITION.md is written first and the exit is 1 (B2).""" + for stream in (sys.stdout, sys.stderr): + if hasattr(stream, "reconfigure"): + stream.reconfigure(errors="surrogateescape") + try: + return _main(argv) + except Exception as exc: # pylint: disable=broad-except + import traceback + frames = [f for f in traceback.extract_tb(exc.__traceback__) + if os.path.basename(f.filename) == os.path.basename(__file__)] + where = f" at {PROG} line {frames[-1].lineno}" if frames else "" + detail = f"{type(exc).__name__}: {exc}{where}" + sweep = _ACTIVE.get("sweep") + acted = bool(sweep is not None and sweep.acting) + if sweep is not None and sweep.opts.porcelain: + prow("refused", "internal error, outside the table: " + detail) + if acted: + try: + if sweep.archive: + sweep._write_disposition() + except Exception: # pylint: disable=broad-except + pass + note(f"INTERNAL ERROR part-way through --yes: {detail}; what was done is in " + f"{tilde(sweep.archive) if sweep.archive else 'the table above'}") + return 1 + note(f"INTERNAL ERROR, nothing was changed: {detail}") + return 2 + + +def _main(argv: list) -> int: try: ns = parse(argv) except Usage as exc: @@ -2921,6 +3053,17 @@ def main(argv: list) -> int: if ns.verb != "sweep": note("usage: lane-worktrees sweep [--dry-run | --yes] ... (see --help)") return 64 + # `--yes` IS SWITCHED OFF until the data-loss paths the reviews found are + # closed (opensoft/openRepoTools#170). The dry run, the porcelain #163's + # gate reads and the report stay whole; acting takes the switch, which the + # suite sets and a real estate does not. + if ns.yes and os.environ.get(YES_SWITCH, "") != "1": + why = (f"--yes is disabled until opensoft/openRepoTools#170 lands; the dry run is " + f"unaffected ({YES_SWITCH}=1 enables it, and is not for a real estate until #170)") + if ns.porcelain: + prow("refused", why) + note("REFUSED: " + why) + return 2 if ns.expire: if ns.lane: note("usage: sweep --expire takes no lane: it reads every lane's archives") @@ -2937,6 +3080,7 @@ def main(argv: list) -> int: return 64 opts = Options(ns) sweep = Sweep(ns.lane, opts) + _ACTIVE["sweep"] = sweep sweep.gather() if sweep.refusal and not sweep.trees: sweep.decide() diff --git a/tests/test_lane_worktrees.py b/tests/test_lane_worktrees.py index e9064a9..57a2e04 100644 --- a/tests/test_lane_worktrees.py +++ b/tests/test_lane_worktrees.py @@ -33,6 +33,7 @@ from __future__ import annotations import datetime as _dt +import getpass import hashlib import json import os @@ -143,6 +144,9 @@ def __init__(self, root: Path): FAKE_HELPER_SPEC=str(self.spec_file), FAKE_HELPER_LOG=str(self.helper_log), FAKE_GH_PRS=str(self.prs_file), CLAUDE_CODE_SESSION_ID=ME, LANE_WORKTREES_SANDBOX_ROOTS=str(self.sandboxes), LANES_WORKSTATION="Eagle", + # --yes is switched off for a real estate until #170; the suite's + # estates are not one. + LANE_WORKTREES_ENABLE_YES="1", GIT_CONFIG_NOSYSTEM="1", PATH=str(self.fakebin) + os.pathsep + _clean_path()) self.env = env @@ -879,7 +883,7 @@ def test_scratch_is_archived_then_removed_and_caches_are_removed(estate): def test_caches_are_never_taken_from_a_foreign_tree(estate): """Copilot on #168: a FOREIGN tree under the lane's root is somebody's, live or not, and so are its caches.""" - foreign = estate.worktree("theirs", "feat/theirs", record=False) + foreign = estate.worktree("theirs", "feat/theirs", where="claude", record=False) (foreign / "__pycache__").mkdir() (foreign / "__pycache__" / "t.pyc").write_bytes(b"\0") proc = estate.sweep(LANE, "--include-caches", "--yes", "--porcelain") @@ -1010,8 +1014,8 @@ def test_a_clone_something_leans_on_is_load_bearing_and_never_removed(estate): assert rows[str(store)][0] == "load-bearing", rows assert str(dependent) in rows[str(store)][2] and "repack" in rows[str(store)][2] assert store.is_dir() - foreign = estate.sweep(LANE, "--porcelain") - assert "LOAD-BEARING" in rows_of(foreign.stdout)[str(store)][2] + again = estate.sweep(LANE, "--porcelain", env={"LANE_WORKTREES_CONF": str(conf)}) + assert "LOAD-BEARING" in rows_of(again.stdout)[str(store)][2] # =================================================================== --expire @@ -1095,6 +1099,103 @@ def test_the_porcelain_exit_codes_lane_end_reads(estate): assert refused.returncode == 2 and "refused\t" in refused.stdout +def test_yes_is_switched_off_without_the_enabling_variable(estate): + """Until #170 lands `--yes` and `--expire --yes` are refused, exit 2, + changing nothing; the dry run and its porcelain are untouched.""" + estate.worktree("w", "feat/w") + _archive(estate, 120, None) + before = snapshot(estate.root, skip=("helper.log", "helper-spec.json", "prs.json")) + off = {"LANE_WORKTREES_ENABLE_YES": None} + for args in ((LANE, "--yes"), (LANE, "--yes", "--porcelain"), ("--expire", "--yes")): + proc = estate.sweep(*args, env=off) + assert proc.returncode == 2, (args, proc.stdout, proc.stderr) + assert "--yes is disabled until opensoft/openRepoTools#170 lands" in proc.stderr + assert snapshot(estate.root, skip=("helper.log", "helper-spec.json", "prs.json")) == before + assert estate.sweep(LANE, "--porcelain", env=off).returncode == 3 + assert estate.sweep("--expire", "--porcelain", env=off).returncode == 3 + + +def test_trees_under_the_lanes_root_or_carrying_its_trailer_are_its_own(estate): + """B1: a tree #97's inventory does not name is the lane's when it stands + under `.lane-worktrees//`, or sits in `.claude/worktrees` on a branch + whose own commits carry this lane's trailer - so the gate does not pass a + lane whose trees were never recorded.""" + conf = estate.root / "sweep.conf" + conf.write_text("foreign_quiet_hours=0\n") + estate.env["LANE_WORKTREES_CONF"] = str(conf) + rooted = estate.worktree("rooted", "feat/rooted", record=False) + trailed = estate.worktree("trailed", "feat/trailed", where="claude", record=False) + estate.commit(trailed, "mine", {"m.txt": "m\n"}) + other = estate.worktree("other", "feat/other", where="claude", record=False) + estate.commit(other, "theirs", {"o.txt": "o\n"}, lane="repoB-2") + proc = estate.sweep(LANE, "--dry-run", "--porcelain") + assert proc.returncode == 3, proc.stdout + proc.stderr + rows = rows_of(proc.stdout) + assert rows[str(rooted)][:2] == ("remove", "retire") and "the lane's by its root" in rows[str(rooted)][2] + assert rows[str(trailed)][1] == "retire" and "Lane: trailer" in rows[str(trailed)][2] + assert rows[str(other)][0] == "foreign" + # NO SNAPSHOT AT ALL, and trees of its own: the gate does not pass. + estate.extra_spec = {"lane-reconcile": {"rc": 8}, + "binding": {"rc": 0, "out": "x\tx\tx\tx\tx\tx\there\tnone\n"}, + "live-holder": {"rc": 8}} + none = estate.sweep(LANE, "--dry-run", "--porcelain") + assert none.returncode == 2, none.stdout + none.stderr + assert "has no #97 snapshot (state NONE)" in none.stdout + assert str(rooted) in rows_of(none.stdout), "the table is still printed" + + +def test_porcelain_escapes_a_tab_and_a_newline_in_a_path(estate): + """B3: a TAB or a newline in a tree's name is still one field of one row.""" + estate.lane_root.mkdir(parents=True, exist_ok=True) + tab = estate.lane_root / "tab\tname" + nl = estate.lane_root / "nl\nname" + for i, path in enumerate((tab, nl)): + estate.git("worktree", "add", "-q", "-b", f"feat/odd{i}", path, "origin/main") + proc = estate.sweep(LANE, "--dry-run", "--porcelain") + assert proc.returncode == 3, proc.stdout + proc.stderr + lines = proc.stdout.splitlines() + assert all(ln.split("\t")[0] in ("lane", "tree", "summary") for ln in lines), lines + trees = [ln for ln in lines if ln.startswith("tree\t")] + assert all(len(ln.split("\t")) == 7 for ln in trees), lines + paths = [ln.split("\t")[3] for ln in trees] + assert str(tab).replace("\t", "\\t") in paths and str(nl).replace("\n", "\\n") in paths + if subprocess.run(["git", "worktree", "list", "--porcelain", "-z"], cwd=str(estate.checkout), + capture_output=True).returncode == 0: + assert len(trees) == 2, "a newline in a registered path is one registration, not two" + + +def test_a_path_that_is_not_utf8_is_a_row_not_a_traceback(estate): + """B2: a worktree named in Latin-1 is reported - exit inside 0/3/2.""" + estate.lane_root.mkdir(parents=True, exist_ok=True) + bad = os.fsencode(str(estate.lane_root)) + b"/caf\xe9" + subprocess.run(["git", "-C", str(estate.checkout), "worktree", "add", "-q", "-b", "feat/latin1", + bad, "origin/main"], env=estate.env, check=True, capture_output=True) + estate.write_spec() + for args in (("--dry-run", "--porcelain"), ("--dry-run",)): + proc = subprocess.run([sys.executable, str(LW), "sweep", LANE, *args], capture_output=True, + env=estate.env, timeout=180, cwd=str(estate.root), + stdin=subprocess.DEVNULL) + assert proc.returncode in (0, 3), (args, proc.stdout[-500:], proc.stderr[-800:]) + assert b"Traceback" not in proc.stderr + assert bad in proc.stdout, "the path is reported as the bytes it is" + + +def test_an_unreadable_pytest_sandbox_root_is_a_row_not_a_traceback(estate): + """B2: an unreadable `pytest-of-$USER` is listed as kept, exit 0.""" + root = estate.sandboxes / f"pytest-of-{getpass.getuser()}" + root.mkdir() + root.chmod(0) + try: + proc = estate.sweep(LANE, "--include-sandboxes", "--dry-run", "--porcelain") + finally: + root.chmod(0o755) + assert proc.returncode == 0, proc.stdout + proc.stderr + assert "Traceback" not in proc.stderr + if os.geteuid() != 0: + rows = items_of(proc.stdout, "sandbox") + assert rows[str(root)][0] == "keep" and "could not be listed" in rows[str(root)][3] + + def test_usage_errors_are_64(estate): for argv in (["--bogus"], [], ["--word", "x"]): proc = estate.sweep(*argv) if argv != [] else estate.sweep() From 296e2f0e54b5fd78b5f16d19c797632502d97034 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:28:42 +0000 Subject: [PATCH 26/29] Document the --yes switch, lane-root ownership and porcelain escaping The manual now leads the sweep section with the --yes switch and #170. It also says which unrecorded trees are a lane's and that a NONE snapshot with trees is refused. The exit contract now covers the escaped fields and the catch-all. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- docs/README-lanes.md | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/docs/README-lanes.md b/docs/README-lanes.md index 250d43c..2484d7f 100644 --- a/docs/README-lanes.md +++ b/docs/README-lanes.md @@ -3313,6 +3313,13 @@ place, under one rule: **nothing is deleted that is not first on origin or in a bundle under the sweeps directory, and a tree a live writer owns is never touched.** +> **`--yes` is switched off until opensoft/openRepoTools#170 lands.** Reviews +> of this act found data-loss paths under `--yes` and `--expire --yes` that are +> listed there and not yet closed. Until they are, both are refused, exit **2**, +> changing nothing, unless the environment carries `LANE_WORKTREES_ENABLE_YES=1` +> — which the test suite sets, and which is **not for a real estate until #170**. +> The dry run, `--porcelain` (#163's gate) and `--report` are unaffected. + ```sh lane-worktrees sweep # the DRY RUN: the table, nothing changed lane-worktrees sweep --yes [--live |--live none] @@ -3324,7 +3331,13 @@ lane-worktrees sweep --expire [--yes] # archives past retention and the disk — **never the derived index** (Amendment 14 clause (b)). The sweep also reads every registration of the lane's checkout and every checkout under its two roots (`/.claude/worktrees/*`, `.lane-worktrees//*`). -A tree the inventory does not name is **FOREIGN**: reported and left, unless +A tree the inventory does not name is still **the lane's** when it stands under +the lane's own root, `.lane-worktrees//` (only this lane's starts make it), +or sits in `/.claude/worktrees` on a branch whose OWN commits — those +origin's default branch lacks — carry this lane's `Lane:` trailer; its rows say +so. A lane with no #97 snapshot at all (state `NONE`) but trees of its own is +**refused, exit 2**: which trees are its is recorded nowhere, so #163's gate does +not pass it. Any other tree the inventory does not name is **FOREIGN**: reported and left, unless `--include-foreign` and a `--word ""` (recorded in every register line it causes). A tree ANOTHER lane's inventory names is that lane's and is never taken from here, word or no word — and one BOTH inventories name is kept, since @@ -3465,13 +3478,19 @@ tree summary ``` -and exits **0** (nothing to retire), **3** (something to retire — a tree of the +Every field is escaped — a backslash is written `\\`, a TAB `\t`, a newline `\n` +and a carriage return `\r` — so a path holding any of them is still one field +of one row; bytes that are not UTF-8 pass through as the bytes they are. The +command exits **0** (nothing to retire), **3** (something to retire — a tree of the lane's on disk, live or not, a stale registration, an unpublished or merged branch of the lane's, included scratch), or **2** (refused: bound elsewhere, -held by another session, managed, or a read failed). FOREIGN trees, links, +held by another session, managed, a lane with trees and no #97 snapshot, a read +failed, `--yes` switched off — or an error no read caught, which is never a +traceback's exit 1 on a dry run). FOREIGN trees, links, caches and sandboxes are never the lane's to retire. `--yes` exits 0 when every act completed and 1 when one failed part-way (the rest go on; each failure is in -the table and in `DISPOSITION.md`); usage is 64. +the table and in `DISPOSITION.md`, which is written even when an error no read +caught stops the run); usage is 64. ### Two protocol lines this act assumes (proposed for the amendment that ratifies it) From 0cf10517e616e0044d7ee06a15ce6e0136ffd254 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:32:05 +0000 Subject: [PATCH 27/29] Run the suite in the foreground when stdin is a terminal Adversarial review B7: `set -m` puts the suite in a background process group with the terminal as its stdin. Anything that reads the terminal (--pdb, breakpoint(), input()) is then stopped by SIGTTIN, and the wrapper waits on it for ever while holding the workstation flock. With a terminal on stdin the suite now runs in the foreground, in the terminal's own process group, and a Ctrl-C reaches all of it from the terminal. Without a terminal the process-group stop is unchanged. The new case runs the wrapper on a pseudo-terminal whose stub suite prints a prompt and reads a line. It passes here. Against the previous run.sh it hangs and fails, with the transcript showing the prompt and the typed line that was never read. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- docs/README-lanes.md | 5 +++- tests/run.sh | 12 ++++++++ tests/test_run_wrapper.py | 58 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) diff --git a/docs/README-lanes.md b/docs/README-lanes.md index fbf0954..7b7d3c0 100644 --- a/docs/README-lanes.md +++ b/docs/README-lanes.md @@ -3559,7 +3559,10 @@ The sweep retires what is already there; these keep more from arriving. and roots every temporary directory of a run under `${XDG_STATE_HOME:-~/.local/state}/openRepoTools/tmp/-/`, which its EXIT trap removes however the run ends — after stopping the suite's whole - process group, so nothing it started outlives the run root or the lock. A SIGKILL is the one end no trap sees; + process group, so nothing it started outlives the run root or the lock. With + a terminal on stdin the suite runs in the FOREGROUND instead, so `--pdb` or a + `breakpoint()` can read it (a background group would be stopped by SIGTTIN, + and the wrapper would wait on it holding the lock). A SIGKILL is the one end no trap sees; the pid in the name is what lets `--include-sandboxes` tell its owner is gone. * **One virtual environment per repository, outside the estate:** `${XDG_CACHE_HOME:-~/.cache}/openRepoTools/venvs//`. `openRepoTools diff --git a/tests/run.sh b/tests/run.sh index f273d66..29023cc 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -179,8 +179,20 @@ make_run_root() { # its own, whose id is its pid, so `stop_group` reaches every process it # started (one that made a session or group of its own is beyond any wrapper). # Monitor mode is off again before the `wait`, so no job notice is printed. +# +# A TERMINAL ON STDIN RUNS THE SUITE IN THE FOREGROUND instead, in this +# wrapper's own process group - the terminal's foreground group. A background +# group reading the terminal (`--pdb`, `breakpoint()`, `input()`) is stopped by +# SIGTTIN, and this wrapper would wait on it for ever holding the +# workstation's lock. In the foreground a Ctrl-C reaches every process of the +# suite from the terminal itself; the cost is that a TERM sent to this wrapper +# from elsewhere is acted on once the suite returns. run_suite() { printf '%s: python3 -m pytest tests -q -p no:cacheprovider %s\n' "$prog" "${*:-}" >&2 + if [ -t 0 ]; then + python3 -m pytest tests -q -p no:cacheprovider --basetemp="$RUN_ROOT/basetemp" ${1+"$@"} + return $? + fi set -m python3 -m pytest tests -q -p no:cacheprovider --basetemp="$RUN_ROOT/basetemp" ${1+"$@"} <&0 & SUITE_PID=$! diff --git a/tests/test_run_wrapper.py b/tests/test_run_wrapper.py index 64362ae..a4ed952 100644 --- a/tests/test_run_wrapper.py +++ b/tests/test_run_wrapper.py @@ -27,6 +27,7 @@ import shutil import signal import subprocess +import sys import time from pathlib import Path @@ -50,6 +51,13 @@ printf 'pid %s\\n' "$$" } > "$FAKE_SUITE_LOG" : > "$TMPDIR/the-suite-wrote-here" +if [ -n "${FAKE_SUITE_READ:-}" ]; then + # A DEBUGGER'S PROMPT: it reads the terminal, as `--pdb` would. + printf 'prompt>\n' + read -r line + printf 'got: %s\n' "$line" + exit 0 +fi if [ -n "${FAKE_SUITE_SLEEP:-}" ]; then # A CHILD OF ITS OWN, as pytest waits on a shell suite or a `git`: the # wrapper must stop it too, not only the process it started. @@ -202,6 +210,56 @@ def test_a_killed_run_removes_its_run_root_and_stops_the_suite(tmp_path, sig, rc assert not (box.lockdir / "openrepotools-pytest.lock.d").exists(), "the lock was kept" +@pytest.mark.skipif(not hasattr(os, "openpty") or sys.platform.startswith("win"), + reason="needs a pseudo-terminal") +def test_a_suite_that_reads_the_terminal_is_not_stopped(tmp_path): + """Adversarial review B7: with a terminal on stdin the suite runs in the + foreground, in the terminal's process group - a debugger's prompt reads + it. A background group would be stopped by SIGTTIN and the wrapper would + wait on it for ever, holding the workstation's lock.""" + import pty + import select + box = Box(tmp_path) + env = dict(box.env, FAKE_SUITE_READ="1") + pid, master = pty.fork() + if pid == 0: # the child: a session on the pty + try: + os.execve(shutil.which("bash") or "/bin/bash", ["bash", str(WRAPPER)], env) + finally: + os._exit(127) + seen, sent, status, finished = b"", False, None, False + deadline = time.time() + 30 + try: + while time.time() < deadline: + ready, _w, _x = select.select([master], [], [], 0.2) + if ready: + try: + chunk = os.read(master, 4096) + except OSError: + chunk = b"" + seen += chunk + if b"prompt>" in seen and not sent: + os.write(master, b"hello\n") + sent = True + done, status = os.waitpid(pid, os.WNOHANG) + if done: + finished = True + break + else: + raise AssertionError(f"the wrapper hung on a suite reading the terminal: {seen!r}") + finally: + if not finished: + try: + os.killpg(pid, signal.SIGKILL) + os.waitpid(pid, 0) + except OSError: + pass + os.close(master) + assert b"got: hello" in seen, seen + assert os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0, (status, seen) + assert box.runs() == [], "the run root outlived the run" + + def _running(pid: int) -> bool: """Alive and not a zombie: an orphan's zombie answers `kill -0` until whoever adopted it reaps it.""" From 0b7a5c604be7719c45d6f4217d3ce2123da25f05 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:46:02 +0000 Subject: [PATCH 28/29] Never adopt a standalone clone under the lane's root 5f4bd84 made every unrecorded tree under .lane-worktrees// the lane's. That included standalone clones, which #162's first protocol line and the manual keep FOREIGN ("a lane creates worktrees, never clones"). Copilot flagged it on 296e2f0. Clones are now excluded from both adoption paths. The ownership case adds a clone under the lane root and requires it to stay foreign and not retire-counted. On Eagle none of the lane's seven root trees is a clone, so the dry run is unchanged there. Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- lane-worktrees | 5 ++++- tests/test_lane_worktrees.py | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lane-worktrees b/lane-worktrees index ec94837..ec1cfd7 100755 --- a/lane-worktrees +++ b/lane-worktrees @@ -1406,7 +1406,10 @@ class Sweep: lane_root = roots[-1] if roots else "" claude_root = roots[0] if self.facts.dir and roots else "" for t in self.trees: - if t.owner != "foreign" or t.foreign_kind == "other-lane": + # A CLONE IS NEVER ADOPTED: a lane creates worktrees, never clones, + # so a standalone clone under its root stays FOREIGN - a finding for + # a person's word, as #162's first protocol line has it. + if t.owner != "foreign" or t.foreign_kind in ("other-lane", "clone"): continue if lane_root and under(t.path, lane_root) and t.real != real(lane_root): t.owner = "lane-root" diff --git a/tests/test_lane_worktrees.py b/tests/test_lane_worktrees.py index 57a2e04..cedbbb0 100644 --- a/tests/test_lane_worktrees.py +++ b/tests/test_lane_worktrees.py @@ -1128,12 +1128,16 @@ def test_trees_under_the_lanes_root_or_carrying_its_trailer_are_its_own(estate): estate.commit(trailed, "mine", {"m.txt": "m\n"}) other = estate.worktree("other", "feat/other", where="claude", record=False) estate.commit(other, "theirs", {"o.txt": "o\n"}, lane="repoB-2") + # A CLONE under the lane's root is never adopted: a lane makes worktrees. + clone = estate.lane_root / "cloned" + estate.git("clone", "-q", GH_URL, clone, cwd=estate.root) proc = estate.sweep(LANE, "--dry-run", "--porcelain") assert proc.returncode == 3, proc.stdout + proc.stderr rows = rows_of(proc.stdout) assert rows[str(rooted)][:2] == ("remove", "retire") and "the lane's by its root" in rows[str(rooted)][2] assert rows[str(trailed)][1] == "retire" and "Lane: trailer" in rows[str(trailed)][2] assert rows[str(other)][0] == "foreign" + assert rows[str(clone)][:2] == ("foreign", "-"), rows[str(clone)] # NO SNAPSHOT AT ALL, and trees of its own: the gate does not pass. estate.extra_spec = {"lane-reconcile": {"rc": 8}, "binding": {"rc": 0, "out": "x\tx\tx\tx\tx\tx\there\tnone\n"}, From d2f80512902bf85b5993134a3cef6913d131565a Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:05:06 +0000 Subject: [PATCH 29/29] Skip the non-UTF-8 path case where the filesystem refuses the name tests-macos on #168's head df0d691 failed one case, 1160 passed: test_a_path_that_is_not_utf8_is_a_row_not_a_traceback. Its fixture's `git worktree add` of a Latin-1 path died with "fatal: could not create directory of '.git/worktrees/caf\xe9': Illegal byte sequence". APFS refuses a non-UTF-8 name, so no such worktree can exist there, and the case has nothing to report. It failed with a CalledProcessError before reaching its assertion. The fixture now makes the raw-bytes directory first. Where the filesystem refuses it (OSError), or git refuses the add, the case skips and says why. Linux accepts the name, so the case runs there as before: git adds a worktree into an existing empty directory. Refs #177 Lane: openRepoTools-3 Co-Authored-By: Claude Opus 5.5 --- tests/test_lane_worktrees.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test_lane_worktrees.py b/tests/test_lane_worktrees.py index cedbbb0..28ca0a9 100644 --- a/tests/test_lane_worktrees.py +++ b/tests/test_lane_worktrees.py @@ -1172,8 +1172,20 @@ def test_a_path_that_is_not_utf8_is_a_row_not_a_traceback(estate): """B2: a worktree named in Latin-1 is reported - exit inside 0/3/2.""" estate.lane_root.mkdir(parents=True, exist_ok=True) bad = os.fsencode(str(estate.lane_root)) + b"/caf\xe9" - subprocess.run(["git", "-C", str(estate.checkout), "worktree", "add", "-q", "-b", "feat/latin1", - bad, "origin/main"], env=estate.env, check=True, capture_output=True) + # A FILESYSTEM THAT REFUSES THE NAME HAS NO SUCH WORKTREE TO REPORT (#177). + # APFS, the macOS runner's, answers a non-UTF-8 name with EILSEQ ("Illegal + # byte sequence"), and git's own `.git/worktrees/caf\xe9` meets the same + # refusal. The directory is made first so the refusal is the filesystem's + # and not a fixture traceback; where it is made, the case stays live. + try: + os.mkdir(bad) + except OSError as exc: + pytest.skip(f"filesystem refuses non-UTF-8 names: {exc}") + add = subprocess.run(["git", "-C", str(estate.checkout), "worktree", "add", "-q", "-b", + "feat/latin1", bad, "origin/main"], env=estate.env, capture_output=True) + if add.returncode != 0: + pytest.skip("filesystem refuses non-UTF-8 names: git worktree add exited " + f"{add.returncode}: {add.stderr.decode('utf-8', 'replace').strip()}") estate.write_spec() for args in (("--dry-run", "--porcelain"), ("--dry-run",)): proc = subprocess.run([sys.executable, str(LW), "sweep", LANE, *args], capture_output=True,