fix(config): show live Herdr sidebar size for uncommitted worker changes

config/herdr-sidebar.ts

@@ -7,31 +7,33 @@
 //   - pane token `who`: the short name of the pane's own workspace (shortName
 //     below); none for a worker, whose title names it.
 //   - pane tokens `pr`, `issue`, `add`, `del`, `files`: the open pull request
-//     for the current branch, the issue it works on, and its size against the
-//     default branch, as "⎇ <pr>", "○ <issue>", "+<added>", "−<deleted>",
-//     "✎ <files>". A worker shows its size from its first commit, before any
-//     pull request opens.
+//     for the current branch, the issue it works on, and the size of the work
+//     in the checkout against the default branch, as "⎇ <pr>", "○ <issue>",
+//     "+<added>", "−<deleted>", "✎ <files>". A worker shows its size as soon
+//     as its diff is non-empty, before any commit or pull request.
 //     The first part present carries the indent.
-// Lookups run in the background on session start and turn end and never fail
-// or slow a turn.
+// Lookups run in the background on session start and turn end, the size also
+// every 10 seconds, and never fail or slow a turn.
 // @ts-nocheck
 
 import { execFile } from "node:child_process";
 import { existsSync, readFileSync } from "node:fs";
+import { lstat, open } from "node:fs/promises";
 import path from "node:path";
 import { promisify } from "node:util";
 
 const BLANK = "\u2800";
 const SOURCE = "code-factory:sidebar";
 const PR_REFRESH_MS = 5 * 60 * 1000;
+const SIZE_REFRESH_MS = 10 * 1000;
 const PARTS = ["pr", "issue", "add", "del", "files"];
 const taskId = process.env.FM_TASK_ID || "";
 const paneId = process.env.HERDR_PANE_ID;
 const workspaceId = process.env.HERDR_WORKSPACE_ID;
 const herdr = process.env.HERDR_BIN_PATH || "herdr";
 
 const run = async (cmd: string, args: string[], cwd?: string): Promise<string> =>
-  (await promisify(execFile)(cmd, args, { cwd, timeout: 15_000, encoding: "utf8" })).stdout;
+  (await promisify(execFile)(cmd, args, { cwd, timeout: 15_000, encoding: "utf8", maxBuffer: 64 << 20 })).stdout;
 
 export function titleFor(topic: string): string {
   return (taskId ? "└ " : "") + topic;
@@ -67,6 +69,55 @@ export function shortstat(out: string) {
   return { files: n(/(\d+) files? changed/), add: n(/(\d+) insertions?/), del: n(/(\d+) deletions?/) };
 }
 
+// Lines git counts in a new file: one for a symlink, none for a binary file (a
+// NUL in its first 8000 bytes, so a binary file is never read past that).
+async function newFileLines(file: string): Promise<number> {
+  const st = await lstat(file);
+  if (st.isSymbolicLink()) return 1;
+  if (!st.isFile()) return 0;
+  const fh = await open(file, "r");
+  try {
+    const buf = Buffer.alloc(8192);
+    let lines = 0;
+    let last = 10;
+    for (let first = true; ; first = false) {
+      const { bytesRead } = await fh.read(buf, 0, buf.length, null);
+      if (!bytesRead) return lines + (last === 10 ? 0 : 1);
+      const chunk = buf.subarray(0, bytesRead);
+      if (first && chunk.subarray(0, 8000).includes(0)) return 0;
+      for (let i = chunk.indexOf(10); i >= 0; i = chunk.indexOf(10, i + 1)) lines++;
+      last = chunk[bytesRead - 1];
+    }
+  } finally {
+    await fh.close();
+  }
+}
+
+// Size of the work in the checkout: committed, staged and unstaged tracked
+// changes since the merge base of `base` and HEAD, plus untracked files that
+// are not ignored. `base` falls back to origin/main, then origin/master; with
+// none of them there is no size ({}), and an empty diff is no size either.
+async function diffStat(cwd: string, base: string) {
+  const git = (...args: string[]) => run("git", ["-C", cwd, ...args]);
+  let ref;
+  for (const r of [base, "origin/main", "origin/master"]) {
+    if (await git("rev-parse", "--verify", "--quiet", `${r}^{commit}`).then(() => true, () => false)) {
+      ref = r;
+      break;
+    }
+  }
+  if (!ref) return {};
+  const stat = shortstat(await git("-c", "diff.autorefreshindex=false", "diff", "--shortstat", "--merge-base", ref));
+  // Counted the way git counts a new file: lines, one for a symlink, none for a binary file.
+  // ponytail: reads every untracked text file each refresh; cap the size read if huge untracked files show up.
+  for (const f of (await git("ls-files", "-z", "--others", "--exclude-standard", ":/")).split("\0")) {
+    if (!f || f.endsWith("/")) continue; // "" ends the list; "dir/" is a nested repository
+    stat.files++;
+    stat.add += await newFileLines(path.resolve(cwd, f)).catch(() => 0);
+  }
+  return stat.files ? stat : {};
+}
+
 // Token values for the pull request line; "" clears a part.
 export function prParts(v: { pr?; issue?; add?; del?; files? }, indent: number): Record<string, string> {
   const text = {
@@ -131,8 +182,10 @@ export default function (pi) {
   let pr; // open pull request for the current branch, from the REST API
   let taskIssue = "";
   let stat = {};
+  let reported; // the size last reported successfully
   let lookedUpAt = 0;
   let refreshing = false;
+  let sizeTimer;
 
   // Runs from a raw timer too, where a throw would take the session down.
   function applyTitle() {
@@ -156,6 +209,7 @@ export default function (pi) {
   }
 
   async function report() {
+    const sent = JSON.stringify(stat);
     let who = "";
     if (!taskId) {
       const label = JSON.parse(await run(herdr, ["workspace", "get", workspaceId])).result?.workspace?.label;
@@ -170,15 +224,17 @@ export default function (pi) {
     const args = ["pane", "report-metadata", paneId, "--source", SOURCE];
     for (const [k, v] of Object.entries({ who, ...parts })) args.push(...(v ? ["--token", `${k}=${v}`] : ["--clear-token", k]));
     await run(herdr, args);
+    reported = sent;
   }
 
   // Display-only: a failed lookup keeps the previous value. Re-reporting on
   // every call also restores tokens a Herdr restart dropped. A lookup asked
   // for while one runs or inside the throttle window is put off to the
-  // window's end, not dropped.
-  async function refresh(cwd?: string) {
+  // window's end, not dropped. `lookup` false (the size timer) skips the pull
+  // request lookup and reports only a size not yet reported successfully.
+  async function refresh(cwd?: string, lookup = true) {
     const wait = lookedUpAt + PR_REFRESH_MS - Date.now();
-    if (cwd && (refreshing || wait > 0) && !trailing) {
+    if (lookup && cwd && (refreshing || wait > 0) && !trailing) {
       trailing = setTimeout(() => {
         trailing = undefined;
         void refresh(current.cwd);
@@ -188,20 +244,19 @@ export default function (pi) {
     if (refreshing) return;
     refreshing = true;
     try {
-      if (cwd && wait <= 0) {
+      if (lookup && cwd && wait <= 0) {
         if (!lookedUpAt) await report().catch(() => {});
         lookedUpAt = Date.now();
         pr = await lookupPr(cwd).catch(() => pr);
         if (taskId) taskIssue = await lookupTaskIssue().catch(() => taskIssue);
       }
-      // A worker's size shows from its first commit, before any PR: diff against
-      // the PR base, else the remote's default branch.
+      // A worker's size shows as soon as its diff is non-empty, before any
+      // commit or PR: against the PR base, else the remote's default branch.
       if (cwd && (pr || taskId)) {
         const base = pr ? `origin/${pr.base?.repo?.default_branch || pr.base?.ref}` : "origin/HEAD";
-        stat = await run("git", ["-C", cwd, "diff", "--shortstat", `${base}...HEAD`]).then(shortstat, () => stat);
-        if (!pr && !stat.files) stat = {};
+        stat = await diffStat(cwd, base).catch(() => stat);
       }
-      await report();
+      if (lookup || JSON.stringify(stat) !== reported) await report();
     } catch {
     } finally {
       refreshing = false;
@@ -210,7 +265,14 @@ export default function (pi) {
 
   pi.on("session_start", (_event, ctx) => {
     track(ctx);
-    if (current) void refresh(current.cwd);
+    if (!current) return;
+    void refresh(current.cwd);
+    // A worker stays in one turn for most of its task: keep the size live.
+    // refresh() skips a tick while the previous refresh still runs.
+    if (!sizeTimer) {
+      sizeTimer = setInterval(() => void refresh(current.cwd, false), SIZE_REFRESH_MS);
+      sizeTimer.unref?.();
+    }
   });
   pi.on("session_switch", (_event, ctx) => track(ctx));
   pi.on("agent_start", (_event, ctx) => track(ctx));
@@ -224,6 +286,8 @@ export default function (pi) {
   pi.on("session_shutdown", () => {
     clearInterval(titleTimer);
     titleTimer = undefined;
+    clearInterval(sizeTimer);
+    sizeTimer = undefined;
     clearTimeout(trailing);
     const clear = ["who", ...PARTS].flatMap((k) => ["--clear-token", k]);
     if (current) void run(herdr, ["pane", "report-metadata", paneId, "--source", SOURCE, ...clear]).catch(() => {});

docs/herdr.md

@@ -90,7 +90,7 @@ The result carries `tokens.short`, `tokens.decisions`, and so on.
 | --- | --- |
 | 1 | The state dot, then the pane's `who` name: the same short names as Spaces. The primary home is bold blue, other homes bold mauve, and a dead helper (`☾ afk`) mauve, not bold. A spawned worker has no `who` name: its line is `└ ` and its session topic, in teal, because a worker works on one thing and its topic says what. |
 | 2 | A home's current session topic, dimmed on a brighter base (`#cdd6f4`) so it reads lighter than the name, without omp's `π` and spinner. Herdr's two-column indent puts it under the name. A worker has no line 2: its topic is on line 1. |
-| 3 | The pull request line, for any agent whose current branch has an open pull request, workers and homes alike, and for every worker: a worker shows its line counts from its first commit, before its pull request opens, and its issue when its task names one. Indented to line up under the name: four columns for a worker, two for a home. |
+| 3 | The pull request line, for any agent whose current branch has an open pull request, workers and homes alike, and for every worker: a worker shows its line counts as soon as its checkout differs from the default branch, before any commit or pull request, and its issue when its task names one. Indented to line up under the name: four columns for a worker, two for a home. |
 
 | Token | Value | Color |
 | --- | --- | --- |
@@ -100,7 +100,7 @@ The result carries `tokens.short`, `tokens.decisions`, and so on.
 | `$del` | `−<deleted>` lines (U+2212) | red |
 | `$files` | `✎ <files>` changed | yellow |
 
-Herdr indents every line after the first by two columns. For a worker, the first part present adds two U+2800 blank characters to that. The line counts come from `git diff --shortstat origin/<default branch>...HEAD` in the session's checkout, so they cover commits since the merge base with the default branch. For a worker with no pull request yet, the base is `origin/HEAD`, the remote's default branch; a checkout without an `origin/HEAD` ref shows no counts until its pull request opens (`git remote set-head origin --auto` adds the ref). The pull request comes from the GitHub REST API through `gh api`. It is looked up with the checkout's repository owner as the head owner first, then with the owner of each GitHub remote in `git remote -v`, in order, so a pull request opened from a fork shows as long as the checkout has the fork as a remote.
+Herdr indents every line after the first by two columns. For a worker, the first part present adds two U+2800 blank characters to that. The line counts cover the work in the session's checkout: `git diff --shortstat --merge-base <base>`, which counts committed, staged, and unstaged changes to tracked files since the merge base of the base and `HEAD`, plus every untracked file that is not ignored, counted as a new file (its lines, one for a symlink, or none for a binary file). The base is `origin/<default branch>` while a pull request is open, else `origin/HEAD`, the remote's default branch; a checkout without that ref uses `origin/main`, then `origin/master`, and with none of them shows no counts. An empty diff shows no counts either. The pull request comes from the GitHub REST API through `gh api`. It is looked up with the checkout's repository owner as the head owner first, then with the owner of each GitHub remote in `git remote -v`, in order, so a pull request opened from a fork shows as long as the checkout has the fork as a remote.
 
 The issue comes from the worker's task record in its home first: the first `issues/<n>` link or `issue <n>` / `issue #<n>` in the task's backlog entry (`data/backlog.md`), then in its brief (`data/<task>/brief.md`). The home is the ancestor of the pane's launch directory that holds `state/<task>.meta`. So a worker shows its issue before its pull request opens, after it merges, and when the body has no closing keyword. Without a task record, the issue is the first one the pull request body closes (`Closes #N`, `Fixes #N`, `Resolves #N`).
 
@@ -113,7 +113,7 @@ The extension does nothing outside Herdr (`HERDR_ENV` is not `1`) or in an omp s
 - Sets the terminal title to the bare session topic, and sets it again within a second whenever omp resets it (rename, `/new`, `/resume`, a cwd change). A spawned worker (`FM_TASK_ID` set) gets `└ ` in front; the Agents layout shows a title that starts with `└` on line 1 and hides it on line 2.
 - Reports the pane tokens `who`, `pr`, `issue`, `add`, `del` and `files` under the source `code-factory:sidebar`, and clears the ones without a value.
 
-It reports when a session starts and when a turn ends. The pull request and the task's issue are looked up then too, at most every 5 minutes; a turn that ends sooner gets its lookup when the 5 minutes are up. The line counts are recomputed on every turn end for a worker, and for a home while a pull request is open. Lookups run in the background with a timeout. A failed lookup keeps the previous value and never fails or slows a turn. Herdr drops pane tokens when its server restarts; the extension reports them again at the next turn end. All tokens are cleared when omp exits.
+It reports when a session starts and when a turn ends. The pull request and the task's issue are looked up then too, at most every 5 minutes; a turn that ends sooner gets its lookup when the 5 minutes are up. The line counts are recomputed on every turn end and every 10 seconds while the session runs, for a worker and for a home while a pull request is open, so they follow edits, stages, and commits inside a long turn; the 10-second refresh runs only the local `git` commands (never rewriting the checkout's index), skips a tick while the previous refresh still runs, and reports only counts not yet reported successfully, so a failed report is retried at the next tick. Lookups run in the background with a timeout. A failed lookup keeps the previous value and never fails or slows a turn. Herdr drops pane tokens when its server restarts; the extension reports them again at the next turn end. All tokens are cleared when omp exits.
 
 Inspect what a pane reports: