diff --git a/web/index.d.ts b/web/index.d.ts index a0a6e41..0a74ed4 100644 --- a/web/index.d.ts +++ b/web/index.d.ts @@ -61,7 +61,10 @@ export class MarkitRenderer { /** Resolves once this source is on the page, or with `superseded: true` when newer * text replaced it first. Results reach the DOM at most once per animation frame * (a macrotask when the page is hidden or has no requestAnimationFrame); the open - * last block is updated in place and settled blocks are never rebuilt. */ + * last block is updated in place and settled blocks are never rebuilt. While + * `finalized` is false, markup still unfinished at the end of the last line (an open + * backtick, `[text](url` before its `)`, a table header before its delimiter row) waits + * until it closes, the line ends or the message finishes, so text on screen only grows. */ render(source: string, options?: { finalized?: boolean }): Promise; /** Colours the whole source, up to 8 MiB of UTF-8; larger sources come back as one plain token * with `plain: true`. Unknown non-empty languages use a generic lexer; `''` and `'text'` stay plain. diff --git a/web/index.js b/web/index.js index 45759e8..7ce37e9 100644 --- a/web/index.js +++ b/web/index.js @@ -5,6 +5,7 @@ import { morphChildren } from './dom-morph.js'; import { engineFor } from './engine.js'; import { scheduleFrame } from './frame.js'; import { GithubFeatures } from './github.js'; +import { StreamHold } from './stream-hold.js'; export { preloadMarkit, terminateMarkitWorkers } from './engine.js'; const SECTIONS = 'section[data-markit-code]'; @@ -23,6 +24,8 @@ export class MarkitRenderer { #payloads = new WeakMap(); #views = new WeakMap(); #urls = new Map(); #imageJobs = new Map(); #listeners; #gpu = new GpuCodeRenderer(); #github; + /** What of an open message is safe to show: unfinished markup at its end waits. */ + #hold = new StreamHold(); constructor(element, options = {}) { this.#element = element; this.#options = options; @@ -77,6 +80,8 @@ export class MarkitRenderer { if (this.#destroyed) return Promise.reject(new Error('Renderer disposed')); if (typeof source !== 'string') return Promise.reject(new TypeError('Markdown must be a string')); const revision = ++this.#revision; + if (finalized) this.#hold.showAll(source); + else source = source.slice(0, this.#hold.shownLength(source)); return new Promise((resolve, reject) => { if (this.#queued) this.#queued.resolve({ superseded: true }); this.#queued = { source, finalized, revision, resolve, reject }; diff --git a/web/package.json b/web/package.json index d5b1885..09a22b4 100644 --- a/web/package.json +++ b/web/package.json @@ -26,6 +26,7 @@ "code-lines.js", "gpu-code.js", "github.js", + "stream-hold.js", "style.css", "pkg", "licenses", diff --git a/web/stream-hold.js b/web/stream-hold.js new file mode 100644 index 0000000..aba6742 --- /dev/null +++ b/web/stream-hold.js @@ -0,0 +1,194 @@ +// While a message streams, its last line can end in markup that is not finished yet: a +// backtick waiting for its pair, `[text](url` before the `)`, `**` before the closing `**`, +// a table header before its delimiter row. Rendered as it stands, that markup shows as +// literal characters, and a moment later, when it closes, they are taken back: the line +// flickers. `StreamHold` finds where unfinished markup starts. The renderer shows the text +// before it, and the rest once the markup closes, the line ends or the message finishes. +// Each update scans only the lines that completed since the last one and the open line. + +const FENCE = /^ {0,3}(`{3,}|~{3,})(.*)$/; +const PIPE_ROW = /^ {0,3}\|/; +/** Block markers ahead of a line's inline text: indent, quotes, a list marker, a task box, a heading. */ +const LINE_PREFIX = /^ {0,3}(?:> ?)*(?:(?:[-*+]|\d{1,9}[.)])(?: +|$)(?:\[[ xX]\] +)?|#{1,6}(?: +|$))?/; +/** + * A line that is only the start of a block marker whose meaning the next character decides: + * a fence, a list marker, a table pipe, a rule or heading underline. + */ +const UNDECIDED_LINE = /^\s*(?:`{1,2}|~{1,2}|\d{1,9}|\|[\s|]*|(?:> ?)*(?:[-*+]|\d{1,9}[.)])\s*|[-=_*](?:\s*[-=_*])*\s*)$/; +const SPACE = /\s/; +const WORD = /[\p{L}\p{N}]/u; +const DIGIT = /[0-9]/; + +export class StreamHold { + #source = ''; #shown = 0; + /** Offset just past the last complete line scanned, and the fence open there. */ + #scanned = 0; #fence = null; + + /** How much of `source`, a message still streaming, to show now. */ + shownLength(source) { + if (!source.startsWith(this.#source)) { this.#scanned = 0; this.#fence = null; this.#shown = 0; } + this.#source = source; + const open = source.lastIndexOf('\n') + 1; + this.#scanFences(source, open); + const held = this.#fence ? fenceHold(source, open, this.#fence) : heldFrom(source, open); + // Shown text never shrinks, even if a later character changes what came before. + this.#shown = Math.max(this.#shown, held === -1 ? source.length : held); + return this.#shown; + } + + /** `source` was shown whole (the message finished). */ + showAll(source) { + this.shownLength(source); + this.#shown = source.length; + } + + #scanFences(source, open) { + while (this.#scanned < open) { + const end = source.indexOf('\n', this.#scanned); + const line = source.slice(this.#scanned, end); + this.#scanned = end + 1; + const fence = FENCE.exec(line); + if (!fence) continue; + const [, marks, rest] = fence; + if (!this.#fence) { + if (!(marks[0] === '`' && rest.includes('`'))) this.#fence = { mark: marks[0], length: marks.length }; + } else if (marks[0] === this.#fence.mark && marks.length >= this.#fence.length && !rest.trim()) this.#fence = null; + } + } +} + +/** Inside a code fence only its closing line can be unfinished: those marks are no code. */ +function fenceHold(source, open, fence) { + const line = source.slice(open); + return line && new RegExp(`^ {0,3}\\${fence.mark}*$`).test(line) ? open : -1; +} + +/** Where unfinished markup starts in the open line (from `open`), or -1 when it has none. */ +function heldFrom(source, open) { + const line = source.slice(open); + const previous = lineBefore(source, open); + const table = tableHold(source, open, line, previous); + if (table !== -1) return table; + if (UNDECIDED_LINE.test(line)) return open; + const at = inlineHold(line, LINE_PREFIX.exec(line)[0].length); + // What stays of the line must read as it will: not a lone list marker or table pipe. + return at === -1 ? -1 : UNDECIDED_LINE.test(line.slice(0, at)) ? open : open + at; +} + +function lineBefore(source, open) { + if (open === 0) return undefined; + const start = source.lastIndexOf('\n', open - 2) + 1; + return { start, text: source.slice(start, open - 1) }; +} + +/** A table shows once its delimiter row is complete; until then its header could be prose. */ +function tableHold(source, open, line, previous) { + if (PIPE_ROW.test(line) && !(previous && PIPE_ROW.test(previous.text))) return open; + if (!previous || !PIPE_ROW.test(previous.text)) return -1; + const before = lineBefore(source, previous.start); + // `previous` is a header whose delimiter row is still being written (or not started). + if (!(before && PIPE_ROW.test(before.text)) && (line === '' || PIPE_ROW.test(line))) + return previous.start; + return -1; +} + +/** Where the first unfinished inline span of `line` opens, scanning from `from`; -1 if none. */ +function inlineHold(line, from) { + const closers = new Set(); + for (let i = from; i < line.length;) { + const c = line[i]; + if (c === '\\') { i += 2; continue; } + if (c === '`') { + const after = codeSpanEnd(line, i); + if (after === -1 || after === line.length) return i; + i = after; continue; + } + if (c === '[' || (c === '!' && line[i + 1] === '[')) { + const after = linkEnd(line, c === '!' ? i + 1 : i); + if (after === -1) return i; + i = after; continue; + } + if (c === '<') { + if (/^(?:[A-Za-z/][^\s<>]*)?$/.test(line.slice(i + 1))) return i; + i++; continue; + } + if (c === '*' || c === '_' || c === '~' || c === '$') { + const length = runLength(line, i); + if (closers.has(i)) { i += length; continue; } + const held = delimiterHold(line, i, length, closers); + if (held !== -1) return held; + i += length; continue; + } + i++; + } + return -1; +} + +function runLength(line, at) { + let end = at; + while (line[end] === line[at]) end++; + return end - at; +} + +/** Just past the code span opening at `at`, or -1 while its closing run has not arrived. */ +function codeSpanEnd(line, at) { + const length = runLength(line, at); + for (let j = line.indexOf('`', at + length); j !== -1; j = line.indexOf('`', j)) { + const run = runLength(line, j); + if (run === length) return j + run; + j += run; + } + return -1; +} + +/** Just past a link or bracket opening at `at`, or -1 while it could still become a link. */ +function linkEnd(line, at) { + let depth = 0, close = -1; + for (let j = at; j < line.length; j++) { + const c = line[j]; + if (c === '\\') { j++; continue; } + if (c === '`') { const after = codeSpanEnd(line, j); if (after === -1) return -1; j = after - 1; continue; } + if (c === '[') depth++; + else if (c === ']' && --depth === 0) { close = j; break; } + } + if (close === -1 || close + 1 === line.length) return -1; + const next = line[close + 1]; + if (next === '[') { const end = line.indexOf(']', close + 2); return end === -1 ? -1 : end + 1; } + // Plain brackets: their text reads on, inline markup inside them included. + if (next !== '(') return at + 1; + let parens = 0; + for (let j = close + 1; j < line.length; j++) { + if (line[j] === '\\') { j++; continue; } + if (line[j] === '(') parens++; + else if (line[j] === ')' && --parens === 0) return j + 1; + } + return -1; +} + +/** + * An emphasis, strikethrough or math run at `at`: where to hold from when it opens a span + * that has not closed yet, else -1. A closing run it finds is noted in `closers`. + */ +function delimiterHold(line, at, length, closers) { + const mark = line[at], before = line[at - 1], next = line[at + length]; + // A run at the very end could still open a span with the next character. + if (next === undefined) return at; + if (mark === '~' && length !== 2) return -1; + if (mark === '$' && (length !== 1 || DIGIT.test(next))) return -1; + if (SPACE.test(next)) return -1; + if (mark === '_' && before !== undefined && WORD.test(before)) return -1; + for (let j = line.indexOf(mark, at + length); j !== -1; j = line.indexOf(mark, j)) { + const run = runLength(line, j), after = line[j + run]; + const closes = !SPACE.test(line[j - 1]) && (mark !== '_' || after === undefined || !WORD.test(after)) + && (mark !== '$' || run === 1) && (mark !== '~' || run === 2); + if (closes) { + // A closing run at the end could still grow, or (math) gain a digit that undoes it. + if (after === undefined) return at; + if (mark === '$' && DIGIT.test(after)) { j += run; continue; } + closers.add(j); + return -1; + } + j += run; + } + return at; +} diff --git a/web/tests/node/stream-hold.test.mjs b/web/tests/node/stream-hold.test.mjs new file mode 100644 index 0000000..6a42ab4 --- /dev/null +++ b/web/tests/node/stream-hold.test.mjs @@ -0,0 +1,98 @@ +// A streaming message must only grow on screen: text markit showed is never taken back when +// the markup around it completes (a backtick that becomes a code span, `[text](url` that +// becomes a link). Run with `npm run test:node`. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { JSDOM } from 'jsdom'; + +const wasm = await readFile(new URL('../../pkg/markit_wasm_bg.wasm', import.meta.url)); +const { MarkitRenderer, preloadMarkit } = await import('../../index.js'); +preloadMarkit({ wasm }); + +function mount() { + const { window } = new JSDOM(''); + const element = window.document.createElement('div'); + window.document.body.append(element); + return { element, renderer: new MarkitRenderer(element, { wasm, flavor: 'agent' }) }; +} + +/** What a reader sees, as text: whitespace runs are one space, trailing space is not yet text. */ +const shown = element => element.textContent.replace(/\s+/g, ' ').trimEnd(); + +/** An agent's answer with every inline and block form that changes once it is closed. */ +const answer = `Starting the thread now. The cleanup just found: \`rm -Rf\` with an empty **\`$TMPDIR\`** would delete from the root, see [the cleanup notes](https://example.com/cleanup) and . + +What changed: + +1. \`scripts/clean.ts\` checks the variable before it deletes *anything*; +2. the sweep skips ~~every path~~ paths outside \`/tmp/ace-*\`, costing $5 less, since $n^2$ grows fast. + +| Case | Result | +| --- | --- | +| empty \`TMPDIR\` | refused | + +\`\`\`ts +export function cleanRoot(env) { + return env.TMPDIR; +} +\`\`\` + +Next I'll run \`bun run test\` and report back in [the thread](https://example.com/thread).`; + +/** Streams `source` in chunks of the given sizes and returns what was on screen after each. */ +async function stream(source, sizes) { + const { element, renderer } = mount(); + const screens = []; + for (let at = 0, index = 0; at < source.length; index++) { + at = Math.min(source.length, at + sizes[index % sizes.length]); + await renderer.render(source.slice(0, at), { finalized: false }); + screens.push(shown(element)); + } + await renderer.render(source); + screens.push(shown(element)); + renderer.destroy(); + return screens; +} + +for (const [name, sizes] of [['a character at a time', [1]], ['uneven chunks', [7, 3, 11, 5, 9, 4]]]) { + test(`a streamed answer never takes back text it showed (${name})`, async () => { + const screens = await stream(answer, sizes); + for (let index = 1; index < screens.length; index++) { + const before = screens[index - 1], after = screens[index]; + assert.ok(after.startsWith(before), `update ${index} took back text:\n showed …${before.slice(-60)}\n then …${after.slice(-60)}`); + } + const fresh = mount(); + await fresh.renderer.render(answer); + assert.equal(screens.at(-1), shown(fresh.element)); + fresh.renderer.destroy(); + }); +} + +test('unfinished inline markup waits until it closes, then shows whole', async () => { + const { element, renderer } = mount(); + await renderer.render('The cleanup found: `rm -R', { finalized: false }); + assert.equal(shown(element), 'The cleanup found:'); + await renderer.render('The cleanup found: `rm -Rf` with', { finalized: false }); + assert.equal(element.querySelector('code')?.textContent, 'rm -Rf'); + assert.equal(shown(element), 'The cleanup found: rm -Rf with'); + renderer.destroy(); +}); + +test('held text shows once its line ends, and a finished message shows everything', async () => { + const { element, renderer } = mount(); + await renderer.render('It is 5` tall and [unsure', { finalized: false }); + assert.equal(shown(element), 'It is 5'); + await renderer.render('It is 5` tall and [unsure\n\nNext', { finalized: false }); + assert.equal(shown(element), 'It is 5` tall and [unsure Next'); + await renderer.render('Ends with `open'); + assert.equal(shown(element), 'Ends with `open'); + renderer.destroy(); +}); + +test('code in an open fence streams as it arrives', async () => { + const { element, renderer } = mount(); + await renderer.render('```ts\nconst a = `templ', { finalized: false }); + assert.match(element.querySelector('code').textContent, /const a = `templ/); + renderer.destroy(); +});