diff --git a/packages/ui/src/__tests__/markdown-body.test.ts b/packages/ui/src/__tests__/markdown-body.test.ts index eca7a03013..15403cdab3 100644 --- a/packages/ui/src/__tests__/markdown-body.test.ts +++ b/packages/ui/src/__tests__/markdown-body.test.ts @@ -18,6 +18,7 @@ */ import { strict as assert } from 'node:assert'; +import { performance } from 'node:perf_hooks'; import { createElement } from 'react'; import { renderToStaticMarkup } from 'react-dom/server'; import { it } from 'node:test'; @@ -31,6 +32,7 @@ import { import { AstryxLocaleProvider } from '../astryx-i18n.js'; import { MakaUriContext, Markdown } from '../markdown.js'; import { LocaleProvider } from '../locale-context.js'; +import { createMarkdownMathCache, prepareMarkdownMath } from '../markdown-math.js'; import { createMermaidConfig, MAX_MERMAID_EDGES, @@ -301,6 +303,410 @@ it('does not let multiline display math cross a fenced code block', () => { assert.match(markup, /inside/); }); +it('keeps escaped brackets in link labels out of display math', () => { + const markup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'zh-CN', + children: createElement(MarkdownBody, { + text: '[\\[DISCUSS\\] Clarify Maka sandbox contracts and runtime dependency access](https://github.com/apache/maka/discussions/5304)', + }), + })); + + assert.match(markup, /]*href="https:\/\/github\.com\/apache\/maka\/discussions\/5304"/); + assert.doesNotMatch(markup, /maka-math-display/); + assert.doesNotMatch(markup, /katex-display/); + assert.match(markup, /\[DISCUSS\] Clarify Maka sandbox contracts and runtime dependency access/); +}); + +it('still renders inline math inside link labels', () => { + const markup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { + text: '[see \\(x+1\\) here](https://example.com)', + }), + })); + + assert.match(markup, /]*href="https:\/\/example\.com"/); + assert.match(markup, /class="maka-math maka-math-inline"/); + assert.doesNotMatch(markup, /maka-math-display/); +}); + +it('keeps dollar display math in link labels as literal text', () => { + const markup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { + text: '[a $$x^2$$ b](https://example.com)', + }), + })); + + assert.match(markup, /]*href="https:\/\/example\.com"/); + assert.doesNotMatch(markup, /maka-math-display/); + assert.match(markup, /\$\$x\^2\$\$/); +}); + +it('still renders display math outside link labels', () => { + const markup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { + text: '[plain](https://example.com)\n\n\\[ y^2 \\]', + }), + })); + + assert.match(markup, /]*href="https:\/\/example\.com"/); + assert.match(markup, /class="maka-math maka-math-display"/); + assert.match(markup, /class="katex-display"/); +}); + +it('preserves escaped brackets across reference link forms', () => { + const cases = [ + { + use: '[\\[DISCUSS\\] Clarify][topic]', + definition: '[topic]: https://example.com/topic', + }, + { + use: '[\\[DISCUSS\\] Clarify][]', + definition: '[\\[DISCUSS\\] Clarify]: https://example.com/collapsed', + }, + { + use: '[\\[DISCUSS\\] Clarify]', + definition: '[\\[DISCUSS\\] Clarify]: https://example.com/shortcut', + }, + ]; + + for (const { use, definition } of cases) { + const markup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { + text: `${use}\n\n${definition}`, + }), + })); + + assert.match(markup, /]*href="https:\/\/example\.com\//); + assert.match(markup, /\[DISCUSS\] Clarify/); + assert.doesNotMatch(markup, /maka-math-display|katex-display/); + } +}); + +it('matches escaped reference identifiers between use and definition', () => { + const markup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { + text: '[visible][\\[topic\\]]\n\n[\\[topic\\]]: https://example.com/ref', + }), + })); + + assert.match(markup, /]*href="https:\/\/example\.com\/ref"/); + assert.match(markup, />visible { + const full = '[label](https://example.com/$$value$$)'; + const cache = createMarkdownMathCache(); + let incremental = ''; + for (let end = 1; end <= full.length; end++) { + incremental = prepareMarkdownMath(full.slice(0, end), cache); + } + + assert.equal(incremental, prepareMarkdownMath(full, createMarkdownMathCache())); + assert.doesNotMatch(incremental, /MAKA_MATH/); + + const markup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { text: full }), + })); + + assert.match(markup, /href="https:\/\/example\.com\/\$\$value\$\$"/); +}); + +it('preserves escape parity when an incremental scan resumes inside a backslash run', () => { + const cases = [ + '\\\\[\\] x]', + '\\\\\\\\[\\] x]', + '\\\\[```\\[(\\]', + ]; + + for (const full of cases) { + const cache = createMarkdownMathCache(); + let incremental = ''; + for (let end = 1; end <= full.length; end++) { + incremental = prepareMarkdownMath(full.slice(0, end), cache); + } + + assert.equal(incremental, prepareMarkdownMath(full, createMarkdownMathCache()), full); + } +}); + +it('matches escaped image reference identifiers between use and definition', () => { + const markup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { + text: '![visible][\\[topic\\]]\n\n[\\[topic\\]]: https://example.com/image.png', + }), + })); + + assert.match(markup, /]*src="https:\/\/example\.com\/image\.png"/); + assert.match(markup, /alt="visible"/); + assert.doesNotMatch(markup, /maka-math-display|katex-display/); +}); + +it('keeps image alt escapes out of math without leaking transport tokens', () => { + const markup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { + text: '![\\[alt\\] preview](https://example.com/x.png)', + }), + })); + + assert.doesNotMatch(markup, /maka-math/); + assert.doesNotMatch(markup, /MAKA_MATH/); +}); + +it('keeps a split image opener identical between incremental and one-shot scans', () => { + const full = '!![alt \\[x\\]](https://example.com/a.png)'; + const cache = createMarkdownMathCache(); + let incremental = ''; + for (let end = 1; end <= full.length; end++) { + incremental = prepareMarkdownMath(full.slice(0, end), cache); + } + + assert.equal(incremental, prepareMarkdownMath(full, createMarkdownMathCache())); + + const markup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { text: full }), + })); + + assert.match(markup, /]*src="https:\/\/example\.com\/a\.png"/); + assert.match(markup, /alt="alt \[x\]"/); + assert.doesNotMatch(markup, /maka-math/); + assert.doesNotMatch(markup, /MAKA_MATH/); +}); + +it('settles bounded labels ending in $ instead of rescanning the stream', () => { + const head = '[price$](https://example.com)'; + const filler = `\n\n${'lorem ipsum dolor sit amet. '.repeat(16_384)}`; + const full = head + filler; + const cache = createMarkdownMathCache(); + const updates = 64; + const started = performance.now(); + let incremental = ''; + for (let step = 1; step <= updates; step++) { + incremental = prepareMarkdownMath(full.slice(0, Math.ceil((full.length * step) / updates)), cache); + } + const elapsed = performance.now() - started; + + assert.equal(incremental, prepareMarkdownMath(full, createMarkdownMathCache())); + assert.equal(cache.safeSourceEnd, full.length); + assert.ok(elapsed < 5_000, `label-$ streaming scan took ${elapsed.toFixed(1)}ms`); +}); + +it('renders images whose alt contains escaped brackets', () => { + const cases = [ + { + text: '![\\[alt\\] preview](https://example.com/x.png)', + alt: '[alt] preview', + }, + { + text: '![\\[alt\\] preview][pic]\n\n[pic]: https://example.com/x.png', + alt: '[alt] preview', + }, + { + text: '![visible][\\[topic\\]]\n\n[\\[topic\\]]: https://example.com/image.png', + alt: 'visible', + }, + { + text: '![\\[topic\\]][]\n\n[\\[topic\\]]: https://example.com/image.png', + alt: '[topic]', + }, + { + text: '![\\[topic\\]]\n\n[\\[topic\\]]: https://example.com/image.png', + alt: '[topic]', + }, + ]; + + for (const { text, alt } of cases) { + const markup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { text }), + })); + + assert.match(markup, /]*src="https:\/\/example\.com\//, text); + assert.match(markup, new RegExp(`alt="${alt.replace(/[[\]]/g, '\\$&')}"`), text); + assert.doesNotMatch(markup, /maka-math/, text); + assert.doesNotMatch(markup, /MAKA_MATH/, text); + } +}); + +it('keeps escaped image alt text identical between incremental and one-shot scans', () => { + const full = '![\\[alt\\] preview](https://example.com/a.png)'; + const cache = createMarkdownMathCache(); + let incremental = ''; + for (let end = 1; end <= full.length; end++) { + incremental = prepareMarkdownMath(full.slice(0, end), cache); + } + + assert.equal(incremental, prepareMarkdownMath(full, createMarkdownMathCache())); + + const markup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { text: full }), + })); + + assert.match(markup, /]*src="https:\/\/example\.com\/a\.png"/); + assert.match(markup, /alt="\[alt\] preview"/); + assert.doesNotMatch(markup, /maka-math/); + assert.doesNotMatch(markup, /MAKA_MATH/); +}); + +it('preserves links and images with labels past the defensive scan bound', () => { + const longLink = `[\\[DISCUSS\\] ${'a'.repeat(4096)}](https://example.com/long)`; + const linkMarkup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { text: longLink }), + })); + + assert.match(linkMarkup, /]*href="https:\/\/example\.com\/long"/); + assert.match(linkMarkup, /\[DISCUSS\]/); + assert.doesNotMatch(linkMarkup, /maka-math-display|katex-display/); + + const longImg = `![\\[alt\\] ${'b'.repeat(4096)}](https://example.com/y.png)`; + const imgMarkup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { text: longImg }), + })); + + assert.match(imgMarkup, /]*src="https:\/\/example\.com\/y\.png"/); + assert.match(imgMarkup, /alt="\[alt\] b/); + assert.doesNotMatch(imgMarkup, /maka-math/); + assert.doesNotMatch(imgMarkup, /MAKA_MATH/); +}); + +it('resolves labels far beyond any scan bound without a length cliff', () => { + const bigLink = `[\\[DISCUSS\\] ${'a'.repeat(100_000)}](https://example.com/huge)`; + const cache = createMarkdownMathCache(); + const prepared = prepareMarkdownMath(bigLink, cache); + + assert.equal(cache.safeSourceEnd, bigLink.length); + + const markup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { text: bigLink }), + })); + + assert.match(markup, /]*href="https:\/\/example\.com\/huge"/); + assert.match(markup, /\[DISCUSS\]/); + assert.doesNotMatch(markup, /maka-math-display|katex-display/); +}); + +it('streams an unfinished label without rescanning from its opener', () => { + const full = `[${'a'.repeat(128_000 - 1)}`; + const cache = createMarkdownMathCache(); + const started = performance.now(); + let incremental = ''; + for (let end = 1024; end <= full.length; end += 1024) { + incremental = prepareMarkdownMath(full.slice(0, end), cache); + } + const elapsed = performance.now() - started; + + assert.equal(incremental, full); + assert.equal(cache.safeSourceEnd, 0); + assert.ok(elapsed < 5_000, `unfinished label streaming took ${elapsed.toFixed(1)}ms`); +}); + +it('resolves a streamed label once its closer arrives', () => { + const head = `[${'b'.repeat(64_000)}`; + const cache = createMarkdownMathCache(); + for (let end = 1024; end <= head.length; end += 1024) { + prepareMarkdownMath(head.slice(0, end), cache); + } + const full = `${head}](https://example.com/closed)`; + const incremental = prepareMarkdownMath(full, cache); + + assert.equal(incremental, prepareMarkdownMath(full, createMarkdownMathCache())); + assert.equal(cache.safeSourceEnd, full.length); + + const markup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { text: full }), + })); + + assert.match(markup, /]*href="https:\/\/example\.com\/closed"/); +}); + +it('streams an unfinished reference identifier without rescanning from its opener', () => { + const head = `[visible][${'c'.repeat(64_000)}`; + const cache = createMarkdownMathCache(); + const started = performance.now(); + let incremental = ''; + for (let end = 1024; end <= head.length; end += 1024) { + incremental = prepareMarkdownMath(head.slice(0, end), cache); + } + if (head.length % 1024 !== 0) { + incremental = prepareMarkdownMath(head, cache); + } + const elapsed = performance.now() - started; + + assert.equal(incremental, head); + assert.equal(cache.safeSourceEnd, 0); + assert.ok(elapsed < 5_000, `unfinished identifier streaming took ${elapsed.toFixed(1)}ms`); + + const id = 'c'.repeat(64_000); + const full = `[visible][${id}]\n\n[${id}]: https://example.com/ref`; + incremental = prepareMarkdownMath(full, cache); + + assert.equal(incremental, prepareMarkdownMath(full, createMarkdownMathCache())); + assert.equal(cache.safeSourceEnd, full.length); + + const markup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { text: full }), + })); + + assert.match(markup, /]*href="https:\/\/example\.com\/ref"/); + assert.match(markup, />visible { + const cases = [ + // Unclosed code span, then its closer plus display math. + { head: '`[', full: '`[` $$x$$' }, + // Unclosed inline math, then its closer plus display math. + { head: '\\([x', full: '\\([x\\) $$y$$' }, + // Unclosed display math, then its closer plus display math. + { head: '$$[', full: '$$[$$ $$y$$' }, + ]; + + for (const { head, full } of cases) { + const cache = createMarkdownMathCache(); + prepareMarkdownMath(head, cache); + const incremental = prepareMarkdownMath(full, cache); + + assert.equal(incremental, prepareMarkdownMath(full, createMarkdownMathCache()), head); + } +}); + +it('does not rescan malformed link tails quadratically', () => { + const input = '[x]('.repeat(32_000); + const cache = createMarkdownMathCache(); + const started = performance.now(); + const prepared = prepareMarkdownMath(input, cache); + const elapsed = performance.now() - started; + + assert.equal(prepared, input); + assert.ok(elapsed < 1_000, `malformed link scan took ${elapsed.toFixed(1)}ms`); + + const streamedInput = '[x]('.repeat(16_000); + const streamedCache = createMarkdownMathCache(); + const streamedStarted = performance.now(); + const chunkSize = streamedInput.length / 16; + for (let end = chunkSize; end <= streamedInput.length; end += chunkSize) { + assert.equal(prepareMarkdownMath(streamedInput.slice(0, end), streamedCache), streamedInput.slice(0, end)); + } + const streamedElapsed = performance.now() - streamedStarted; + + assert.ok(streamedElapsed < 1_000, `streaming malformed link scan took ${streamedElapsed.toFixed(1)}ms`); +}); + it('keeps the copy control in a toolbar above a one-line code scroll viewport', () => { const markup = renderToStaticMarkup(createElement(LocaleProvider, { locale: 'en', diff --git a/packages/ui/src/markdown-body.tsx b/packages/ui/src/markdown-body.tsx index af17e36ee6..451386ec98 100644 --- a/packages/ui/src/markdown-body.tsx +++ b/packages/ui/src/markdown-body.tsx @@ -49,7 +49,9 @@ import { MermaidDiagram } from './mermaid-diagram.js'; import { createMarkdownMathCache, MarkdownMath, + MARKDOWN_MATH_PLUGINS, prepareMarkdownMath, + restoreTransportTokens, } from './markdown-math.js'; import { parseAttachmentResourceRef } from '@maka/core/attachments'; import { useAttachmentImageSource } from './attachment-image.js'; @@ -195,6 +197,7 @@ export function MarkdownBody(props: { // the one combination neither half of the argument asks for. density={density} components={components} + inlinePlugins={MARKDOWN_MATH_PLUGINS} isStreaming={props.streaming} settledText={props.settledText} transformSource={transformMathSource} @@ -272,25 +275,28 @@ function MarkdownCode(props: { } function MarkdownImage(props: { src: string; alt: string }) { + // Alt arrives as a raw string, so restore any transport tokens the math + // preprocessing left there back to plain text before rendering. + const alt = restoreTransportTokens(props.alt); const attachment = parseAttachmentResourceRef(props.src); const attachmentSrc = useAttachmentImageSource( attachment ? { artifactId: attachment.artifactId } : undefined, ); if (attachment) { - if (!attachmentSrc) return [{props.alt}]; + if (!attachmentSrc) return [{alt}]; return ( {props.alt} ); } - if (!isSafeMarkdownImageUrl(props.src)) return [{props.alt}]; + if (!isSafeMarkdownImageUrl(props.src)) return [{alt}]; // Remote images can be badges or sentence-level icons, so preserve Maka's // existing inline presentation. Session attachments above are content // previews and deliberately own a block presentation instead. - return {props.alt}; + return {alt}; } function isSafeMarkdownImageUrl(url: string): boolean { diff --git a/packages/ui/src/markdown-math.tsx b/packages/ui/src/markdown-math.tsx index 7a7ba1c077..1e01473f27 100644 --- a/packages/ui/src/markdown-math.tsx +++ b/packages/ui/src/markdown-math.tsx @@ -18,6 +18,12 @@ */ import katex from 'katex'; +import type { MarkdownInlinePlugin } from '@astryxdesign/core/Markdown'; + +const TOKEN_START = '\uE000MAKA_MATH:'; +const TOKEN_END = '\uE001'; +const TOKEN_PATTERN = /\uE000MAKA_MATH:([012]):([0-9a-f]+)\uE001/g; +const LITERAL_TOKEN_PATTERN = /^\uE000MAKA_MATH:[012]:[0-9a-f]+\uE001/; /** * Renders upstream `components.math` nodes through KaTeX. The delimiter-free @@ -68,13 +74,39 @@ export function prepareMarkdownMath( // keeps the scanner on the changing tail; it does not make the full-string // identity check itself incremental. const extendsPrevious = source.startsWith(cache.source); + if (!extendsPrevious) { + pendingLabelScans.delete(cache); + } else { + const pending = pendingLabelScans.get(cache); + if (pending !== undefined) { + // The unfinished label is the only unsettled business: resume its scan + // over the appended bytes instead of re-walking from its opener. + const continued = scanLabel(source, pending); + if (continued.kind === 'pending') { + pendingLabelScans.set(cache, continued.state); + cache.text += source.slice(cache.source.length); + cache.source = source; + return cache.text; + } + pendingLabelScans.delete(cache); + } + } const sourceStart = extendsPrevious ? cache.safeSourceEnd : 0; const textStart = extendsPrevious ? cache.safeTextEnd : 0; - const translatedTail = translateMarkdownMath( - source.slice(sourceStart), - sourceStart === 0 || source[sourceStart - 1] === '\n', - cache.text[textStart - 1], - ); + const translatedTail = translateMarkdownMath(source.slice(sourceStart), { + startsAtLineStart: sourceStart === 0 || source[sourceStart - 1] === '\n', + priorTextChar: cache.text[textStart - 1], + leadingBackslashes: countPrecedingBackslashes(source, sourceStart), + onLabelPending: (state) => { + // The scan ran on the sliced tail, so its positions are relative to + // sourceStart; the continuation resumes on the full source and needs + // absolute positions. + state.index += sourceStart; + if (state.runStart !== null) state.runStart += sourceStart; + state.codeSearchFrom += sourceStart; + pendingLabelScans.set(cache, state); + }, + }); const text = `${extendsPrevious ? cache.text.slice(0, textStart) : ''}${translatedTail.text}`; cache.source = source; @@ -84,9 +116,46 @@ export function prepareMarkdownMath( return text; } +export const MARKDOWN_MATH_PLUGINS = [{ + pattern: TOKEN_PATTERN, + render: (match, key) => { + const formula = decodeFormula(match[2] ?? ''); + if (match[1] === '2') return formula; + const displayMode = match[1] === '1'; + const html = katex.renderToString(formula, { + displayMode, + output: 'htmlAndMathml', + strict: 'warn', + throwOnError: false, + trust: false, + }); + return ( + + ); + }, +}] satisfies MarkdownInlinePlugin[]; + const ZWSP = '\u200B'; const WORD_CHAR = /[\w$]/; +type TranslateMarkdownMathOptions = { + startsAtLineStart?: boolean; + priorTextChar?: string; + allowDisplayMath?: boolean; + protectEscapedBrackets?: boolean; + isFinalSegment?: boolean; + leadingBackslashes?: number; + onLabelPending?: (state: LabelScanState) => void; +}; + /** * Translate Maka's math delimiters into the upstream grammar and neutralize * bare `$` so prose never forms accidental inline math. @@ -100,8 +169,15 @@ const WORD_CHAR = /[\w$]/; */ function translateMarkdownMath( source: string, - startsAtLineStart = true, - priorTextChar?: string, + { + startsAtLineStart = true, + priorTextChar, + allowDisplayMath = true, + protectEscapedBrackets = false, + isFinalSegment = false, + leadingBackslashes = 0, + onLabelPending, + }: TranslateMarkdownMathOptions = {}, ): { text: string; safeSourceEnd: number; @@ -123,6 +199,8 @@ function translateMarkdownMath( const fence = atLineStart ? readFence(source, index) : undefined; if (fence?.kind === 'pending') { text += source.slice(index); + index = source.length; + if (isFinalSegment) markSafe(); break; } if (fence?.kind === 'match') { @@ -134,6 +212,35 @@ function translateMarkdownMath( continue; } + // Hide escaped label brackets from both the math delimiter scan and the + // Markdown bracket matcher, then restore them through the literal plugin. + if ( + protectEscapedBrackets + && source[index] === '\\' + && (source[index + 1] === '[' || source[index + 1] === ']') + ) { + text += transportToken(source[index + 1] ?? '', '2'); + index += 2; + atLineStart = false; + markSafe(); + continue; + } + + const literalToken = readLiteralToken(source, index); + if (literalToken?.kind === 'pending') { + text += source.slice(index); + index = source.length; + if (isFinalSegment) markSafe(); + break; + } + if (literalToken?.kind === 'match') { + text += transportToken(literalToken.source, '2'); + index = literalToken.end; + atLineStart = false; + markSafe(); + continue; + } + if (source[index] === '`') { let runEnd = index + 1; while (source[runEnd] === '`') runEnd++; @@ -143,7 +250,11 @@ function translateMarkdownMath( text += run; index = runEnd; atLineStart = false; - canMarkSafe = false; + if (isFinalSegment) { + markSafe(); + } else { + canMarkSafe = false; + } continue; } const end = close + run.length; @@ -154,6 +265,83 @@ function translateMarkdownMath( continue; } + const link = readMarkdownLink(source, index, leadingBackslashes); + if (link?.kind === 'pending') { + // Remember the scan only when everything before its opener already + // settled: an earlier unresolved backtick or math opener must be + // reparsed together with later chunks, and resuming just the label + // would skip it forever. + if (link.labelState !== undefined && safeSourceEnd === index) { + onLabelPending?.(link.labelState); + } + text += source.slice(index, link.end); + index = link.end; + atLineStart = false; + if (isFinalSegment) { + markSafe(); + } else { + canMarkSafe = false; + } + continue; + } + if (link?.kind === 'match') { + // A bare closed label with nothing after it can still grow an inline + // or reference tail, so it must not settle: resuming after it would + // scan that tail without the label context. Once any byte follows the + // label the link question is decided and settling is safe again. + const mayGrowTail = link.end === link.labelEnd + 1 && link.end >= source.length; + // Display math renders as a block, which cannot live inside an inline + // link label, and Markdown reads `\[` / `\]` there as literal escaped + // brackets. Re-run every closed label with display math disabled so + // escaped brackets survive for Markdown to unescape; inline math still + // renders inside. Astryx decides later whether the label is an inline + // link, reference use, shortcut, image, or definition, so all forms + // must share this representation. Image alt text included: + // Astryx keeps alt as a raw string, and the image component restores + // literal tokens, so no private-use characters reach the DOM. + const translatedLabel = translateMarkdownMath(source.slice(link.labelStart, link.labelEnd), { + startsAtLineStart: false, + priorTextChar: source[link.labelStart - 1], + allowDisplayMath: false, + protectEscapedBrackets: true, + isFinalSegment: true, + }); + // The explicit identifier of a full reference must go through the same + // transform, or use-site and definition IDs diverge and the link breaks. + let translatedRefText = ''; + let refSafe = true; + if (link.refLabelStart !== undefined && link.refLabelEnd !== undefined) { + const translatedRef = translateMarkdownMath( + source.slice(link.refLabelStart, link.refLabelEnd), + { + startsAtLineStart: false, + priorTextChar: source[link.refLabelStart - 1], + allowDisplayMath: false, + protectEscapedBrackets: true, + isFinalSegment: true, + }, + ); + translatedRefText = translatedRef.text; + refSafe = translatedRef.safeSourceEnd >= link.refLabelEnd - link.refLabelStart; + } + const refStart = link.refLabelStart ?? link.end; + const refEnd = link.refLabelEnd ?? link.end; + text += source.slice(index, link.labelStart) + + translatedLabel.text + + source.slice(link.labelEnd, refStart) + + translatedRefText + + source.slice(refEnd, link.end); + index = link.end; + atLineStart = source[index - 1] === '\n'; + const labelSafe = translatedLabel.safeSourceEnd >= link.labelEnd - link.labelStart; + if (labelSafe && refSafe && !mayGrowTail) { + markSafe(); + } else { + canMarkSafe = false; + } + continue; + } + // Destinations take `$` verbatim: escaping there would reach the href, // where `\$` does not round-trip back to `$`. const destination = readLinkDestination(source, index); @@ -193,13 +381,17 @@ function translateMarkdownMath( const delimited = readDelimitedMath(source, index, '\\(', '\\)', false) - ?? readDelimitedMath(source, index, '\\[', '\\]', true) - ?? readDelimitedMath(source, index, '$$', '$$', true); + ?? (allowDisplayMath ? readDelimitedMath(source, index, '\\[', '\\]', true) : undefined) + ?? (allowDisplayMath ? readDelimitedMath(source, index, '$$', '$$', true) : undefined); if (delimited?.kind === 'pending') { text += source.slice(index, delimited.end); index = delimited.end; atLineStart = false; - canMarkSafe = false; + if (isFinalSegment) { + markSafe(); + } else { + canMarkSafe = false; + } continue; } if (delimited?.kind === 'match') { @@ -219,7 +411,7 @@ function translateMarkdownMath( // the span stays uncommitted: the next chunk rescans from the opener // and decides the closing guard with the neighbor in hand. const next = source[delimited.end]; - if (next === undefined) markSafe(); + if (isFinalSegment || next === undefined) markSafe(); text += inlineMathSource( delimited.formula, text.length > 0 ? text[text.length - 1] : priorTextChar, @@ -227,12 +419,12 @@ function translateMarkdownMath( ); index = delimited.end; atLineStart = source[index - 1] === '\n'; - if (next !== undefined) markSafe(); + if (isFinalSegment || next !== undefined) markSafe(); continue; } const character = source[index] ?? ''; - if (character === '$' && !isEscaped(source, index)) { + if (character === '$' && !isEscaped(source, index, leadingBackslashes)) { text += '\\$'; } else { text += character; @@ -240,8 +432,9 @@ function translateMarkdownMath( index++; atLineStart = character === '\n'; if ( - index < source.length || - (character !== '\\' && character !== '$' && character !== '`') + isFinalSegment + || index < source.length + || (character !== '\\' && character !== '$' && character !== '`') ) { markSafe(); } @@ -422,6 +615,34 @@ function readFence( return { kind: 'match', end: source.length, closed: false }; } +function readLiteralToken( + source: string, + index: number, +): + | { kind: 'match'; source: string; end: number } + | { kind: 'pending' } + | undefined { + if (source[index] !== TOKEN_START[0]) return undefined; + if (!source.startsWith(TOKEN_START, index)) { + const tail = source.slice(index); + return tail.length < TOKEN_START.length && TOKEN_START.startsWith(tail) + ? { kind: 'pending' } + : undefined; + } + const tokenEnd = source.indexOf(TOKEN_END, index + TOKEN_START.length); + if (tokenEnd < 0) { + const payload = source.slice(index + TOKEN_START.length); + return /^(?:[012](?::[0-9a-f]*)?)?$/.test(payload) + ? { kind: 'pending' } + : undefined; + } + const candidate = source.slice(index, tokenEnd + TOKEN_END.length); + const match = LITERAL_TOKEN_PATTERN.exec(candidate); + if (!match) return undefined; + const token = match[0]; + return { kind: 'match', source: token, end: index + token.length }; +} + function readDelimitedMath( source: string, index: number, @@ -457,6 +678,286 @@ function readDelimitedMath( return { kind: 'match', formula, display, end: close + closing.length }; } +const MAX_LINK_LABEL_DEPTH = 32; +// NOTE: label scans deliberately have no length cap (tails keep theirs). +// Finding a close is linear, settling is permanent, and per-chunk rescan cost +// while a label is still open matches the base behaviour for unclosed math. +// Capping labels instead degrades complete long labels to literal text, +// which both breaks the link and pushes a giant literal run downstream that +// renders far slower than the structured link would have. +const MAX_LINK_TAIL_DEPTH = 32; +const MAX_LINK_TAIL_LENGTH = 65536; + +type MarkdownLinkScan = + | { + kind: 'match'; + labelStart: number; + labelEnd: number; + end: number; + refLabelStart?: number; + refLabelEnd?: number; + } + | { kind: 'pending'; end: number; labelState?: LabelScanState } + | undefined; + +// Resume state for an unfinished first-label scan, keyed by the owning cache. +// Only the top-level scan stores here: nested scans re-derive on the next +// full pass, and any closure funnels through a full reprocess anyway. +const pendingLabelScans = new WeakMap(); + +/** + * Recognize a bounded Markdown label starting at `index` so its contents can + * be re-scanned without display math. Astryx resolves whether a closed label + * is an inline link, reference use, shortcut, or definition after this pass; + * treating all of them alike keeps reference identities stable. + */ +function readMarkdownLink( + source: string, + index: number, + leadingBackslashes: number, +): MarkdownLinkScan { + let openerEnd: number; + if (source[index] === '!') { + // A trailing `!` may yet become an image opener once `[` arrives; caching + // it as safe would lose the `!` context and mistype the label as a link. + if (index + 1 >= source.length) return { kind: 'pending', end: index + 1 }; + if (source[index + 1] !== '[') return undefined; + if (isEscaped(source, index, leadingBackslashes)) return undefined; + openerEnd = index + 2; + } else if (source[index] === '[') { + if (isEscaped(source, index, leadingBackslashes)) return undefined; + openerEnd = index + 1; + } else { + return undefined; + } + + const firstScan = scanLabel(source, initialLabelScanState(openerEnd)); + if (firstScan.kind === 'pending') { + return { kind: 'pending', end: source.length, labelState: firstScan.state }; + } + if (firstScan.kind === 'invalid') return { kind: 'pending', end: firstScan.end }; + const labelEnd = firstScan.end; + + const match = (end: number) => ({ + kind: 'match' as const, + labelStart: openerEnd, + labelEnd, + end, + }); + + const tail = source[labelEnd + 1] ?? ''; + if (tail === '(') { + const tailEnd = findInlineTailEnd(source, labelEnd + 1); + if (tailEnd === 'pending') return { kind: 'pending', end: source.length }; + if (typeof tailEnd !== 'number') return match(tailEnd.end); + return match(tailEnd); + } + if (tail === '[') { + // The identifier scan resumes the same way the first label does: its + // pending state is forwarded so streamed updates continue it instead of + // restarting at the reference opener on every update. An invalid + // identifier is definitive, so it keeps the old match-and-settle path. + const refScan = scanLabel(source, initialLabelScanState(labelEnd + 2)); + if (refScan.kind === 'pending') { + return { kind: 'pending', end: source.length, labelState: refScan.state }; + } + if (refScan.kind === 'invalid') return match(refScan.end); + return { + ...match(refScan.end + 1), + refLabelStart: labelEnd + 2, + refLabelEnd: refScan.end, + }; + } + return match(labelEnd + 1); +} + +function countPrecedingBackslashes(source: string, pos: number): number { + let count = 0; + let i = pos - 1; + while (i >= 0 && source[i] === '\\') { + count++; + i--; + } + return count; +} + +/** Whether the character at `pos` is backslash-escaped (odd run before it). */ +function isEscaped( + source: string, + pos: number, + leadingBackslashes = 0, +): boolean { + let count = countPrecedingBackslashes(source, pos); + if (pos - count === 0) count += leadingBackslashes; + return count % 2 === 1; +} + +/** + * Resumable scan for the `]` closing a link label, skipping escapes, code + * spans, and nested labels. Absolute positions stay valid across streaming + * appends, so an unfinished scan can continue over new bytes instead of + * re-walking from the opener on every update. + */ +type LabelScanState = { + index: number; + depth: number; + escapeNext: boolean; + checkBlank: boolean; + runStart: number | null; + codeDelimLen: number; + codeSearchFrom: number; +}; + +function initialLabelScanState(from: number): LabelScanState { + return { + index: from, + depth: 0, + escapeNext: false, + checkBlank: false, + runStart: null, + codeDelimLen: 0, + codeSearchFrom: 0, + }; +} + +type LabelScanResult = + | { kind: 'pending'; state: LabelScanState } + | { kind: 'closed'; end: number } + | { kind: 'invalid'; end: number }; + +/** + * Find the `]` closing a link label opened before `from`, skipping escapes, + * code spans, and nested labels. Blank lines and excessive nesting can never + * form a label here; running out of input means more text may still complete + * it. There is deliberately no length bound: a close found anywhere resolves + * and settles, so incomplete input is the only case that rescans per chunk. + * Both the first label and reference identifiers scan through here, so a + * pending second label resumes the same way the first one does. + */ +function scanLabel(source: string, st: LabelScanState): LabelScanResult { + const pending = (): LabelScanResult => ({ kind: 'pending', state: st }); + while (st.index < source.length) { + if (st.escapeNext) { + // A trailing backslash left this pending; the pair only exists once the + // escaped character has arrived. Skip both together, exactly as a fresh + // scan would. + if (st.index + 1 >= source.length) return pending(); + st.escapeNext = false; + st.index += 2; + continue; + } + if (st.checkBlank) { + st.checkBlank = false; + // The newline at st.index was already seen; only a second newline + // makes it a blank line. Otherwise consume it as an ordinary char and + // process the new character normally below. + if (st.index + 1 >= source.length) { + st.checkBlank = true; + return pending(); + } + if (source[st.index + 1] === '\n') return { kind: 'invalid', end: st.index }; + st.index++; + } + if (st.runStart !== null || source[st.index] === '`') { + if (st.runStart === null) st.runStart = st.index; + while (source[st.index] === '`') st.index++; + if (st.index >= source.length) return pending(); + st.codeDelimLen = st.index - st.runStart; + st.runStart = null; + st.codeSearchFrom = st.index; + } + if (st.codeDelimLen > 0) { + const close = source.indexOf('`'.repeat(st.codeDelimLen), st.codeSearchFrom); + if (close < 0) { + st.codeSearchFrom = Math.max(st.codeSearchFrom, source.length - (st.codeDelimLen - 1)); + return pending(); + } + st.index = close + st.codeDelimLen; + st.codeDelimLen = 0; + st.codeSearchFrom = 0; + continue; + } + const ch = source[st.index] ?? ''; + if (ch === '\n') { + if (st.index + 1 >= source.length) { + st.checkBlank = true; + return pending(); + } + if (source[st.index + 1] === '\n') return { kind: 'invalid', end: st.index }; + st.index++; + continue; + } + if (ch === '\\') { + if (st.index + 1 >= source.length) { + st.escapeNext = true; + return pending(); + } + st.index += 2; + continue; + } + if (ch === '[') { + st.depth++; + if (st.depth > MAX_LINK_LABEL_DEPTH) { + return { kind: 'invalid', end: findInvalidLinkBoundary(source, st.index) }; + } + st.index++; + continue; + } + if (ch === ']') { + if (st.depth === 0) return { kind: 'closed', end: st.index }; + st.depth--; + st.index++; + continue; + } + st.index++; + } + return pending(); +} + +/** + * Find the end of an inline `(destination)` tail starting at its `(`. A + * pending tail consumes the remaining source in one pass; resuming at the + * opener would rescan the same suffix for every `[label](` in a stream. + */ +function findInlineTailEnd( + source: string, + from: number, +): number | 'pending' | { kind: 'invalid'; end: number } { + let depth = 1; + let i = from + 1; + while (i < source.length) { + if (i - from >= MAX_LINK_TAIL_LENGTH) { + return { kind: 'invalid', end: findInvalidLinkBoundary(source, i) }; + } + const ch = source[i] ?? ''; + if (ch === '\n' && source[i + 1] === '\n') return { kind: 'invalid', end: i }; + if (ch === '\\') { + if (i + 1 >= source.length) return 'pending'; + i += 2; + continue; + } + if (ch === '(') { + depth++; + if (depth > MAX_LINK_TAIL_DEPTH) { + return { kind: 'invalid', end: findInvalidLinkBoundary(source, i) }; + } + i++; + continue; + } + if (ch === ')') { + depth--; + if (depth === 0) return i + 1; + } + i++; + } + return 'pending'; +} + +function findInvalidLinkBoundary(source: string, from: number): number { + const blankLine = source.indexOf('\n\n', from); + return blankLine >= 0 ? blankLine : source.length; +} + function findPendingFenceBoundary(source: string, from: number): number { const fenceMatch = /(?:^|\n) {0,3}(?:`{3,}|~{3,})/g; fenceMatch.lastIndex = from; @@ -464,12 +965,37 @@ function findPendingFenceBoundary(source: string, from: number): number { return fence ? fence.index + (source[fence.index] === '\n' ? 1 : 0) : -1; } -function isEscaped(source: string, index: number): boolean { - let backslashes = 0; - let i = index - 1; - while (i >= 0 && source[i] === '\\') { - backslashes++; - i--; +function transportToken(value: string, kind: '0' | '1' | '2'): string { + return `${TOKEN_START}${kind}:${encodeFormula(value)}${TOKEN_END}`; +} + +const TRANSPORT_TOKEN_RESTORE_PATTERN = /\uE000MAKA_MATH:[012]:([0-9a-f]+)\uE001/g; + +/** + * Restore transport tokens in image alt text to plain text. Astryx keeps alt + * as a raw string, so tokens that survive preprocessing would otherwise leak + * private-use characters into the DOM. Literal tokens decode to their + * characters; math tokens decode to their formula text, since KaTeX cannot + * render inside an attribute. + */ +export function restoreTransportTokens(text: string): string { + return text.replace(TRANSPORT_TOKEN_RESTORE_PATTERN, (_, encoded: string) => + decodeFormula(encoded), + ); +} + +function encodeFormula(formula: string): string { + let encoded = ''; + for (const byte of new TextEncoder().encode(formula)) { + encoded += byte.toString(16).padStart(2, '0'); + } + return encoded; +} + +function decodeFormula(encoded: string): string { + const bytes = new Uint8Array(encoded.length / 2); + for (let index = 0; index < bytes.length; index++) { + bytes[index] = Number.parseInt(encoded.slice(index * 2, index * 2 + 2), 16); } - return backslashes % 2 === 1; + return new TextDecoder().decode(bytes); }