diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 846ac00..abd8003 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,9 @@ jobs: node-version: 22 cache: npm - run: npm ci + # LibreOffice, so the test that turns Word and PowerPoint files into pages runs (it's skipped where it's missing). + - if: runner.os == 'Linux' + run: sudo apt-get update -qq && sudo apt-get install -y -qq --no-install-recommends libreoffice-writer-nogui libreoffice-impress-nogui - run: npm run typecheck - run: npm test diff --git a/CHANGELOG.md b/CHANGELOG.md index efeef07..56e10a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Search results are grouped by document. A PDF that matches on several pages is one result with its best passage, links to the other pages ("Also on p. 2, p. 7") and its other passages a click away; the heading counts documents and passages. Quick search shows one row per document. +- Open shows the document inside Duct, on its own, with your notebooks beside it; **Open in app** hands it to the program it belongs to. Closing one of two documents in the workspace leaves the other full width, and **One / Side by side / Stacked** switches between them. +- Selected text can go to any notebook: the pop-up says **Add to ""**, and its menu lists every notebook you can add to, or starts a new one named after the document. +- Find in every document: the words you searched for are highlighted when it opens, "2 of 5" with previous and next (Shift+Enter, Enter) steps through every match on every page, and the find box takes other words too. +- Documents look like themselves: Word files with their headings, lists, tables and pictures; spreadsheets as tables with a tab per sheet; Markdown formatted; emails with their headers and body. Where LibreOffice is installed, Word, OpenDocument, RTF, PowerPoint and Keynote files are shown as their real pages instead (converted on this computer, once per version). **Text** still shows what Duct read. HTML from documents is cleaned to a short list of safe tags, and an email's remote images are never loaded. `GET /api/document-view`, `GET /api/document-pdf`. +- Search inside audio: voice notes (including WhatsApp's), recorded calls, meetings and interviews. What's said is transcribed on this computer by Whisper, never uploaded; the 77 MB model downloads once, the first time, and the privacy ledger lists it under On-device models. Results say "at 2:00" and play from there; the workspace shows the recording with its transcript, a time on every line. Off until turned on in Settings › Features; songs (tagged with an artist and an album) are skipped. Not available on Intel Macs. +- **Open at login** (Settings › This computer) starts Duct when you sign in, so quick search and watched folders are always ready. + +### Fixed +- Long file names in All documents ran over the folder, status and buttons; they now end in "…", with the full name and path on hover. +- Highlights lit up a searched word inside another word ("tax" in "taxation"). + + ## [1.0.0-alpha.2] - 2026-10-09 ### Fixed diff --git a/CLAUDE.md b/CLAUDE.md index 58a71d7..79b302f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,13 +20,14 @@ npx electron-builder --mac dir -c.mac.identity=null # unsigned packaged app in r When launching Electron from a VS Code terminal, unset `ELECTRON_RUN_AS_NODE` first (`env -u ELECTRON_RUN_AS_NODE npm run electron:start`). Otherwise Electron runs as plain Node and `require('electron')` fails. -CI (`.github/workflows/ci.yml`) runs typecheck and tests on Linux, macOS and Windows, plus a build. Publishing to npm and GHCR happens on a GitHub release (`publish.yml`). Desktop installers aren't built in CI yet: signing and notarization need Apple and Windows certificates. +CI (`.github/workflows/ci.yml`) runs typecheck and tests on Linux, macOS and Windows, plus a build. Publishing to npm and GHCR happens on a GitHub release (`publish.yml`). Desktop installers are built on each release by `installers.yml` (unsigned until there are Apple and Windows certificates). `scripts/check-runtime-deps.mjs` fails a build whose code imports a package that isn't in `dependencies`. ## Layout ``` src/ index.ts Duct class: the whole pipeline, change detection, config, watched folders, diff, schema extraction + document-view.ts How the workspace shows non-PDF files: LibreOffice pages if installed, else cleaned HTML per format (cleanHtml allowlist), else text library.ts addToLibrary(): copy a file into the Library (~/Duct Library), dedupe by hash, index it version.ts VERSION read from package.json types.ts All public interfaces (DuctConfig, RuntimeConfig, Chunk, Searcher, VectorStore, LLMProvider, …) diff --git a/assets/ui/app.css b/assets/ui/app.css index 3002a5e..f5f83f7 100644 --- a/assets/ui/app.css +++ b/assets/ui/app.css @@ -148,6 +148,7 @@ body.no-mascot .mascot-slot { display: none; } .type.slide { background: #2d1d0b; color: #ffb74d; } .type.mail { background: #22163a; color: #b388ff; } .type.img { background: #0e2830; color: #64d8e8; } +.type.audio { background: #2b1d10; color: #fbbf74; } .type.zip { background: #262613; color: #e6e675; } /* ---------- results ---------- */ @@ -163,7 +164,17 @@ body.no-mascot .mascot-slot { display: none; } .result.selected { background: var(--s1); border-color: var(--border2); } .result-main { flex: 1; min-width: 0; } .result-title { display: flex; align-items: baseline; gap: 8px; flex-wrap: wrap; } -.result-title .name { font-weight: 600; color: var(--text); overflow: hidden; text-overflow: ellipsis; } +.result-title .name { font-weight: 600; color: var(--text); overflow: hidden; overflow-wrap: anywhere; display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; } +.also-on { margin-top: 6px; font-size: 12.5px; color: var(--subtle); } +.also-on .link-btn { font-family: var(--mono); font-size: 11.5px; color: var(--lime); } +.more-btn { font: inherit; font-size: 12px; color: var(--subtle); background: none; border: 0; padding: 0; margin-top: 8px; cursor: pointer; text-decoration: underline dotted; text-underline-offset: 2px; } +.more-btn:hover, .more-btn:focus-visible { color: var(--text); } +.more-btn[aria-expanded="true"]::after { content: ' ▴'; } +.more-btn[aria-expanded="false"]::after { content: ' ▾'; } +.more-passages { list-style: none; margin: 8px 0 0; padding: 0 0 0 12px; border-left: 2px solid var(--border2); display: flex; flex-direction: column; gap: 10px; } +.more-passages li { cursor: pointer; } +.more-passages .snippet { margin-top: 2px; } +.more-passages .btn { margin-top: 4px; } .where { font-family: var(--mono); font-size: 11px; color: var(--lime); } .section { font-size: 12px; color: var(--subtle); } .result .folder { font-family: var(--mono); font-size: 11px; color: var(--muted); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; margin-top: 1px; } diff --git a/assets/ui/app.js b/assets/ui/app.js index 088e1ce..46d51d0 100644 --- a/assets/ui/app.js +++ b/assets/ui/app.js @@ -110,6 +110,8 @@ function markSnippet(snippet) { return esc(snippet).split(MARK_START).join('').split(MARK_END).join('') } +const WORD_CHAR = /[\p{L}\p{N}]/u + /** Highlights words in raw text and returns escaped HTML. */ function highlight(text, terms) { const words = terms.map(t => t.toLowerCase()).filter(t => t.length > 1) @@ -118,6 +120,12 @@ function highlight(text, terms) { const marked = new Array(text.length).fill(false) for (const word of words) { for (let i = lower.indexOf(word); i !== -1; i = lower.indexOf(word, i + 1)) { + // Whole words, or a word with a short ending ("terminates" for "terminate"), never inside another word: + // "tax" shouldn't light up in "taxation". + if (i > 0 && WORD_CHAR.test(lower[i - 1])) continue + let end = i + word.length + while (end < lower.length && WORD_CHAR.test(lower[end])) end++ + if (end - (i + word.length) > 3) continue for (let j = i; j < i + word.length; j++) marked[j] = true } } @@ -167,7 +175,7 @@ const BADGES = { pdf: ['PDF', 'pdf'], docx: ['DOC', 'doc'], doc: ['DOC', 'doc'], odt: ['ODT', 'doc'], rtf: ['RTF', 'doc'], pages: ['PAGES', 'doc'], md: ['MD', 'doc'], html: ['HTML', 'doc'], epub: ['EPUB', 'doc'], xlsx: ['XLS', 'sheet'], ods: ['ODS', 'sheet'], numbers: ['NUM', 'sheet'], pptx: ['PPT', 'slide'], odp: ['ODP', 'slide'], key: ['KEY', 'slide'], eml: ['EML', 'mail'], msg: ['MSG', 'mail'], txt: ['TXT', ''], - code: ['CODE', ''], svg: ['SVG', 'img'], image: ['IMG', 'img'], zip: ['ZIP', 'zip'], url: ['WEB', 'doc'], + code: ['CODE', ''], svg: ['SVG', 'img'], image: ['IMG', 'img'], zip: ['ZIP', 'zip'], url: ['WEB', 'doc'], audio: ['AUDIO', 'audio'], } const badge = format => { const [label, cls] = BADGES[format] || ['FILE', '']; return '' + label + '' } @@ -176,6 +184,7 @@ const GROUPS = [ { id: 'docs', label: 'Documents', test: f => (f.kind === 'document' && f.format !== 'pdf') || f.kind === 'ebook' }, { id: 'sheets', label: 'Spreadsheets', test: f => f.kind === 'spreadsheet' }, { id: 'slides', label: 'Presentations', test: f => f.kind === 'presentation' }, + { id: 'audio', label: 'Audio', test: f => f.kind === 'audio' }, { id: 'email', label: 'Email', test: f => f.kind === 'email' }, { id: 'images', label: 'Images', test: f => f.kind === 'image' }, { id: 'text', label: 'Text & code', test: f => f.kind === 'text' || f.kind === 'code' }, @@ -216,7 +225,7 @@ const FEATURE_GROUPS = [ ['developerApi', 'Developer API', 'The /v1 API for apps: collections, API keys, and indexing your own text by id.'], ] }, ] -const KIND_LABELS = { document: 'Documents (PDF, Word, Pages, Markdown, HTML…)', spreadsheet: 'Spreadsheets', presentation: 'Presentations', ebook: 'E-books', email: 'Email', text: 'Text, CSV, JSON and subtitles', code: 'Source code', image: 'Images and SVG', archive: 'ZIP archives' } +const KIND_LABELS = { document: 'Documents (PDF, Word, Pages, Markdown, HTML…)', spreadsheet: 'Spreadsheets', presentation: 'Presentations', ebook: 'E-books', email: 'Email', text: 'Text, CSV, JSON and subtitles', code: 'Source code', image: 'Images and SVG', audio: 'Audio and voice notes: what’s said, transcribed on this computer (downloads a 77 MB speech model the first time; songs are skipped)', archive: 'ZIP archives' } const DESKTOP_PREFS = [ ['openAtLogin', 'Open at login', 'Start Duct when you sign in to this computer, so quick search and watched folders are always ready.'], ['island', 'Notch companion', 'Duct at the top of the screen: progress, quick search and a drop zone.'], @@ -369,6 +378,8 @@ async function saveTags(path, tags) { } function pageRef(chunk) { + // Audio is transcribed a page per minute: page 3 starts at 2:00. + if (chunk.documentFormat === 'audio') return 'at ' + (chunk.page - 1) + ':00' const label = state.info.formats.find(f => f.format === chunk.documentFormat)?.pageLabel || 'p.' return label + ' ' + chunk.page } @@ -685,26 +696,65 @@ function terms(result) { return [...new Set([...phrases, ...marked, ...words])].slice(0, 12) } +/** + * Results are grouped by document: a PDF that matches on five pages is one card, with its best passage, links to the + * other pages and the other passages a click away. state.results stays the flat, ranked list of passages; state.cards + * holds, for each card in order, the indexes of its passages (best first), and state.selected is a card index. + */ +function groupResults(results) { + const byDoc = new Map() + results.forEach((r, i) => { + const key = r.chunk.documentPath + if (!byDoc.has(key)) byDoc.set(key, []) + byDoc.get(key).push(i) + }) + return [...byDoc.values()] +} +const cardResult = i => state.cards && state.cards[i] ? state.results[state.cards[i][0]] : undefined + +function resultLabel(c, link) { + if (c.documentFormat === 'audio') return c.page ? 'Play from ' + (c.page - 1) + ':00' : 'Play' + return link ? 'Open link' : c.page ? 'Open at ' + pageRef(c) : 'Open' +} + function renderResults() { const results = state.results - $('#resultsTitle').textContent = results.length ? plural(results.length, 'result') + ' for “' + state.query + '”' : 'No results for “' + state.query + '”' - $('#results').innerHTML = results.map((r, i) => { + state.cards = groupResults(results) + const docs = state.cards.length + $('#resultsTitle').textContent = !results.length ? 'No results for “' + state.query + '”' + : plural(docs, 'document') + (results.length > docs ? ' · ' + plural(results.length, 'passage') : '') + ' for “' + state.query + '”' + $('#results').innerHTML = state.cards.map((idx, card) => { + const r = results[idx[0]] const c = r.chunk const link = isLink(c.documentPath) const folder = link ? new URL(c.documentPath).host : folderOf(c.documentPath).startsWith(state.info.libraryDir) ? 'Duct Library' : folderOf(c.documentPath) - return '
  • ' + badge(c.documentFormat) + + const others = idx.slice(1) + // Other pages, in page order, each opening the document there. + const pages = [...new Map(others.filter(j => results[j].chunk.page && results[j].chunk.page !== c.page).map(j => [results[j].chunk.page, j])).entries()].sort((a, b) => a[0] - b[0]) + const alsoOn = pages.length ? '
    Also on ' + pages.slice(0, 12).map(([, j]) => + '').join(', ') + + (pages.length > 12 ? ' and ' + (pages.length - 12) + ' more' : '') + '
    ' : '' + const more = others.length ? '' + + '' : '' + return '
  • ' + badge(c.documentFormat) + '
    ' + esc(fileName(c.documentPath)) + '' + (c.page ? '' + esc(pageRef(c)) + '' : '') + (c.heading ? '› ' + esc(c.heading) + '' : '') + '
    ' + (c.metadata && c.metadata.whatsappChat ? whatsappLine(c.metadata) : '
    ' + esc(folder) + '
    ') + (r.why ? '' : '') + '
    ' + (r.snippet ? markSnippet(r.snippet) : highlight(c.content.slice(0, 260), terms(r))) + '
    ' + - '
    ' + + alsoOn + + '
    ' + (link ? '' : '') + '' + (feature('export') ? '' : '') + (desktop && desktop.revealDocument && !link ? '' : '') + - (r.why ? '' : '') + '
    • ' + (r.why ? '' : '') + '' + more + '' }).join('') const empty = $('#resultsEmpty') empty.hidden = results.length > 0 @@ -719,7 +769,7 @@ function select(i, scroll = true) { if (i < 0) { closePreview(); return } const el = $$('.result')[i] if (scroll && el) el.scrollIntoView({ block: 'nearest' }) - if (window.innerWidth > 1180) renderPreview(state.results[i]) + if (window.innerWidth > 1180) renderPreview(cardResult(i)) } $('#results').addEventListener('click', e => { @@ -728,6 +778,16 @@ $('#results').addEventListener('click', e => { const r = state.results[Number(item.dataset.i)] const act = e.target.closest('[data-act]') if (act && act.dataset.act === 'open') return openResult(r) + if (act && act.dataset.act === 'open-at') return openResult(state.results[Number(act.dataset.ri)]) + if (act && act.dataset.act === 'more') { + const list = item.querySelector('.more-passages') + list.hidden = !list.hidden + act.setAttribute('aria-expanded', String(!list.hidden)) + return + } + // A passage from the expanded list: show it in the preview. + const other = e.target.closest('.more-passages li') + if (other) { renderPreview(state.results[Number(other.dataset.ri)]); return } if (act && act.dataset.act === 'workspace') return openWorkspace(r.chunk.documentPath, '', r.chunk.page, terms(r)) if (act && act.dataset.act === 'reveal') return desktop.revealDocument(r.chunk.documentPath) if (act && act.dataset.act === 'copy-passage') return copyText('“' + r.chunk.content.trim() + '”\n— ' + sourceLine(r.chunk), 'Passage copied with its source') @@ -738,7 +798,7 @@ $('#results').addEventListener('click', e => { act.setAttribute('aria-expanded', String(!box.hidden)) return } - select(Number(item.dataset.i), false) + select(Number(item.dataset.card), false) if (window.innerWidth <= 1180) renderPreview(r) }) $('#results').addEventListener('dblclick', e => { @@ -758,7 +818,7 @@ function renderPreview(r) { '
    ' + esc(label) + (c.page ? ' · ' + esc(pageRef(c)) : '') + (c.heading ? ' · ' + esc(c.heading) : '') + '
    ' + esc(link ? c.documentPath : folderOf(c.documentPath)) + (c.metadata && (c.metadata.author || c.metadata.year) ? '
    ' + esc([c.metadata.author, c.metadata.year].filter(Boolean).join(', ')) : '') + '
    ' + renderTagEditor(c.documentPath) + - '
    ' + + '
    ' + (desktop && desktop.revealDocument && !link ? '' : '') + (c.metadata && typeof c.metadata.webUrl === 'string' && /^https:\/\//.test(c.metadata.webUrl) ? 'Open in ' + (c.metadata.connector === 'gdrive' ? 'Google Drive' : 'Microsoft 365') + '' : '') + (link ? '' : '') + @@ -974,13 +1034,13 @@ $('#searchForm').addEventListener('submit', e => { if (!value) return if (state.mode === 'ask') { ask(value); $('#q').value = ''; return } state.query = value - if (state.selected >= 0 && state.results.length) openResult(state.results[state.selected]) + if (state.selected >= 0 && state.results.length) openResult(cardResult(state.selected)) else runSearch() }) $('#q').addEventListener('keydown', e => { if (state.mode !== 'search') return - if (e.key === 'ArrowDown') { e.preventDefault(); select(Math.min(state.results.length - 1, state.selected + 1)) } + if (e.key === 'ArrowDown') { e.preventDefault(); select(Math.min((state.cards || []).length - 1, state.selected + 1)) } if (e.key === 'ArrowUp') { e.preventDefault(); select(Math.max(0, state.selected - 1)) } if (e.key === 'Escape') { $('#q').value = ''; state.query = ''; setView('home') } }) @@ -1626,6 +1686,7 @@ const LEDGER_WHAT = { ai: 'The question and the passages needed to answer it, or text to index for search by meaning: only because you chose this provider.', cloud: 'Requests to read the sources you connected. Files come in; nothing of yours goes out.', signin: 'Signing in to a service you connected.', + models: 'Downloading the speech model for audio search, once. Your recordings are transcribed on this computer and never sent.', web: 'Fetching web pages you added to Duct.', } let ledgerDays = 1 diff --git a/docs/formats.md b/docs/formats.md index e62e39c..f8d19c5 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -12,6 +12,7 @@ Duct reads these formats. The list lives in one place, `src/formats.ts`. The ind | Spreadsheets | .xlsx .xlsm .xls .xlsb, .ods | One page per sheet ("sheet 2") | | Presentations | .pptx .pptm .ppsx .potx, .odp, .key | One page per slide, with speaker notes | | E-books | .epub | One page per chapter ("ch. 3"). DRM-protected books are reported as unreadable | +| Audio | .mp3 .m4a .wav .ogg .oga .opus .flac .aac .amr .wma .aiff .aif .caf .3gp | Off until turned on (Settings › Features › Audio and voice notes). What's said is transcribed on this computer with Whisper (base, about 77 MB, downloaded once the first time; the privacy ledger lists it under On-device models). The transcript is a page per minute, each line with its time; results say "at 2:00" and open the recording there. Files tagged with an artist and an album are taken as songs and skipped; recordings over 3 hours aren't transcribed. Not available on Intel Macs | | Email | .eml, .msg (Outlook) | Subject, sender, recipients, date and body are searchable. Supported attachments are indexed under "Attachment: name" | | Web and Markdown | .html .htm .xhtml, .md .markdown .mdx | | | Text | .txt .text .csv .tsv .json .jsonl .ndjson .log .xml .yaml .yml .toml .ini .cfg .conf .tex .bib .rst .adoc .asciidoc .org .nfo .srt .vtt | UTF-8, UTF-16 and Windows-1252 are detected. Subtitles keep only the spoken lines | diff --git a/package-lock.json b/package-lock.json index bbf1938..9a6ee33 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0-alpha.2", "license": "(Apache-2.0 AND Elastic-2.0)", "dependencies": { + "@huggingface/transformers": "^4.3.1", "@kenjiuno/msgreader": "^1.28.0", "@lottiefiles/dotlottie-web": "^0.81.0", "canvas": "^3.2.3", @@ -17,6 +18,7 @@ "commander": "^12.0.0", "express": "^4.18.0", "express-rate-limit": "^8.7.1", + "ffmpeg-static": "^5.3.0", "jszip": "^3.10.1", "mammoth": "^1.6.0", "marked": "^12.0.0", @@ -57,6 +59,50 @@ "voyageai": "^0.3.0" } }, + "node_modules/@derhuerst/http-basic": { + "version": "8.2.4", + "resolved": "https://registry.npmjs.org/@derhuerst/http-basic/-/http-basic-8.2.4.tgz", + "integrity": "sha512-F9rL9k9Xjf5blCz8HsJRO4diy111cayL2vkY2XE4r4t3n0yPXVYy3KD3nJ1qbrSn9743UWSXH4IwuCa/HWlGFw==", + "license": "MIT", + "dependencies": { + "caseless": "^0.12.0", + "concat-stream": "^2.0.0", + "http-response-object": "^3.0.1", + "parse-cache-control": "^1.0.1" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@derhuerst/http-basic/node_modules/concat-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", + "engines": [ + "node >= 6.0" + ], + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.0.2", + "typedarray": "^0.0.6" + } + }, + "node_modules/@derhuerst/http-basic/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/@develar/schema-utils": { "version": "2.6.5", "resolved": "https://registry.npmjs.org/@develar/schema-utils/-/schema-utils-2.6.5.tgz", @@ -1049,6 +1095,34 @@ "node": ">=18.0.0" } }, + "node_modules/@huggingface/jinja": { + "version": "0.5.10", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.10.tgz", + "integrity": "sha512-SgS1D1bglQ94ceD4ZCL6eayUDy9uV1xuyk61OjgSxU5GJh7upqZCKII0JoTYMc6wWsg3JUgaPRX0ElQzXPk3Cw==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@huggingface/tokenizers": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@huggingface/tokenizers/-/tokenizers-0.2.0.tgz", + "integrity": "sha512-LidMHe1FpcSYH4vcSjXooda34pC0M8m1gmDOv7SQi6Iv+ib1zX7J3sHBCC9/3FPCkn2k1b55FneVqMRxrr99pg==", + "license": "Apache-2.0" + }, + "node_modules/@huggingface/transformers": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-4.3.1.tgz", + "integrity": "sha512-5+voSeUCFK6PCKlY15dPhNOKq3/zOTxDN7oC+Wy9dFb1QZ0TSQomZRRy7URRK8YRuUoE9fGMQTTPC5d/pWBJhA==", + "license": "Apache-2.0", + "dependencies": { + "@huggingface/jinja": "^0.5.10", + "@huggingface/tokenizers": "^0.2.0", + "onnxruntime-node": "1.30.0", + "onnxruntime-web": "1.31.0-dev.20260914-8d85527a0", + "sharp": "^0.35.4" + } + }, "node_modules/@img/colour": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", @@ -2042,6 +2116,63 @@ "url": "https://github.com/sponsors/Boshen" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, "node_modules/@rolldown/binding-android-arm64": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.2.tgz", @@ -2803,6 +2934,15 @@ "node": ">= 0.6" } }, + "node_modules/adm-zip": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.6.1.tgz", + "integrity": "sha512-Xwrja8nx9e5o2N1my4DsKCeKpdrnACyr1wtbPxBDgGzKzKyE9kRtBFA8mWldI+RVlD7CBZNWY/wQ2+ydwOR6kQ==", + "license": "MIT", + "engines": { + "node": ">=14.0" + } + }, "node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -3518,6 +3658,12 @@ "node": "^18.12.0 || >= 20.9.0" } }, + "node_modules/caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==", + "license": "Apache-2.0" + }, "node_modules/chai": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", @@ -4150,9 +4296,7 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "dev": true, "license": "MIT", - "optional": true, "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", @@ -4169,9 +4313,7 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", - "dev": true, "license": "MIT", - "optional": true, "dependencies": { "define-data-property": "^1.0.1", "has-property-descriptors": "^1.0.0", @@ -4868,9 +5010,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true, "license": "MIT", - "optional": true, "engines": { "node": ">=10" }, @@ -5128,6 +5268,79 @@ } } }, + "node_modules/ffmpeg-static": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/ffmpeg-static/-/ffmpeg-static-5.3.0.tgz", + "integrity": "sha512-H+K6sW6TiIX6VGend0KQwthe+kaceeH/luE8dIZyOP35ik7ahYojDuqlTV1bOrtEwl01sy2HFNGQfi5IDJvotg==", + "hasInstallScript": true, + "license": "GPL-3.0-or-later", + "dependencies": { + "@derhuerst/http-basic": "^8.2.0", + "env-paths": "^2.2.0", + "https-proxy-agent": "^5.0.0", + "progress": "^2.0.3" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/ffmpeg-static/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/ffmpeg-static/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/ffmpeg-static/node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/ffmpeg-static/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/ffmpeg-static/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, "node_modules/filelist": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.6.tgz", @@ -5186,6 +5399,12 @@ "node": ">= 0.8" } }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, "node_modules/form-data": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", @@ -5448,9 +5667,7 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", - "dev": true, "license": "MIT", - "optional": true, "dependencies": { "define-properties": "^1.2.1", "gopd": "^1.0.1" @@ -5507,6 +5724,12 @@ "dev": true, "license": "ISC" }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC" + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -5521,9 +5744,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "dev": true, "license": "MIT", - "optional": true, "dependencies": { "es-define-property": "^1.0.0" }, @@ -5693,6 +5914,21 @@ "dev": true, "license": "MIT" }, + "node_modules/http-response-object": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/http-response-object/-/http-response-object-3.0.2.tgz", + "integrity": "sha512-bqX0XTF6fnXSQcEJ2Iuyr75yVakyjIDCqroJQ/aHfSdlM743Cwqoi2nDYMzLGWUcuTWGWy8AAvOKXTfiv6q9RA==", + "license": "MIT", + "dependencies": { + "@types/node": "^10.0.3" + } + }, + "node_modules/http-response-object/node_modules/@types/node": { + "version": "10.17.60", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", + "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==", + "license": "MIT" + }, "node_modules/http2-wrapper": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-1.0.3.tgz", @@ -6393,6 +6629,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/lop": { "version": "0.4.2", "resolved": "https://registry.npmjs.org/lop/-/lop-0.4.2.tgz", @@ -6908,9 +7150,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true, "license": "MIT", - "optional": true, "engines": { "node": ">= 0.4" } @@ -6962,6 +7202,106 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/onnxruntime-common": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.30.0.tgz", + "integrity": "sha512-7fdVWjAID1dVhH/G8qK3APARunV4VkBFoCQAP7qp4Wkab0mrorvmc+sqiT+mKXOzDqdjN5j+/Z9nb4gzNPWcyA==", + "license": "MIT" + }, + "node_modules/onnxruntime-node": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.30.0.tgz", + "integrity": "sha512-twhs1C2C/BFkz1yc5OY0KIU2GUq6DURO7hD4bx5Q2Qy3nAMJwRXW8xU3NVczE29VA9lolLOYepoD8fjTGOfIqw==", + "hasInstallScript": true, + "license": "MIT", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "adm-zip": "^0.6.0", + "global-agent": "^4.1.3", + "onnxruntime-common": "1.30.0" + } + }, + "node_modules/onnxruntime-node/node_modules/global-agent": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-4.1.3.tgz", + "integrity": "sha512-KUJEViiuFT3I97t+GYMikLPJS2Lfo/S2F+DQuBWzuzaMPnvt5yyZePzArx36fBzpGTxZjIpDbXLeySLgh+k76g==", + "license": "BSD-3-Clause", + "dependencies": { + "globalthis": "^1.0.2", + "matcher": "^4.0.0", + "semver": "^7.3.5", + "serialize-error": "^8.1.0" + }, + "engines": { + "node": ">=10.0" + } + }, + "node_modules/onnxruntime-node/node_modules/matcher": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-4.0.0.tgz", + "integrity": "sha512-S6x5wmcDmsDRRU/c2dkccDwQPXoFczc5+HpQ2lON8pnvHlnvHAHj5WlLVvw6n6vNyHuVugYrFohYxbS+pvFpKQ==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/onnxruntime-node/node_modules/serialize-error": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz", + "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/onnxruntime-node/node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.31.0-dev.20260914-8d85527a0", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.31.0-dev.20260914-8d85527a0.tgz", + "integrity": "sha512-Iy7rtadoBgxS/LLvDr3QW38DB1PNXRnr0GJMcL0TAt7c9qjgVQl83UlGCVyeAnK2InpmW8Uc3PL8XIuqtDeF6g==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^25.1.24", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.31.0-dev.20260911-2a43ec07e", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, + "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { + "version": "1.31.0-dev.20260911-2a43ec07e", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.31.0-dev.20260911-2a43ec07e.tgz", + "integrity": "sha512-gBuF6U32YErKIAt+yD7DeGBpjRxhJ1uJwako6ygjokSZULDU6Pd+KWComX8Tumk1hYVHRxjPj7mdMnD0ryAPOw==", + "license": "MIT" + }, "node_modules/openai": { "version": "4.104.0", "resolved": "https://registry.npmjs.org/openai/-/openai-4.104.0.tgz", @@ -7076,6 +7416,11 @@ "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", "license": "(MIT AND Zlib)" }, + "node_modules/parse-cache-control": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parse-cache-control/-/parse-cache-control-1.0.1.tgz", + "integrity": "sha512-60zvsJReQPX5/QP0Kzfd/VrpjScIQ7SHBW6bFCYfEP+fp0Eppr1SHhIO5nd1PjZtvclzSzES9D/p5nFJurwfWg==" + }, "node_modules/parse5": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", @@ -7220,6 +7565,12 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" + }, "node_modules/plist": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/plist/-/plist-3.1.0.tgz", @@ -7370,7 +7721,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.4.0" @@ -7409,6 +7759,29 @@ "dev": true, "license": "ISC" }, + "node_modules/protobufjs": { + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/proxy-addr": { "version": "2.0.8", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", diff --git a/package.json b/package.json index 0f2f45c..fac56df 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,8 @@ "package.json", "assets/mascot/**/*", "assets/ocr/**/*", - "assets/ui/**/*" + "assets/ui/**/*", + "!node_modules/onnxruntime-web/**" ], "mac": { "category": "public.app-category.productivity", @@ -60,14 +61,23 @@ "dmg", "zip" ], - "icon": "electron/icon.icns" + "icon": "electron/icon.icns", + "files": [ + "!node_modules/onnxruntime-node/bin/napi-v*/linux/**", + "!node_modules/onnxruntime-node/bin/napi-v*/win32/**" + ] }, "win": { "target": [ "nsis", "zip" ], - "icon": "electron/icon.png" + "icon": "electron/icon.png", + "files": [ + "!node_modules/onnxruntime-node/bin/napi-v*/linux/**", + "!node_modules/onnxruntime-node/bin/napi-v*/darwin/**", + "!node_modules/onnxruntime-node/bin/napi-v*/win32/arm64/**" + ] }, "linux": { "target": [ @@ -76,7 +86,12 @@ ], "category": "Office", "icon": "electron/icon.png", - "executableName": "duct" + "executableName": "duct", + "files": [ + "!node_modules/onnxruntime-node/bin/napi-v*/darwin/**", + "!node_modules/onnxruntime-node/bin/napi-v*/win32/**", + "!node_modules/onnxruntime-node/bin/napi-v*/linux/arm64/**" + ] }, "deb": { "packageName": "duct", @@ -86,10 +101,13 @@ "main": "electron/main.cjs" }, "asarUnpack": [ - "assets/ocr/**" + "assets/ocr/**", + "node_modules/ffmpeg-static/**", + "node_modules/onnxruntime-node/**" ] }, "dependencies": { + "@huggingface/transformers": "^4.3.1", "@kenjiuno/msgreader": "^1.28.0", "@lottiefiles/dotlottie-web": "^0.81.0", "canvas": "^3.2.3", @@ -98,6 +116,7 @@ "commander": "^12.0.0", "express": "^4.18.0", "express-rate-limit": "^8.7.1", + "ffmpeg-static": "^5.3.0", "jszip": "^3.10.1", "mammoth": "^1.6.0", "marked": "^12.0.0", @@ -162,5 +181,10 @@ "publishConfig": { "access": "public", "provenance": true + }, + "overrides": { + "voyageai": { + "@huggingface/transformers": "$@huggingface/transformers" + } } } diff --git a/src/document-view.ts b/src/document-view.ts new file mode 100644 index 0000000..91885b0 --- /dev/null +++ b/src/document-view.ts @@ -0,0 +1,225 @@ +// How the workspace shows a document that isn't a PDF, as close to the original as this computer allows: +// +// 1. Documents and presentations (Word, OpenDocument, RTF, PowerPoint, Keynote) are turned into a PDF by LibreOffice, +// when it's installed, and shown with the PDF viewer: the pages as they look, with find and highlights. +// 2. Otherwise each format is drawn from its own structure: Word with its headings, lists, tables and pictures; +// spreadsheets as tables, a sheet at a time; Markdown formatted; emails with their headers and body. +// 3. Anything else is shown as the text Duct read (the workspace's text view). +// +// Everything happens on this computer. HTML that comes from a document is cleaned to an allowlist of tags and +// attributes before it reaches the page: no scripts, styles, event handlers, links that navigate, or images other +// than ones embedded in the file (an email's remote images are never fetched; they'd tell the sender it was opened). + +import { execFile } from 'node:child_process' +import { createHash } from 'node:crypto' +import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync } from 'node:fs' +import { tmpdir, userInfo } from 'node:os' +import { basename, extname, join } from 'node:path' + +export type DocumentView = + | { kind: 'pdf'; via: 'libreoffice' } + | { kind: 'html'; via: 'word' | 'sheet' | 'markdown' | 'email' | 'html'; html: string; sections?: { title: string; page: number }[] } + | { kind: 'text' } + +/** Formats LibreOffice turns into faithful pages. Spreadsheets aren't here: as PDFs they're cut into pages. */ +export const LIBREOFFICE_FORMATS = new Set(['.doc', '.docx', '.docm', '.dot', '.dotx', '.odt', '.ott', '.rtf', '.wpd', '.ppt', '.pptx', '.pps', '.ppsx', '.odp', '.key', '.pages']) +const SHEET_FORMATS = new Set(['.xlsx', '.xlsm', '.xls', '.xlsb', '.ods', '.csv', '.tsv']) +const MAX_ROWS = 2000 +const MAX_COLS = 60 + +// ---------------------------------------------------------------- LibreOffice + +let sofficePath: string | null | undefined + +/** Where LibreOffice is installed, if it is (looked up once). DUCT_SOFFICE overrides; DUCT_SOFFICE=off disables. */ +export function findLibreOffice(): string | null { + if (sofficePath !== undefined) return sofficePath + const configured = process.env['DUCT_SOFFICE'] + if (configured === 'off') return (sofficePath = null) + const candidates = [ + configured, + '/Applications/LibreOffice.app/Contents/MacOS/soffice', + 'C:\\Program Files\\LibreOffice\\program\\soffice.exe', + 'C:\\Program Files (x86)\\LibreOffice\\program\\soffice.exe', + '/usr/bin/soffice', '/usr/bin/libreoffice', '/usr/local/bin/soffice', '/opt/homebrew/bin/soffice', '/snap/bin/libreoffice', + '/usr/lib/libreoffice/program/soffice', '/opt/libreoffice/program/soffice', + ] + return (sofficePath = candidates.find((p): p is string => !!p && existsSync(p)) ?? null) +} + +/** For tests. */ +export function resetLibreOfficeLookup(): void { sofficePath = undefined } + +const converting = new Map>() + +/** + * The document as a PDF made by LibreOffice, cached by the file's path, size and modification time so it's converted + * once. Each conversion uses its own LibreOffice profile, so it works while LibreOffice is open, and runs headless + * with a time limit. + */ +export function convertToPdf(path: string, cacheDir: string, timeoutMs = 120_000): Promise { + const soffice = findLibreOffice() + if (!soffice) return Promise.reject(new Error('LibreOffice isn’t installed')) + const st = statSync(path) + const key = createHash('sha256').update(`${path}\0${st.size}\0${st.mtimeMs}`).digest('hex').slice(0, 32) + const out = join(cacheDir, `${key}.pdf`) + if (existsSync(out)) return Promise.resolve(out) + const running = converting.get(key) + if (running) return running + const job = new Promise((resolve, reject) => { + mkdirSync(cacheDir, { recursive: true, mode: 0o700 }) + const work = join(cacheDir, `${key}.work`) + rmSync(work, { recursive: true, force: true }) + mkdirSync(work, { recursive: true, mode: 0o700 }) + const profile = 'file://' + join(work, 'profile').replace(/\\/g, '/').replace(/^([A-Za-z]):/, '/$1:') + execFile(soffice, [`-env:UserInstallation=${profile}`, '--headless', '--norestore', '--nologo', '--convert-to', 'pdf', '--outdir', work, path], + { timeout: timeoutMs, windowsHide: true }, (err, stdout, stderr) => { + const made = join(work, basename(path, extname(path)) + '.pdf') + if (!err && existsSync(made)) { renameSync(made, out); rmSync(work, { recursive: true, force: true }); resolve(out); return } + rmSync(work, { recursive: true, force: true }) + const said = `${stderr || ''} ${stdout || ''}`.replace(/\s+/g, ' ').trim().slice(0, 300) + reject(new Error(err?.killed ? 'LibreOffice took too long to open this document' : `LibreOffice couldn’t open this document${said ? ` (${said})` : ''}`)) + }) + }).finally(() => converting.delete(key)) + converting.set(key, job) + return job +} + +/** A private cache folder for converted documents: in Duct's data folder, or a per-user temporary one. */ +export function viewCacheDir(dataDir: string | undefined): string { + return dataDir ? join(dataDir, 'view-cache') : join(tmpdir(), `duct-view-cache-${userInfo().uid >= 0 ? userInfo().uid : 'user'}`) +} + +// ---------------------------------------------------------------- HTML, cleaned + +const ALLOWED: Record = { + p: [], br: [], hr: [], div: [], span: [], blockquote: [], pre: [], code: [], + h1: [], h2: [], h3: [], h4: [], h5: [], h6: [], + strong: [], b: [], em: [], i: [], u: [], s: [], sub: [], sup: [], small: [], mark: [], + ul: [], ol: ['start'], li: [], dl: [], dt: [], dd: [], + table: [], thead: [], tbody: [], tfoot: [], tr: [], th: ['colspan', 'rowspan', 'scope'], td: ['colspan', 'rowspan'], caption: [], + img: ['src', 'alt', 'width', 'height'], figure: [], figcaption: [], a: [], section: [], article: [], header: [], footer: [], +} +/** Removed with everything inside them; any other unknown tag is replaced by its contents. */ +const DROP = new Set(['script', 'style', 'noscript', 'template', 'iframe', 'frame', 'frameset', 'object', 'embed', 'applet', 'form', 'input', 'button', 'select', 'textarea', 'option', 'link', 'meta', 'base', 'title', 'head', 'svg', 'math', 'video', 'audio', 'source', 'track', 'canvas', 'dialog', 'portal']) +/** Duct's own markup: classes the workspace styles, and data-page on sheets (any element may carry these). */ +const OWN_CLASSES = new Set(['sheet', 'sheet-name', 'note', 'mail-head', 'mail-body', 'img-missing', 'link']) +const SAFE_IMAGE = /^data:image\/(png|jpe?g|gif|webp|bmp);base64,[a-z0-9+/=\s]+$/i + +/** Keeps only allowlisted tags and attributes. Images only if embedded; links become plain text in a link style. */ +export async function cleanHtml(html: string): Promise { + const cheerio = await import('cheerio') + const $ = cheerio.load(html, null, false) + $('*').each((_, node) => { + const el = node as unknown as { tagName?: string; attribs?: Record } + if (!el.tagName) return + const tag = el.tagName.toLowerCase() + const $el = $(node) + if (DROP.has(tag)) { $el.remove(); return } + const allowed = ALLOWED[tag] + if (!allowed) { $el.replaceWith($el.contents()); return } + for (const name of Object.keys(el.attribs ?? {})) { + const value = el.attribs![name] ?? '' + if (name === 'class') { const own = value.split(/\s+/).filter(c => OWN_CLASSES.has(c)); if (own.length) { $el.attr('class', own.join(' ')); continue } } + if (name === 'data-page' && /^\d{1,5}$/.test(value)) continue + const keep = allowed.includes(name) && ( + name === 'src' ? SAFE_IMAGE.test(value) + : ['colspan', 'rowspan', 'start', 'width', 'height'].includes(name) ? /^\d{1,4}$/.test(value) + : name === 'scope' ? /^(row|col|rowgroup|colgroup)$/.test(value) : true) + if (!keep) $el.removeAttr(name) + } + if (tag === 'img' && !$el.attr('src')) { $el.replaceWith($el.attr('alt') ? `[${escapeHtml($el.attr('alt')!)}]` : ''); return } + if (tag === 'a') $el.attr('class', 'link') + }) + // Comments can hide conditional markup; nothing in them is needed. + const dropComments = (nodes: { type: string; children?: unknown[] }[]) => { + for (const n of [...nodes]) { + if (n.type === 'comment') $(n as never).remove() + else if (n.children) dropComments(n.children as never) + } + } + dropComments($.root()[0]!.children as never) + return $.html() +} + +export function escapeHtml(s: string): string { + return s.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"') +} + +// ---------------------------------------------------------------- each format, drawn from its structure + +async function wordHtml(path: string): Promise { + const mammoth = await import('mammoth') + const result = await mammoth.convertToHtml({ path }, { + convertImage: mammoth.images.imgElement(async image => ({ src: `data:${image.contentType};base64,${await image.read('base64')}` })), + }) + return result.value +} + +async function sheetHtml(path: string): Promise<{ html: string; sections: { title: string; page: number }[] }> { + const XLSX = await import('xlsx') + const workbook = XLSX.read(readFileSync(path), { type: 'buffer', cellDates: true }) + const sections: { title: string; page: number }[] = [] + const parts: string[] = [] + workbook.SheetNames.forEach((name, i) => { + const rows = XLSX.utils.sheet_to_json(workbook.Sheets[name]!, { header: 1, raw: false, blankrows: false }) + const shown = rows.slice(0, MAX_ROWS) + const cols = Math.min(MAX_COLS, Math.max(0, ...shown.map(r => r.length))) + const cell = (v: unknown) => escapeHtml(v == null ? '' : String(v)) + const body = shown.map((r, ri) => '' + Array.from({ length: cols }, (_, c) => (ri === 0 ? `${cell(r[c])}` : `${cell(r[c])}`)).join('') + '').join('') + const more = rows.length > MAX_ROWS ? `

    Showing the first ${MAX_ROWS.toLocaleString('en')} of ${rows.length.toLocaleString('en')} rows. Open it in its app to see the rest.

    ` : '' + sections.push({ title: name, page: i + 1 }) + parts.push(`

    ${escapeHtml(name)}

    ${shown.length ? `
    ${body}
    ` : '

    This sheet is empty.

    '}${more}
    `) + }) + return { html: parts.join(''), sections } +} + +async function markdownHtml(path: string): Promise { + const { marked } = await import('marked') + return marked.parse(readFileSync(path, 'utf8'), { async: false, gfm: true }) as string +} + +async function emailHtml(path: string, ext: string): Promise { + const head = (rows: [string, string | undefined][]) => '
    ' + rows.filter(([, v]) => v).map(([k, v]) => `
    ${k}
    ${escapeHtml(v!)}
    `).join('') + '
    ' + const textBody = (t: string) => t.split(/\n{2,}/).map(p => `

    ${escapeHtml(p).replace(/\n/g, '
    ')}

    `).join('') + if (ext === '.eml') { + const { default: PostalMime } = await import('postal-mime') + const mail = await PostalMime.parse(readFileSync(path)) + const who = (a?: { name?: string; address?: string } | null) => !a ? undefined : a.name && a.address ? `${a.name} <${a.address}>` : a.name || a.address + const list = (as?: { name?: string; address?: string }[]) => as?.map(who).filter(Boolean).join(', ') + const files = (mail.attachments ?? []).map(a => a.filename).filter(Boolean).join(', ') + return head([['Subject', mail.subject], ['From', who(mail.from as never)], ['To', list(mail.to as never)], ['Cc', list(mail.cc as never)], ['Date', readableDate(mail.date)], ['Attachments', files]]) + + '
    ' + (mail.html ? mail.html : textBody(mail.text ?? '')) + '
    ' + } + const mod = await import('@kenjiuno/msgreader') as unknown as { default: { default?: unknown } } + const MsgReader = ('default' in mod.default ? mod.default.default : mod.default) as new (b: ArrayBuffer) => { getFileData(): Record } + const buffer = readFileSync(path) + const data = new MsgReader(buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength) as ArrayBuffer).getFileData() as { + subject?: string; senderName?: string; senderEmail?: string; body?: string; bodyHtml?: string; messageDeliveryTime?: string + recipients?: { name?: string; email?: string }[]; attachments?: { fileName?: string }[] + } + const to = (data.recipients ?? []).map(r => [r.name, r.email ? `<${r.email}>` : ''].filter(Boolean).join(' ')).join(', ') + return head([['Subject', data.subject], ['From', [data.senderName, data.senderEmail ? `<${data.senderEmail}>` : ''].filter(Boolean).join(' ')], ['To', to], + ['Date', readableDate(data.messageDeliveryTime)], ['Attachments', (data.attachments ?? []).map(a => a.fileName).filter(Boolean).join(', ')]]) + + '
    ' + (data.bodyHtml ? data.bodyHtml : textBody(data.body ?? '')) + '
    ' +} + +/** "2 March 2026, 08:00 UTC" from an email's date, or the date as written if it can't be read. */ +function readableDate(value: string | undefined): string | undefined { + if (!value) return undefined + const d = new Date(value) + if (Number.isNaN(d.getTime())) return value + return d.toLocaleString('en-GB', { day: 'numeric', month: 'long', year: 'numeric', hour: '2-digit', minute: '2-digit', timeZone: 'UTC' }) + ' UTC' +} + +/** How to show this file (path must be an indexed document; callers check). */ +export async function documentView(path: string, options: { libreOffice?: boolean } = {}): Promise { + const ext = extname(path).toLowerCase() + if (LIBREOFFICE_FORMATS.has(ext) && options.libreOffice !== false && findLibreOffice()) return { kind: 'pdf', via: 'libreoffice' } + if (ext === '.docx' || ext === '.docm') return { kind: 'html', via: 'word', html: await cleanHtml(await wordHtml(path)) } + if (SHEET_FORMATS.has(ext)) { const s = await sheetHtml(path); return { kind: 'html', via: 'sheet', html: await cleanHtml(s.html), sections: s.sections } } + if (ext === '.md' || ext === '.markdown') return { kind: 'html', via: 'markdown', html: await cleanHtml(await markdownHtml(path)) } + if (ext === '.eml' || ext === '.msg') return { kind: 'html', via: 'email', html: await cleanHtml(await emailHtml(path, ext)) } + if (ext === '.html' || ext === '.htm') return { kind: 'html', via: 'html', html: await cleanHtml(readFileSync(path, 'utf8')) } + return { kind: 'text' } +} diff --git a/src/extract/audio.ts b/src/extract/audio.ts new file mode 100644 index 0000000..02daa22 --- /dev/null +++ b/src/extract/audio.ts @@ -0,0 +1,15 @@ +// Audio: what's said in a recording, transcribed on this computer (src/speech.ts), a page per minute. +import { statSync } from 'node:fs' +import { MAX_AUDIO_SECONDS, probeAudio, transcribe, transcriptPages } from '../speech.js' +import type { ExtractedDocument } from '../types.js' +import { UnsupportedFileError } from './common.js' + +export async function extractAudio(path: string, modelsDir: string): Promise { + const info = await probeAudio(path).catch(() => { throw new UnsupportedFileError('No audio found in this file') }) + // Music libraries aren't what audio search is for: songs carry an artist and an album. + if (info.music) throw new UnsupportedFileError('A song, not a recording of speech') + const metadata: Record = { size: statSync(path).size, durationSec: Math.round(info.durationSec), ...(info.title ? { title: info.title } : {}) } + if (info.durationSec > MAX_AUDIO_SECONDS) throw new Error(`It’s longer than ${MAX_AUDIO_SECONDS / 3600} hours, so Duct doesn’t transcribe it`) + const pages = transcriptPages(await transcribe(path, modelsDir)) + return { path, format: 'audio', content: pages.join('\n'), pages, metadata } +} diff --git a/src/extract/common.ts b/src/extract/common.ts index ba13739..bb1d6d8 100644 --- a/src/extract/common.ts +++ b/src/extract/common.ts @@ -13,6 +13,8 @@ export class UnsupportedFileError extends Error { export interface ExtractOptions { ocr?: boolean + /** Where the speech model is kept, for audio (see src/speech.ts). */ + modelsDir?: string /** How deep we are inside containers (emails, ZIPs); nested containers stop at MAX_DEPTH. */ depth?: number } diff --git a/src/extract/index.ts b/src/extract/index.ts index 5c4ed08..46c4739 100644 --- a/src/extract/index.ts +++ b/src/extract/index.ts @@ -1,6 +1,7 @@ import { chatSections, chatTitle, isChatFileName, parseWhatsAppChat } from '../whatsapp-chat.js' import { readFileSync, statSync } from 'node:fs' -import { basename, extname } from 'node:path' +import { homedir } from 'node:os' +import { basename, extname, join } from 'node:path' import type { ExtractedDocument, DocumentFormat } from '../types.js' import { formatForPath } from '../formats.js' import { ocrPdf } from '../ocr/index.js' @@ -10,6 +11,13 @@ import { UnsupportedFileError, cleanDetails, decodeText, decodeXml, isZip, readP import { extractDoc, extractOdp, extractOdt, extractRtf, isOle } from './office.js' import { extractEpub, extractIwork, extractZip } from './packages.js' import { extractEml, extractMsg } from './email.js' +import { extractAudio } from './audio.js' + + +/** Where the speech model goes when the caller doesn't say: beside the CLI's index. */ +function defaultModelsDir(): string { + return join(process.env['DUCT_HOME'] || join(homedir(), '.duct'), 'models') +} export { UnsupportedFileError } from './common.js' export type { ExtractOptions } from './common.js' @@ -197,6 +205,7 @@ export async function extract(path: string, options: ExtractOptions = {}): Promi case 'code': return extractCode(path) case 'svg': return extractSvg(path) case 'zip': return extractZip(path, options, extract) + case 'audio': return extractAudio(path, options.modelsDir ?? defaultModelsDir()) case 'image': if (options.ocr) return extractImage(path) return { path, format: 'image', content: '', metadata: { size: statSync(path).size, needsOcr: true } } diff --git a/src/features.ts b/src/features.ts index 1059ef8..1140448 100644 --- a/src/features.ts +++ b/src/features.ts @@ -40,7 +40,7 @@ export interface Features { export type FeatureName = Exclude -export const FORMAT_KINDS: FormatKind[] = ['document', 'spreadsheet', 'presentation', 'ebook', 'email', 'text', 'code', 'image', 'archive'] +export const FORMAT_KINDS: FormatKind[] = ['document', 'spreadsheet', 'presentation', 'ebook', 'email', 'text', 'code', 'image', 'audio', 'archive'] export const FEATURE_NAMES: FeatureName[] = [ 'ask', 'semanticSearch', 'schemaExtraction', 'fileNameSearch', 'webPages', 'uploads', 'watchedFolders', 'ocrOnDemand', 'export', 'diff', 'developerApi', 'deadlines', 'publicLinks', @@ -66,7 +66,8 @@ export const FEATURE_LABELS: Record = { export function defaultFeatures(): Features { return { ...Object.fromEntries(FEATURE_NAMES.map(n => [n, true])) as Record, - formats: Object.fromEntries(FORMAT_KINDS.map(k => [k, true])) as Record, + // Audio is off until someone turns it on: transcribing downloads a speech model, and music folders aren't speech. + formats: Object.fromEntries(FORMAT_KINDS.map(k => [k, k !== 'audio'])) as Record, } } diff --git a/src/formats.ts b/src/formats.ts index ffc5513..bd56e0e 100644 --- a/src/formats.ts +++ b/src/formats.ts @@ -4,7 +4,7 @@ import { basename, extname } from 'node:path' import type { DocumentFormat } from './types.js' -export type FormatKind = 'document' | 'spreadsheet' | 'presentation' | 'ebook' | 'email' | 'text' | 'code' | 'image' | 'archive' +export type FormatKind = 'document' | 'spreadsheet' | 'presentation' | 'ebook' | 'email' | 'text' | 'code' | 'image' | 'audio' | 'archive' export interface FileFormat { format: DocumentFormat @@ -52,6 +52,8 @@ export const FORMATS: FileFormat[] = [ }, { format: 'svg', extensions: ['.svg'], label: 'SVG', kind: 'image' }, { format: 'image', extensions: ['.png', '.jpg', '.jpeg', '.tiff', '.tif', '.bmp', '.gif', '.webp', '.heic', '.heif', '.avif'], label: 'Image (OCR)', kind: 'image' }, + // Transcribed on this computer (src/speech.ts); off until turned on in Settings, since it downloads a speech model. + { format: 'audio', extensions: ['.mp3', '.m4a', '.wav', '.ogg', '.oga', '.opus', '.flac', '.aac', '.amr', '.wma', '.aiff', '.aif', '.caf', '.3gp'], label: 'Audio', kind: 'audio', pageLabel: 'minute' }, { format: 'zip', extensions: ['.zip'], label: 'ZIP archive', kind: 'archive' }, ] diff --git a/src/index.ts b/src/index.ts index 275ebb0..d36d302 100644 --- a/src/index.ts +++ b/src/index.ts @@ -803,7 +803,7 @@ export class Duct { if (isUrl(filePath)) { return await extractUrl(filePath, { blockPrivate: this.blockPrivateUrls }) } - return await extract(filePath, { ocr: ocr ?? this.ocr }) + return await extract(filePath, { ocr: ocr ?? this.ocr, ...(this.persistPath ? { modelsDir: join(this.persistPath, 'models') } : {}) }) } private findDocument(path: string) { diff --git a/src/island.ts b/src/island.ts index f7328b7..64822b0 100644 --- a/src/island.ts +++ b/src/island.ts @@ -498,7 +498,7 @@ kbd { font-family: var(--mono); font-size: 10px; background: var(--s2); border: }, } const docName = r => r.name || fileName(r.chunk.documentPath) - const pageText = (format, page) => (PAGE_LABELS[format] || 'p.') + ' ' + page + const pageText = (format, page) => format === 'audio' ? 'at ' + (page - 1) + ':00' : (PAGE_LABELS[format] || 'p.') + ' ' + page /** With nothing typed: recent searches and documents, so the shortcut also takes you back to what you were doing. */ function showRecent() { @@ -521,6 +521,14 @@ kbd { font-family: var(--mono); font-size: 10px; background: var(--s2); border: function showResults(list, help) { if (!lastQuery) { showRecent(); return } + // One row per document, at its best passage; other pages that match are counted on the row. + const byDoc = new Map() + for (const r of list) { + const k = r.chunk.documentPath + if (!byDoc.has(k)) byDoc.set(k, { ...r, otherPages: new Set() }) + else if (r.chunk.page && r.chunk.page !== byDoc.get(k).chunk.page) byDoc.get(k).otherPages.add(r.chunk.page) + } + list = [...byDoc.values()] results = list selected = 0 document.getElementById('footKeys').hidden = !list.length @@ -534,7 +542,7 @@ kbd { font-family: var(--mono); font-size: 10px; background: var(--s2); border: } box.innerHTML = list.map((r, i) => '
    ' + - '
    ' + esc(docName(r)) + (r.chunk.page ? '' + esc(pageText(r.chunk.documentFormat, r.chunk.page)) + '' : '') + '
    ' + + '
    ' + esc(docName(r)) + (r.chunk.page ? '' + esc(pageText(r.chunk.documentFormat, r.chunk.page)) + (r.otherPages.size ? ' +' + r.otherPages.size + ' more' : '') + '' : '') + '
    ' + '
    ' + (r.snippet ? markSnippet(r.snippet) : esc(r.chunk.content.slice(0, 160))) + '
    ' + '
    ').join('') listHeight = list.length * 62 diff --git a/src/ledger.ts b/src/ledger.ts index 44a635f..90eb3bb 100644 --- a/src/ledger.ts +++ b/src/ledger.ts @@ -9,13 +9,14 @@ import { existsSync, readFileSync, writeFileSync } from 'node:fs' import { syncBuiltinESMExports } from 'node:module' import { join } from 'node:path' -export type LedgerCategory = 'tensflare' | 'ai' | 'cloud' | 'signin' | 'web' +export type LedgerCategory = 'tensflare' | 'ai' | 'cloud' | 'signin' | 'models' | 'web' export const LEDGER_CATEGORY_LABELS: Record = { tensflare: 'Tensflare (account, usage counts, feedback, hosted AI, notebook links)', ai: 'AI provider you chose', cloud: 'Cloud sources you connected', signin: 'Sign-in providers', + models: 'On-device models', web: 'Web pages you added', } @@ -53,6 +54,8 @@ export function categorize(host: string): LedgerCategory { if (AI_HOSTS.includes(h)) return 'ai' if (CLOUD_HOSTS.includes(h) || h.endsWith('.amazonaws.com') || h.endsWith('.r2.cloudflarestorage.com') || h.endsWith('.wasabisys.com') || h.endsWith('.backblazeb2.com') || h.endsWith('.sharepoint.com')) return 'cloud' if (SIGNIN_HOSTS.includes(h)) return 'signin' + // The speech model for audio search, downloaded once from Hugging Face. + if (h === 'huggingface.co' || h.endsWith('.huggingface.co') || h === 'hf.co' || h.endsWith('.hf.co')) return 'models' return 'web' } diff --git a/src/server.ts b/src/server.ts index 2cbffb0..098ec56 100644 --- a/src/server.ts +++ b/src/server.ts @@ -6,6 +6,7 @@ import { existsSync, mkdirSync, realpathSync, unlinkSync } from 'node:fs' import { randomBytes } from 'node:crypto' import { createRequire } from 'node:module' import { dirname, extname, isAbsolute, join, relative, resolve } from 'node:path' +import { LIBREOFFICE_FORMATS, convertToPdf, documentView, viewCacheDir } from './document-view.js' import { fileURLToPath } from 'node:url' import type { Duct, Note, Notebook } from './index.js' import { canDo, notebookRole } from './notebooks.js' @@ -69,6 +70,8 @@ const CSP = [ "frame-ancestors 'none'", ].join('; ') // The island and PDF viewer pages still carry their script inline. +/** Indexed web pages are stored by their address, not a file path. */ +const isRemote = (path: string) => /^https?:\/\//i.test(path) const INLINE_SCRIPT_CSP = CSP.replace("script-src 'self'", "script-src 'self' 'unsafe-inline'") const uiDir = fileURLToPath(new URL('../assets/ui', import.meta.url)) const API_KEY_FIELDS = ['openaiKey', 'geminiKey', 'cohereKey', 'voyageKey', 'mistralKey', 'jinaKey'] as const @@ -973,6 +976,32 @@ export function createServer(duct: Duct, opts?: ServerOptions) { }) // A document as text in sections, for the side-by-side workspace (formats the browser can't draw itself). + // The workspace's view of a document that isn't a PDF: pages from LibreOffice when it's installed, else the format's + // own structure as cleaned HTML, else text (see src/document-view.ts). + app.get('/api/document-view', async (req, res) => { + const path = typeof req.query.path === 'string' ? req.query.path : '' + const doc = path ? duct.getDocument(path) : undefined + if (!doc || doc.status === 'failed' || isRemote(doc.path)) { res.status(404).json({ error: 'Document not found' }); return } + try { + res.setHeader('Cache-Control', 'no-store') + res.json(await documentView(doc.path)) + } catch { + res.json({ kind: 'text' }) // a file the richer view can't draw still has its text + } + }) + app.get('/api/document-pdf', async (req, res) => { + const path = typeof req.query.path === 'string' ? req.query.path : '' + const doc = path ? duct.getDocument(path) : undefined + if (!doc || !LIBREOFFICE_FORMATS.has(extname(doc.path).toLowerCase()) || isRemote(doc.path)) { res.status(404).json({ error: 'Document not found' }); return } + try { + const pdf = await convertToPdf(doc.path, viewCacheDir(duct.dataDir)) + res.setHeader('Cache-Control', 'no-store') + res.type('application/pdf').sendFile(pdf) + } catch (err) { + sendError(res, err, 422) + } + }) + app.get('/api/document-text', async (req, res) => { const path = typeof req.query.path === 'string' ? req.query.path : '' const doc = path ? duct.getDocument(path) : undefined diff --git a/src/speech.ts b/src/speech.ts new file mode 100644 index 0000000..9909996 --- /dev/null +++ b/src/speech.ts @@ -0,0 +1,131 @@ +// Speech to text on this computer, for searching inside audio: voice notes, recorded calls and meetings, interviews. +// +// Audio is decoded with ffmpeg (the ffmpeg-static binary that ships with Duct) to 16 kHz mono and transcribed by +// Whisper running locally through transformers.js and ONNX Runtime. Nothing is uploaded. The model (Whisper base, +// about 77 MB) is downloaded once, the first time a recording is transcribed, into Duct's data folder; the privacy +// ledger shows that download. Audio search is a file family that's off until someone turns it on. + +import { execFile } from 'node:child_process' +import { existsSync } from 'node:fs' + +export interface SpeechSegment { + /** Seconds from the start. */ + start: number + end: number + text: string +} + +export interface AudioInfo { + durationSec: number + /** A song rather than speech: tagged with both an artist and an album. */ + music: boolean + title?: string +} + +export const SPEECH_MODEL = 'onnx-community/whisper-base' +/** Recordings longer than this aren't transcribed: hours of audio would hold the indexer for a long time. */ +export const MAX_AUDIO_SECONDS = 3 * 60 * 60 + +/** The ffmpeg binary: ffmpeg-static's, moved out of the app archive in the desktop app; DUCT_FFMPEG overrides. */ +export async function ffmpegPath(): Promise { + if (process.env['DUCT_FFMPEG']) return process.env['DUCT_FFMPEG'] + const mod = await import('ffmpeg-static') as unknown as { default: string | null } + const path = mod.default?.replace(`app.asar${pathSep()}`, `app.asar.unpacked${pathSep()}`) + if (!path || !existsSync(path)) throw new Error('The audio decoder (ffmpeg) is missing from this installation') + return path +} +const pathSep = () => (process.platform === 'win32' ? '\\' : '/') + +function run(file: string, args: string[]): Promise<{ stdout: Buffer; stderr: string; code: number }> { + return new Promise(resolve => { + execFile(file, args, { encoding: 'buffer', maxBuffer: 2 * 1024 * 1024 * 1024, windowsHide: true, timeout: 30 * 60_000 }, (err, stdout, stderr) => { + resolve({ stdout: stdout as Buffer, stderr: (stderr as Buffer).toString('utf8'), code: err ? ((err as { code?: number }).code ?? 1) : 0 }) + }) + }) +} + +/** Duration and tags, read from ffmpeg's description of the file. */ +export async function probeAudio(path: string): Promise { + const { stderr } = await run(await ffmpegPath(), ['-hide_banner', '-i', path]) + const d = /Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/.exec(stderr) + if (!d || !/Stream #\d+:\d+.*Audio:/.test(stderr)) throw new Error('No audio found in this file') + const tag = (name: string) => new RegExp(`^\\s+${name}\\s*:\\s*(.+)$`, 'mi').exec(stderr)?.[1]?.trim() + const artist = tag('artist') ?? tag('album_artist') + const album = tag('album') + return { durationSec: Number(d[1]) * 3600 + Number(d[2]) * 60 + Number(d[3]), music: !!(artist && album), title: tag('title') } +} + +/** The recording as 16 kHz mono samples, which is what Whisper takes. */ +export async function decodeAudio(path: string): Promise { + const { stdout, code, stderr } = await run(await ffmpegPath(), ['-v', 'error', '-i', path, '-vn', '-ac', '1', '-ar', '16000', '-f', 'f32le', 'pipe:1']) + if (code !== 0 || !stdout.length) throw new Error(`Couldn't read the audio: ${stderr.split('\n')[0] || 'unknown format'}`) + const copy = new Uint8Array(stdout.byteLength) + copy.set(stdout) + return new Float32Array(copy.buffer) +} + +export type Transcriber = (audio: Float32Array) => Promise + +let transcriber: Promise | null = null +let transcriberFor = '' +let override: Transcriber | null = null + +/** Tests replace the model with a function. */ +export function setTranscriber(fn: Transcriber | null): void { override = fn; transcriber = null } + +/** Whisper, loaded once per models folder (downloading it the first time). */ +function loadTranscriber(modelsDir: string): Promise { + if (override) return Promise.resolve(override) + if (transcriber && transcriberFor === modelsDir) return transcriber + transcriberFor = modelsDir + transcriber = (async () => { + if (process.platform === 'darwin' && process.arch === 'x64') throw new SpeechUnavailableError('Searching inside audio needs a Mac with Apple silicon, Windows or Linux.') + const { pipeline, env } = await import('@huggingface/transformers') + env.cacheDir = modelsDir + env.allowLocalModels = false + const asr = await pipeline('automatic-speech-recognition', SPEECH_MODEL, { dtype: 'q8' }) as unknown as ( + audio: Float32Array, options: Record) => Promise<{ text: string; chunks?: { timestamp: [number, number | null]; text: string }[] }> + return async (audio: Float32Array) => { + const out = await asr(audio, { return_timestamps: true, chunk_length_s: 30, stride_length_s: 5 }) + const chunks = out.chunks?.length ? out.chunks : [{ timestamp: [0, audio.length / 16000] as [number, number], text: out.text }] + return chunks.map(c => ({ start: c.timestamp[0] ?? 0, end: c.timestamp[1] ?? c.timestamp[0] ?? 0, text: c.text.trim() })).filter(c => c.text) + } + })() + transcriber.catch(() => { transcriber = null }) // try again next time (offline, say) + return transcriber +} + +export class SpeechUnavailableError extends Error { + constructor(message: string) { super(message); this.name = 'SpeechUnavailableError' } +} + +// One recording at a time: transcription uses every core it can get. +let queue: Promise = Promise.resolve() + +/** What's said in the recording, with timestamps. */ +export function transcribe(path: string, modelsDir: string): Promise { + const job = queue.then(async () => (await loadTranscriber(modelsDir))(await decodeAudio(path))) + queue = job.catch(() => {}) + return job +} + +/** "2:05", or "1:02:05" past an hour. */ +export function clock(seconds: number): string { + const s = Math.max(0, Math.floor(seconds)) + const h = Math.floor(s / 3600), m = Math.floor((s % 3600) / 60), sec = s % 60 + return (h ? `${h}:${String(m).padStart(2, '0')}` : `${m}`) + ':' + String(sec).padStart(2, '0') +} + +/** + * The transcript a page per minute, each line starting with its time ("[2:05] …"), so search results point to the + * minute and the viewer can play from the line. + */ +export function transcriptPages(segments: SpeechSegment[]): string[] { + const pages: string[][] = [] + for (const seg of segments) { + const minute = Math.floor(seg.start / 60) + while (pages.length <= minute) pages.push([]) + pages[minute]!.push(`[${clock(seg.start)}] ${seg.text}`) + } + return pages.map(lines => lines.join('\n')) +} diff --git a/src/types.ts b/src/types.ts index 0886725..4c5658b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -12,7 +12,7 @@ export interface ExtractedDocument { export type DocumentFormat = | 'pdf' | 'docx' | 'doc' | 'odt' | 'rtf' | 'pages' | 'md' | 'html' | 'epub' | 'xlsx' | 'ods' | 'numbers' | 'pptx' | 'odp' | 'key' - | 'eml' | 'msg' | 'txt' | 'code' | 'svg' | 'image' | 'zip' | 'url' + | 'eml' | 'msg' | 'txt' | 'code' | 'svg' | 'image' | 'audio' | 'zip' | 'url' export interface DocumentInfo { path: string diff --git a/src/workspace.ts b/src/workspace.ts index cb2e103..db8e3ff 100644 --- a/src/workspace.ts +++ b/src/workspace.ts @@ -79,6 +79,38 @@ main { flex: 1; min-height: 0; display: flex; } .picker li .nm { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } .picker .hint { color: var(--subtle); font-family: var(--mono); font-size: 11px; } +.rich { background: #FFFFFF; color: #1C1C1A; max-width: 860px; margin: 24px auto 60px; padding: 56px clamp(20px, 6vw, 72px); border-radius: 2px; box-shadow: 0 2px 18px rgba(0,0,0,.45); font-family: -apple-system, 'Segoe UI', Calibri, Arial, sans-serif; font-size: 15px; line-height: 1.6; overflow-wrap: anywhere; } +.rich h1, .rich h2, .rich h3, .rich h4 { line-height: 1.25; margin: 1.3em 0 .5em; } +.rich h1 { font-size: 1.7em; } .rich h2 { font-size: 1.35em; } .rich h3 { font-size: 1.15em; } +.rich p { margin: 0 0 .8em; } +.rich table { border-collapse: collapse; margin: 1em 0; font-size: 14px; } +.rich th, .rich td { border: 1px solid #D4D4CF; padding: 5px 8px; vertical-align: top; text-align: left; } +.rich th { background: #F3F3EF; font-weight: 600; } +.rich img { max-width: 100%; height: auto; } +.rich blockquote { margin: 1em 0; padding-left: 14px; border-left: 3px solid #D4D4CF; color: #4A4A46; } +.rich pre, .rich code { font-family: var(--mono); font-size: 13px; background: #F3F3EF; border-radius: 4px; } +.rich pre { padding: 10px 12px; overflow: auto; white-space: pre-wrap; } +.rich .link { color: #1D4ED8; text-decoration: underline; } +.rich .img-missing, .rich .note { color: #6B6B66; font-size: 13px; } +.rich-sheet { max-width: none; margin: 0; padding: 16px; border-radius: 0; box-shadow: none; } +.rich-sheet .sheet { overflow: auto; max-height: none; } +.rich-sheet table { font-size: 13px; font-variant-numeric: tabular-nums; margin: 0; } +.rich-sheet .sheet-name { font-size: 13px; font-family: var(--mono); text-transform: uppercase; letter-spacing: .06em; color: #4A4A46; margin: 18px 0 8px; } +.rich-sheet section:first-child .sheet-name { margin-top: 0; } +.rich-email .mail-head { display: grid; grid-template-columns: max-content 1fr; gap: 4px 14px; margin: 0 0 18px; padding-bottom: 14px; border-bottom: 1px solid #D4D4CF; font-size: 14px; } +.rich-email .mail-head dt { color: #6B6B66; } .rich-email .mail-head dd { margin: 0; } +.sheet-tabs { position: sticky; top: 0; z-index: 2; display: flex; gap: 4px; padding: 8px 16px; background: var(--s1); border-bottom: 1px solid var(--border); overflow-x: auto; } +.sheet-tabs button { flex: none; } +mark.ws-find { background: #FDE68A; color: #111110; border-radius: 2px; } +mark.ws-find.current { background: var(--lime); outline: 2px solid var(--lime); } +.audio-pane { position: absolute; inset: 0; display: flex; flex-direction: column; } +.audio-pane > .scroller { position: relative; inset: auto; flex: 1; min-height: 0; } +.player { flex: none; padding: 10px 14px; background: var(--s1); border-bottom: 1px solid var(--border); } +.player audio { width: 100%; } +.player .hint { margin: 6px 0 0; color: var(--subtle); font-size: 12px; } +.transcript .line { margin: 0 0 6px; display: flex; gap: 10px; align-items: baseline; line-height: 1.6; } +.transcript .line .time { flex: none; font-family: var(--mono); font-size: 11px; color: var(--lime); padding: 0 4px; } +.transcript .line.playing { background: var(--lime-bg); border-radius: 4px; } .text-view { padding: 24px clamp(16px, 4vw, 48px) 60px; background: var(--black); min-height: 100%; } .text-view .sec { max-width: 78ch; margin: 0 auto 26px; } .text-view .sec-label { user-select: none; -webkit-user-select: none; font-family: var(--mono); font-size: 10px; letter-spacing: .08em; text-transform: uppercase; color: var(--lime); margin: 0 0 8px; padding-top: 8px; border-top: 1px solid var(--border); } @@ -187,6 +219,7 @@ main { flex: 1; min-height: 0; display: flex; } const TEXT_FORMATS = new Set(['md', 'code', 'txt']) const RASTER = new Set(['png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp']) const WS = new RegExp('\\\\s+', 'g') + const WORD_CHAR = new RegExp('[\\\\p{L}\\\\p{N}]', 'u') function h(tag, props, ...kids) { const el = document.createElement(tag) @@ -282,6 +315,7 @@ main { flex: 1; min-height: 0; display: flex; } this.pageInput.hidden = this.pageCount.hidden = this.zoomIn.hidden = this.zoomOut.hidden = this.modeBtn.hidden = true this.findInput.hidden = this.prevMatch.hidden = this.nextMatch.hidden = this.matchCount.hidden = true this.findBus = null + this.domMarks = null } clear() { @@ -320,6 +354,7 @@ main { flex: 1; min-height: 0; display: flex; } this.path = path this.format = doc.format this.mode = 'native' + this.view = null this.page = page || 1 this.terms = terms || [] this.nameBtn.textContent = (doc.displayName || fileName(path)) + ' ▾' @@ -338,14 +373,17 @@ main { flex: 1; min-height: 0; display: flex; } try { if (this.format === 'pdf' && this.mode === 'native') await this.renderPdf() else if (this.format === 'image' && RASTER.has(ext) && this.mode === 'native') this.renderImage(doc) + else if (this.format === 'audio') await this.renderAudio() + else if (this.format !== 'pdf' && this.format !== 'image' && this.mode === 'native') await this.renderView() else await this.renderText() } catch (err) { this.fail("Couldn't open this document: " + (err && err.message ? err.message : err)) } - if (this.format === 'image' || this.format === 'pdf') { + const rich = this.view && this.view.kind !== 'text' + if (this.format === 'image' || this.format === 'pdf' || rich) { this.modeBtn.hidden = false - this.modeBtn.textContent = this.mode === 'native' ? 'Text' : (this.format === 'pdf' ? 'PDF' : 'Image') - this.modeBtn.title = this.mode === 'native' ? 'Show the extracted text' : 'Show the original' + this.modeBtn.textContent = this.mode === 'native' ? 'Text' : (this.format === 'pdf' || this.view?.kind === 'pdf' ? 'Pages' : this.format === 'image' ? 'Image' : 'Formatted') + this.modeBtn.title = this.mode === 'native' ? 'Show the text Duct read' : 'Show the document as it looks' } } @@ -362,7 +400,7 @@ main { flex: 1; min-height: 0; display: flex; } return '/api/file/' + encodeURIComponent(fileName(this.path)) + '?path=' + encodeURIComponent(this.path) } - async renderPdf() { + async renderPdf(url = this.fileUrl()) { const { pdfjsLib, EventBus, PDFLinkService, PDFFindController, PDFViewer } = await loadPdf() const scroller = h('div', { class: 'scroller' }, h('div', { class: 'pdfViewer' })) this.body.append(scroller) @@ -389,7 +427,7 @@ main { flex: 1; min-height: 0; display: flex; } this.resizer = new ResizeObserver(() => { if (this.fit && viewer.pagesCount) viewer.currentScaleValue = 'page-width' }) this.resizer.observe(scroller) const doc = await pdfjsLib.getDocument({ - url: this.fileUrl(), cMapUrl: '/vendor/pdfjs/cmaps/', cMapPacked: true, standardFontDataUrl: '/vendor/pdfjs/standard_fonts/', + url, cMapUrl: '/vendor/pdfjs/cmaps/', cMapPacked: true, standardFontDataUrl: '/vendor/pdfjs/standard_fonts/', wasmUrl: '/vendor/pdfjs/wasm/', isEvalSupported: false, }).promise this.pdf = { doc, viewer } @@ -399,7 +437,7 @@ main { flex: 1; min-height: 0; display: flex; } /** Finds the words in the find box (or the searched-for terms) in the PDF: again = the next or previous match. */ find(again, previous) { - if (!this.findBus) return + if (!this.findBus) { this.domFind(again, previous); return } const typed = this.findInput.value.trim() const query = typed === this.terms.join(' ') && this.terms.length ? this.terms : typed const any = Array.isArray(query) ? query.length > 0 : !!query @@ -411,6 +449,152 @@ main { flex: 1; min-height: 0; display: flex; } this.matchCount.textContent = m.total ? m.current + ' of ' + m.total : 'No matches' } + /** + * The document as it looks, from the server's view of it (src/document-view.ts): pages laid out by LibreOffice + * when it's installed, or the format's own structure as cleaned HTML, or else the text. + */ + async renderView() { + const q = '?path=' + encodeURIComponent(this.path) + this.body.replaceChildren(h('div', { class: 'empty', text: 'Opening…' })) + this.view ||= await (await api('/api/document-view' + q)).json() + if (this.view.kind === 'pdf') { + this.body.replaceChildren(h('div', { class: 'empty', text: 'Laying out the pages with LibreOffice…' })) + try { + await api('/api/document-pdf' + q, { method: 'HEAD' }) // converts once, then it's cached + this.body.replaceChildren() + await this.renderPdf('/api/document-pdf' + q) + return + } catch { + this.view = { kind: 'text' } // LibreOffice couldn't: show the text instead + } + } + this.body.replaceChildren() + if (this.view.kind === 'html') this.renderRich(this.view) + else await this.renderText() + } + + /** + * A recording: the player, and the transcript Duct made of it, a line per passage with its time. Clicking a time + * plays from there; the line being played is highlighted. Opened from a result, it's cued at that minute. + */ + async renderAudio() { + const data = await (await api('/api/document-text?path=' + encodeURIComponent(this.path))).json() + const player = h('audio', { controls: true, preload: 'metadata', src: this.fileUrl(), 'aria-label': 'Recording' }) + player.addEventListener('error', () => { bar.append(h('p', { class: 'hint', text: 'This kind of recording can’t play here. “Open in app” plays it; the transcript is below.' })) }, { once: true }) + const bar = h('div', { class: 'player' }, player) + const view = h('div', { class: 'text-view transcript' }) + const LINE = new RegExp('^\\\\[((?:\\\\d+:)?\\\\d+:\\\\d\\\\d)\\\\]\\\\s*(.*)$') + const seconds = t => t.split(':').reduce((acc, n) => acc * 60 + Number(n), 0) + const lines = [] + for (const sec of data.sections) { + const block = h('div', { class: 'sec', 'data-page': sec.page || false }) + for (const raw of sec.text.split('\\n')) { + const m = LINE.exec(raw) + if (!m) { if (raw.trim()) block.append(h('p', { class: 'line', text: raw })); continue } + const at = seconds(m[1]) + const row = h('p', { class: 'line', 'data-at': at }, + h('button', { class: 'ghost time', title: 'Play from ' + m[1], 'aria-label': 'Play from ' + m[1], text: m[1], onclick: () => { player.currentTime = at; player.play().catch(() => {}) } }), + h('span', { text: m[2] })) + lines.push(row) + block.append(row) + } + view.append(block) + } + if (!lines.length) view.append(h('div', { class: 'empty', text: 'No speech was found in this recording.' })) + player.addEventListener('timeupdate', () => { + const t = player.currentTime + let cur = null + for (const l of lines) { if (Number(l.dataset.at) <= t + 0.2) cur = l; else break } + for (const l of lines) l.classList.toggle('playing', l === cur) + }) + const scroller = h('div', { class: 'scroller' }, view) + this.body.append(h('div', { class: 'audio-pane' }, bar, scroller)) + // From a result: cue the recording at its minute (it doesn't start playing by itself). + if (this.page > 1) { + const first = lines.find(l => Number(l.dataset.at) >= (this.page - 1) * 60) + if (first) { player.currentTime = Number(first.dataset.at); first.scrollIntoView({ block: 'center' }) } + } + this.findInput.hidden = false + this.findInput.value = this.terms.join(' ') + if (this.terms.length) this.domFind(false, false) + } + + renderRich(view) { + // Cleaned on the server; checked again here, since this page may run scripts. + const tpl = document.createElement('template') + tpl.innerHTML = view.html + for (const el of [...tpl.content.querySelectorAll('*')]) { + if (/^(script|style|iframe|frame|object|embed|link|meta|base|form|svg|math)$/i.test(el.tagName)) { el.remove(); continue } + for (const a of [...el.attributes]) if (/^on/i.test(a.name) || /^(href|style|srcdoc|formaction|xlink:href|action)$/i.test(a.name) || (a.name === 'src' && !/^data:image\//i.test(a.value))) el.removeAttribute(a.name) + } + const page = h('div', { class: 'rich rich-' + view.via }) + page.append(tpl.content) + const scroller = h('div', { class: 'scroller' }) + if (view.sections && view.sections.length > 1) { + // Spreadsheets: a tab per sheet. + const tabs = h('div', { class: 'sheet-tabs', role: 'tablist', 'aria-label': 'Sheets' }, view.sections.map(sec => + h('button', { role: 'tab', text: sec.title, onclick: () => page.querySelector('[data-page="' + sec.page + '"]')?.scrollIntoView({ block: 'start' }) }))) + scroller.append(tabs) + } + scroller.append(page) + this.body.append(scroller) + this.findInput.hidden = false + this.findInput.value = this.terms.join(' ') + if (this.terms.length) this.domFind(false, false) + } + + /** Find in the formatted or text view: marks every match, and steps through them in order. */ + domFind(again, previous) { + const root = this.body.querySelector('.rich, .text-view') + if (!root) return + const typed = this.findInput.value.trim() + if (!again || !this.domMarks) { + for (const m of [...root.querySelectorAll('mark.ws-find')]) m.replaceWith(...m.childNodes) + root.normalize() + this.domMarks = [] + this.domAt = -1 + const words = (typed === this.terms.join(' ') && this.terms.length ? this.terms : typed.split(WS)).map(w => w.toLowerCase()).filter(w => w.length > 1) + if (words.length) { + const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT) + const nodes = [] + while (walker.nextNode()) nodes.push(walker.currentNode) + for (const node of nodes) { + const text = node.nodeValue, lower = text.toLowerCase() + const hits = [] + for (const w of words) for (let i = lower.indexOf(w); i !== -1; i = lower.indexOf(w, i + 1)) { + if (i > 0 && WORD_CHAR.test(lower[i - 1])) continue + hits.push([i, i + w.length]) + } + if (!hits.length) continue + hits.sort((a, b) => a[0] - b[0]) + const frag = document.createDocumentFragment() + let at = 0 + for (const [from, to] of hits) { + if (from < at) continue + frag.append(text.slice(at, from)) + const m = h('mark', { class: 'ws-find', text: text.slice(from, to) }) + frag.append(m) + this.domMarks.push(m) + at = to + } + frag.append(text.slice(at)) + node.replaceWith(frag) + } + } + } + const marks = this.domMarks + this.prevMatch.hidden = this.nextMatch.hidden = this.matchCount.hidden = !typed + if (!marks.length) { this.matchCount.textContent = typed ? 'No matches' : ''; return } + if (!again) { + // Start at the result's page or sheet, if the document has them. + const first = marks.findIndex(m => (parseInt(m.closest('[data-page]')?.dataset.page || '1', 10) || 1) >= (this.page || 1)) + this.domAt = first === -1 ? 0 : first + } else this.domAt = (this.domAt + (previous ? -1 : 1) + marks.length) % marks.length + marks.forEach((m, i) => m.classList.toggle('current', i === this.domAt)) + marks[this.domAt].scrollIntoView({ block: 'center' }) + this.matchCount.textContent = (this.domAt + 1) + ' of ' + marks.length + } + renderImage(doc) { const img = h('img', { alt: doc?.displayName || fileName(this.path), src: this.fileUrl() }) img.addEventListener('error', () => { this.mode = 'text'; this.render() }) @@ -447,6 +631,9 @@ main { flex: 1; min-height: 0; display: flex; } const target = view.querySelector('[data-page="' + this.page + '"]') if (target && this.page > 1) target.scrollIntoView() } + this.findInput.hidden = false + this.findInput.value = this.terms.join(' ') + if (this.terms.length) this.domFind(false, false) } zoom(factor) { diff --git a/test/audio.test.ts b/test/audio.test.ts new file mode 100644 index 0000000..a240a90 --- /dev/null +++ b/test/audio.test.ts @@ -0,0 +1,93 @@ +import { describe, it, expect, afterAll, afterEach } from 'vitest' +import { execFileSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Duct } from '../src/index.js' +import { categorize } from '../src/ledger.js' +import { clock, decodeAudio, ffmpegPath, probeAudio, setTranscriber, transcriptPages } from '../src/speech.js' + +const dir = mkdtempSync(join(tmpdir(), 'duct-audio-')) +afterAll(() => rmSync(dir, { recursive: true, force: true })) +afterEach(() => setTranscriber(null)) + +/** A WAV of `seconds` of a quiet tone: real audio for ffmpeg to read; the words come from the stand-in model. */ +function wav(path: string, seconds: number): string { + const rate = 16000, n = rate * seconds + const buf = Buffer.alloc(44 + n * 2) + buf.write('RIFF', 0); buf.writeUInt32LE(36 + n * 2, 4); buf.write('WAVE', 8); buf.write('fmt ', 12) + buf.writeUInt32LE(16, 16); buf.writeUInt16LE(1, 20); buf.writeUInt16LE(1, 22); buf.writeUInt32LE(rate, 24) + buf.writeUInt32LE(rate * 2, 28); buf.writeUInt16LE(2, 32); buf.writeUInt16LE(16, 34); buf.write('data', 36); buf.writeUInt32LE(n * 2, 40) + for (let i = 0; i < n; i++) buf.writeInt16LE(Math.round(Math.sin(i / 8) * 2000), 44 + i * 2) + writeFileSync(path, buf) + return path +} + +describe('transcripts', () => { + it('are a page per minute, each line with its time', () => { + expect(clock(5)).toBe('0:05') + expect(clock(125)).toBe('2:05') + expect(clock(3725)).toBe('1:02:05') + expect(transcriptPages([ + { start: 1, end: 4, text: 'Hi Ada, this is Chidi.' }, + { start: 50, end: 58, text: 'The rent is due on Friday.' }, + { start: 130, end: 140, text: 'Send the tenancy agreement.' }, + ])).toEqual(['[0:01] Hi Ada, this is Chidi.\n[0:50] The rent is due on Friday.', '', '[2:10] Send the tenancy agreement.']) + }) + + it('come from audio ffmpeg decodes to 16 kHz mono', async () => { + const path = wav(join(dir, 'tone.wav'), 2) + expect(await ffmpegPath()).toBeTruthy() + expect((await probeAudio(path)).durationSec).toBeCloseTo(2, 1) + const samples = await decodeAudio(path) + expect(samples.length).toBeGreaterThan(31000) + expect(samples.length).toBeLessThan(33000) + }) +}) + +describe('searching inside audio', () => { + it('is off until turned on, then finds what was said, at its minute', async () => { + const folder = join(dir, 'notes') + mkdirSync(folder, { recursive: true }) + wav(join(folder, 'Voice note from Chidi.wav'), 3) + let calls = 0 + setTranscriber(async samples => { + calls++ + expect(samples.length).toBeGreaterThan(40000) + return [{ start: 0.5, end: 2, text: 'Hi Ada, the pioneer status certificate arrived.' }, { start: 75, end: 80, text: 'The tax holiday starts in March.' }] + }) + const duct = new Duct({ embed: false }) + await duct.index(folder) + expect(calls).toBe(0) // audio is off by default + expect(await duct.search('pioneer')).toHaveLength(0) + + duct.setFeatures({ formats: { audio: true } }) + await duct.index(folder) + expect(calls).toBe(1) + const [hit] = await duct.search('tax holiday') + expect(hit!.chunk.documentFormat).toBe('audio') + expect(hit!.chunk.page).toBe(2) // 1:15 is in the second minute + expect(hit!.chunk.content).toContain('[1:15] The tax holiday starts in March.') + await duct.index(folder) + expect(calls).toBe(1) // unchanged recordings aren't transcribed again + }) + + it('leaves songs alone', async () => { + const tone = wav(join(dir, 'tone2.wav'), 1) + const song = join(dir, 'song.mp3') + execFileSync(await ffmpegPath(), ['-y', '-v', 'error', '-i', tone, '-metadata', 'artist=Burna Boy', '-metadata', 'album=Twice as Tall', song]) + expect((await probeAudio(song)).music).toBe(true) + let calls = 0 + setTranscriber(async () => { calls++; return [] }) + const duct = new Duct({ embed: false, features: { formats: { audio: true } } as never }) + await duct.index(song) + expect(calls).toBe(0) + expect(duct.getDocuments()).toHaveLength(0) + }) + + it('shows the speech model download in the privacy ledger', () => { + expect(categorize('huggingface.co')).toBe('models') + expect(categorize('cdn-lfs.huggingface.co')).toBe('models') + expect(categorize('cas-bridge.xethub.hf.co')).toBe('models') + }) +}) diff --git a/test/document-view.test.ts b/test/document-view.test.ts new file mode 100644 index 0000000..ff5d23f --- /dev/null +++ b/test/document-view.test.ts @@ -0,0 +1,148 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import { mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { Server } from 'node:http' +import type { AddressInfo } from 'node:net' +import { cleanHtml, convertToPdf, documentView, findLibreOffice, resetLibreOfficeLookup } from '../src/document-view.js' +import { Duct } from '../src/index.js' +import { createServer } from '../src/server.js' +import { makeDocx } from './helpers.js' + +const dir = mkdtempSync(join(tmpdir(), 'duct-view-')) +// Looked up before the tests below switch LibreOffice off for themselves. +const hasLibreOffice = !!findLibreOffice() +resetLibreOfficeLookup() +afterAll(() => rmSync(dir, { recursive: true, force: true })) +// These tests are about the views drawn without LibreOffice, whether or not this machine has it. +beforeAll(() => { process.env['DUCT_SOFFICE'] = 'off'; resetLibreOfficeLookup() }) +afterAll(() => { delete process.env['DUCT_SOFFICE']; resetLibreOfficeLookup() }) + +describe('cleaning HTML from documents', () => { + it('removes scripts, handlers, styles, frames and links that navigate', async () => { + const html = await cleanHtml(` +

    Hello there

    + + logo +
    +
    cell
    + old`) + expect(html).not.toMatch(/script|onclick|onerror|onmouseover|style|iframe|javascript:|svg|form|input|tracker|class="x"|id=|href/i) + expect(html).toContain('Hello') + expect(html).toContain('there') + expect(html).toContain('[logo]') // a remote image is never fetched; its description stays + expect(html).toContain('colspan="2"') + expect(html).toContain('old') // unknown tags keep their text + }) + + it('keeps embedded images and harmless structure', async () => { + const png = 'data:image/png;base64,iVBORw0KGgo=' + const html = await cleanHtml(`

    Title

    • One
    chart`) + expect(html).toContain('

    Title

    ') + expect(html).toContain('One') + expect(html).toContain(`src="${png}"`) + expect(html).not.toContain('svg+xml') + }) +}) + +describe('views of each format', () => { + it('draws Word documents from their structure', async () => { + const path = join(dir, 'Lease.docx') + writeFileSync(path, await makeDocx(['The tenant may terminate on 30 days notice.', 'Rent is due monthly.'])) + const view = await documentView(path) + expect(view).toMatchObject({ kind: 'html', via: 'word' }) + expect((view as { html: string }).html).toMatch(/

    The tenant may terminate on 30 days notice\.<\/p>/) + }) + + it('shows spreadsheets as tables, a sheet at a time', async () => { + const XLSX = await import('xlsx') + const book = XLSX.utils.book_new() + XLSX.utils.book_append_sheet(book, XLSX.utils.aoa_to_sheet([['Item', 'Amount'], ['Rent', 1200], ['Bonus', 50]]), 'Budget') + XLSX.utils.book_append_sheet(book, XLSX.utils.aoa_to_sheet([['Name'], ['Ada']]), 'People') + const path = join(dir, 'Budget.xlsx') + writeFileSync(path, XLSX.write(book, { type: 'buffer', bookType: 'xlsx' })) + const view = await documentView(path) as { kind: string; html: string; sections: { title: string; page: number }[] } + expect(view.kind).toBe('html') + expect(view.sections).toEqual([{ title: 'Budget', page: 1 }, { title: 'People', page: 2 }]) + expect(view.html).toContain('Amount') + expect(view.html).toContain('1200') + expect(view.html).toContain('<b>Bonus</b>') // cell text is text, never markup + expect(view.html).toContain('data-page="2"') + }) + + it('formats Markdown, and cleans HTML written inside it', async () => { + const path = join(dir, 'Notes.md') + writeFileSync(path, '# Notes\n\n- **Bold** point\n\n\n') + const view = await documentView(path) as { kind: string; html: string } + expect(view.html).toContain('

    Notes

    ') + expect(view.html).toContain('Bold') + expect(view.html).not.toMatch(/onerror| { + const path = join(dir, 'Offer.eml') + writeFileSync(path, [ + 'From: Ada Okafor ', 'To: chidi@okafor.ng', 'Subject: The offer', 'Date: Mon, 5 Oct 2026 09:00:00 +0100', + 'MIME-Version: 1.0', 'Content-Type: text/html; charset=utf-8', '', + '

    Please see the attached offer.

    ', + ].join('\r\n')) + const view = await documentView(path) as { kind: string; via: string; html: string } + expect(view.via).toBe('email') + expect(view.html).toContain('
    From
    Ada Okafor <ada@okafor.ng>
    ') + expect(view.html).toContain('attached') + expect(view.html).not.toMatch(/track\.example|script/) + }) + + it('falls back to the text for other formats', async () => { + const path = join(dir, 'notes.txt') + writeFileSync(path, 'plain') + expect(await documentView(path)).toEqual({ kind: 'text' }) + }) +}) + +describe('the view routes', () => { + let server: Server + let base: string + const docx = join(dir, 'Indexed.docx') + beforeAll(async () => { + writeFileSync(docx, await makeDocx(['Indexed text.'])) + const duct = new Duct({ embed: false }) + await duct.index(docx) + server = createServer(duct, { libraryDir: join(dir, 'lib') }).listen(0, '127.0.0.1') + await new Promise(r => server.once('listening', r)) + base = `http://127.0.0.1:${(server.address() as AddressInfo).port}` + }) + afterAll(() => server.close()) + + it('only show indexed documents', async () => { + const ok = await (await fetch(`${base}/api/document-view?path=${encodeURIComponent(docx)}`)).json() + expect(ok.kind).toBe('html') + const other = join(dir, 'Notes.md') // on disk, but not indexed + expect((await fetch(`${base}/api/document-view?path=${encodeURIComponent(other)}`)).status).toBe(404) + expect((await fetch(`${base}/api/document-pdf?path=${encodeURIComponent('/etc/passwd')}`)).status).toBe(404) + // Without LibreOffice, asking for pages says so instead of failing silently. + const pdf = await fetch(`${base}/api/document-pdf?path=${encodeURIComponent(docx)}`) + expect(pdf.status).toBe(422) + }) +}) + +// Runs where LibreOffice is installed (CI installs it on Linux). +describe('pages from LibreOffice', () => { + const soffice = () => { delete process.env['DUCT_SOFFICE']; resetLibreOfficeLookup(); return findLibreOffice() } + afterAll(() => { process.env['DUCT_SOFFICE'] = 'off'; resetLibreOfficeLookup() }) + + it.skipIf(!hasLibreOffice)( + 'turns documents into PDF pages, converting each version once', async () => { + // RTF, which LibreOffice reads natively (the tests' minimal .docx and .pptx are too bare for its import). + expect(soffice()).toBeTruthy() + const cache = join(dir, 'cache') + const rtf = join(dir, 'Contract.rtf') + writeFileSync(rtf, '{\\rtf1\\ansi\\deff0 {\\fonttbl {\\f0 Arial;}}\\f0\\fs24 Clause 1. The tenant may terminate on 30 days notice.\\par }') + expect(await documentView(rtf)).toEqual({ kind: 'pdf', via: 'libreoffice' }) + const pdf = await convertToPdf(rtf, cache) + expect(readFileSync(pdf).subarray(0, 5).toString()).toBe('%PDF-') + const again = statSync(pdf).mtimeMs + expect(await convertToPdf(rtf, cache)).toBe(pdf) // cached + expect(statSync(pdf).mtimeMs).toBe(again) + }, 180_000) +}) diff --git a/test/formats.test.ts b/test/formats.test.ts index 01fa878..2704160 100644 --- a/test/formats.test.ts +++ b/test/formats.test.ts @@ -153,7 +153,7 @@ describe('Apple iWork', () => { const numbers = await makeZip({ 'QuickLook/Preview.pdf': makeTextPdf([[[12, 700, 'Expense report']], [[12, 700, 'Total travel spend']]]) }) const doc = await extract(file('expenses.numbers', numbers)) expect(doc.pages).toEqual(['Expense report', 'Total travel spend']) - }) + }, 30_000) // reads PDFs; slow Windows runners have taken 10 s it('skips files named .key that are not Keynote decks (e.g. TLS keys) without reporting a failure', async () => { const dir = join(work, 'keys') diff --git a/test/index-store.test.ts b/test/index-store.test.ts index 870c9fb..598f81d 100644 --- a/test/index-store.test.ts +++ b/test/index-store.test.ts @@ -78,7 +78,7 @@ describe('change detection', () => { expect(byName['broken.pdf'].status).toBe('failed') expect(byName['broken.pdf'].error).toBeTruthy() expect(duct.stats().documents).toBe(1) - }) + }, 30_000) // reads PDFs; slow Windows runners have taken 6 s }) describe('watched folders', () => { diff --git a/test/whatsapp.test.ts b/test/whatsapp.test.ts index 4286e6d..155ea56 100644 --- a/test/whatsapp.test.ts +++ b/test/whatsapp.test.ts @@ -73,7 +73,9 @@ describe('importing an export', () => { const duct = new Duct({ embed: false }) const lib = join(work, 'lib') const r = await importWhatsApp(duct, lib, zipPath, 'WhatsApp Chat - Okafor Holdings.zip') - expect(r).toMatchObject({ chat: 'Okafor Holdings', messages: 6, attachments: 1, skipped: 1 }) + // The voice note is kept with the chat; it's transcribed only once audio search is turned on. + expect(r).toMatchObject({ chat: 'Okafor Holdings', messages: 6, attachments: 2, skipped: 0 }) + expect(duct.getDocuments().some(d => d.path.endsWith('.opus'))).toBe(false) const doc = duct.getDocuments().find(d => d.displayName === 'Supply agreement.docx · from Chidi Okafor')! expect(doc.path.endsWith(join('WhatsApp', 'Okafor Holdings', 'Supply agreement.docx'))).toBe(true)