diff --git a/.github/workflows/desktop-packages.yml b/.github/workflows/desktop-packages.yml index 7aee76e..a44e79b 100644 --- a/.github/workflows/desktop-packages.yml +++ b/.github/workflows/desktop-packages.yml @@ -165,6 +165,43 @@ jobs: dist/filecollector-windows-*-x64.zip.sha256 if-no-files-found: error + # 可选的 LibreOffice 转换器包 (Office→PDF 视觉转换链路): 主包体积不变, + # 用户按需下载解压到便携包根目录。LO_VERSION 固定于脚本内, 升级时手动更新。 + # PR 不构建 (350MB 下载纯耗时); main/tag 上构建, tag 时随 Release 发布。 + converter-package: + name: Build Windows LibreOffice converter package + runs-on: windows-latest + if: github.event_name != 'pull_request' + defaults: + run: + shell: msys2 {0} + steps: + - name: Check out source + uses: actions/checkout@v4 + + - name: Set up MSYS2 UCRT64 + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: false + install: >- + unzip + zip + + - name: Build LibreOffice converter package + run: | + chmod +x tools/build-converter-package.sh + tools/build-converter-package.sh + + - name: Upload converter package + uses: actions/upload-artifact@v4 + with: + name: filecollector-windows-converter-x64 + path: | + dist/filecollector-converter-windows-*-x64.zip + dist/filecollector-converter-windows-*-x64.zip.sha256 + if-no-files-found: error + macos: name: Build macOS app bundle (ARM64) runs-on: macos-latest @@ -188,7 +225,7 @@ jobs: publish: name: Publish desktop packages - needs: [lint, deb, flatpak, windows, macos] + needs: [lint, deb, flatpak, windows, macos, converter-package] if: startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-24.04 permissions: @@ -216,6 +253,12 @@ jobs: name: filecollector-windows-x64 path: release-assets + - name: Download Windows converter package + uses: actions/download-artifact@v4 + with: + name: filecollector-windows-converter-x64 + path: release-assets + - name: Download macOS package uses: actions/download-artifact@v4 with: @@ -228,8 +271,9 @@ jobs: test "$(find release-assets -maxdepth 1 -name '*.deb' -type f | wc -l)" -eq 1 test "$(find release-assets -maxdepth 1 -name '*.flatpak' -type f | wc -l)" -eq 1 test "$(find release-assets -maxdepth 1 -name 'filecollector-windows-*-x64.zip' -type f | wc -l)" -eq 1 + test "$(find release-assets -maxdepth 1 -name 'filecollector-converter-windows-*-x64.zip' -type f | wc -l)" -eq 1 test "$(find release-assets -maxdepth 1 -name 'filecollector-macos-*-arm64.zip' -type f | wc -l)" -eq 1 - test "$(find release-assets -maxdepth 1 -name '*.sha256' -type f | wc -l)" -eq 3 + test "$(find release-assets -maxdepth 1 -name '*.sha256' -type f | wc -l)" -eq 4 - name: Create or update GitHub Release uses: softprops/action-gh-release@v2 diff --git a/README.md b/README.md index 372e291..c1c0d46 100644 --- a/README.md +++ b/README.md @@ -324,6 +324,14 @@ FileCollector 支持自动将二进制文件转换为 Markdown 格式,无需 - **文档文件**(PDF、DOCX、PPTX、XLSX、ODT、ODP、ODS、RTF 等):先通过 LibreOffice 转换为 PDF,再通过 `pdftoppm` 渲染为图片序列,逐页发送给 VLM。 - **转换缓存**:转换结果缓存在工作目录下的 `.filecollector_cache/` 目录中,基于文件 SHA256 哈希判断是否需要重新转换,避免重复处理。 - **可配置扩展名**:在 AI 设置对话框中可自定义允许被 VLM 处理的二进制文件扩展名列表,修改后自动重新评估预处理队列。 +- **失败原因透明**:转换失败时队列行与预览区会显示具体原因(如缺少 LibreOffice),不再只提示"转换失败"。 + +### 处理方式:视觉转换 / 文本抽取 + +在 **AI 设置 → VLM** 的 **处理方式** 下拉中可选两种二进制文档处理流程: + +- **视觉转换**(默认):文档 → PDF → 页面图片 → VLM/OCR,保留图表、公式、表格等视觉结构。Office 文档需要 LibreOffice —— Windows 用户可下载独立的**转换器包**(Release 附件 `filecollector-converter-windows-x64-*.zip`),解压到便携包根目录(与 `bin/` 同级)即可,应用会自动探测 `tools/LibreOffice/program/soffice.exe`;已安装官方 LibreOffice 的用户无需转换器包,应用同样自动探测。 +- **文本抽取**:纯本地解析 DOCX/XLSX/PPTX/ODT/ODP/ODS(ZIP+XML 结构)与 PDF(内置 `pdftotext`),**无需联网、无需 AI 凭据**,秒级完成;代价是丢失图表、公式与嵌入图片,仅适合纯文字型文档。扫描版 PDF 抽不出文本时自动回退视觉转换流程。两种模式的结果在缓存中分别标记,互不污染,切换模式后对已有条目点"重新转换"即可按新模式重跑。 ### VLM 服务商 diff --git a/README_EN.md b/README_EN.md index 69f81ba..97ad9ae 100644 --- a/README_EN.md +++ b/README_EN.md @@ -324,6 +324,14 @@ FileCollector can automatically convert binary files into Markdown format, elimi - **Document files** (PDF, DOCX, PPTX, XLSX, ODT, ODP, ODS, RTF, etc.): First converted to PDF via LibreOffice, then rendered as image sequences via `pdftoppm`, and sent page-by-page to VLM. - **Conversion cache**: Converted results are cached in the `.filecollector_cache/` directory under the working directory. The system uses SHA256 file hashes to determine whether re-conversion is needed, avoiding redundant processing. - **Configurable extensions**: In the AI Settings dialog, you can customize the list of binary file extensions allowed for VLM processing. Changes automatically trigger re-evaluation of the preprocessing queue. +- **Transparent failure reasons**: When conversion fails, the queue row and preview area show the concrete reason (e.g., missing LibreOffice) instead of a bare "conversion failed". + +### Processing mode: Visual conversion / Text extraction + +Choose how binary documents are processed via the **Processing Mode** dropdown in **AI Settings → VLM**: + +- **Visual conversion** (default): document → PDF → page images → VLM/OCR, preserving charts, formulas, and table layouts. Office documents need LibreOffice — Windows users can download the standalone **converter package** (Release asset `filecollector-converter-windows-x64-*.zip`) and extract it into the portable package root (alongside `bin/`); the app auto-detects `tools/LibreOffice/program/soffice.exe`. If the official LibreOffice is already installed, no converter package is needed — it is auto-detected as well. +- **Text extraction**: Pure local parsing of DOCX/XLSX/PPTX/ODT/ODP/ODS (ZIP+XML) and PDF (built-in `pdftotext`) — **no network, no AI credentials required**, finishes in seconds; the trade-off is losing charts, formulas, and embedded images, so it suits text-only documents. Scanned PDFs that yield no text automatically fall back to the visual conversion flow. Results of both modes are tagged separately in the cache; after switching modes, click "retry" on existing items to re-run with the new mode. ### VLM Providers diff --git a/docs/USAGE.md b/docs/USAGE.md index 5741851..26fca26 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -121,6 +121,8 @@ 「设置 → AI 设置」中的 **AI 助手(侧边栏)** 与 **视觉语言模型(VLM)** 均支持**多模型配置方案(Model Profile)**:可同时预置多套模型配置,各自通过下拉菜单一键切换。 +VLM 顶部还有 **处理方式** 下拉,决定二进制文档走哪条管线:**视觉转换**(默认,文档渲染为图片后送 VLM/OCR,保留图表/公式/表格;Office 文件需要 LibreOffice,Windows 可下载 Release 里的转换器包解压到便携包根目录)或 **文本抽取**(本地直接解析 Office/PDF,快速离线且无需 AI 凭据,但丢失图表/公式/嵌入图片;扫描版 PDF 自动回退视觉转换)。两种模式的缓存结果分开标记,切换后对已有条目点"重新转换"即可按新模式重跑。 + **配置文件位置**:`~/.config/filecollector/settings.json` #### 配置文件格式 diff --git a/docs/USAGE_EN.md b/docs/USAGE_EN.md index db10004..7b383ba 100644 --- a/docs/USAGE_EN.md +++ b/docs/USAGE_EN.md @@ -120,6 +120,8 @@ After selecting a working directory, use the search box at the top of the left-s Both **AI Assistant (Sidebar)** and **Vision-Language Model (VLM)** in "Settings → AI Settings" support **Model Profiles**: you can pre-configure multiple model setups and switch between them instantly from a dropdown menu. +The VLM section also has a **Processing Mode** dropdown that selects the pipeline for binary documents: **Visual conversion** (default — documents are rendered to images and sent to VLM/OCR, keeping charts/formulas/tables; Office files need LibreOffice — Windows users can download the converter package from Release assets and extract it into the portable package root) or **Text extraction** (parses Office/PDF locally — fast, offline, no AI credentials required, but charts/formulas/embedded images are lost; scanned PDFs automatically fall back to visual conversion). Cached results are tagged per mode; after switching, click "retry" on existing items to re-run with the new mode. + **Config file location**: `~/.config/filecollector/settings.json` #### Config file format diff --git a/docs/building/windows.md b/docs/building/windows.md index 8e6ec8f..f5fe29a 100644 --- a/docs/building/windows.md +++ b/docs/building/windows.md @@ -34,7 +34,13 @@ chmod +x tools/build-windows-package.sh tools/build-windows-package.sh ``` -脚本会下载并静态编译 cmark-gfm,运行 Meson 单元测试,递归收集 UCRT64 运行时 DLL(含 GTK 图像加载器的间接依赖)、GTK 动态模块、GSettings schema、完整 Adwaita/hicolor 图标主题,以及项目实际使用的两个 XApp symbolic 图标。然后它会验证所有源码引用的 symbolic 图标都能在包内主题中解析,并在打包前后用 `tools/verify_windows_package.py` 对全部 PE 文件做导入表闭包校验;运行时 DLL 数量不足或存在未解析的非系统 DLL 时,构建直接失败。 +脚本会下载并静态编译 cmark-gfm,运行 Meson 单元测试,递归收集 UCRT64 运行时 DLL(含 GTK 图像加载器的间接依赖)、GTK 动态模块、GSettings schema、完整 Adwaita/hicolor 图标主题、zh_CN 翻译文件(.mo)、项目实际使用的两个 XApp symbolic 图标,以及 ZIP 导出依赖的 zip.exe(含 msys-2.0.dll)。然后它会验证所有源码引用的 symbolic 图标都能在包内主题中解析,并在打包前后用 `tools/verify_windows_package.py` 对全部 PE 文件做导入表闭包校验;运行时 DLL 数量不足或存在未解析的非系统 DLL 时,构建直接失败。 + +> 注意:locale 与 zip.exe 曾因 MSYS2 参数路径转换(`meson --prefix=/usr` 的 +> 参数被改写为 MSYS2 安装根下的路径)而静默缺失,导致 v4.9.1 及之前的 +> Windows 包为纯英文界面且 ZIP 导出不可用。脚本现通过 `MSYS2_ARG_CONV_EXCL` +> 排除转换,并在打包后硬性校验 `locale/zh_CN/LC_MESSAGES/filecollector.mo` +> 与 `bin/zip.exe` 存在于 ZIP 内。 排错提示:DLL 收集脚本按 `MSYSTEM_PREFIX` 判定运行时目录,并用 `ldd` 递归解析依赖。若本地构建得到的 `bin/` 里几乎没有 DLL,先确认是在 MSYS2 **UCRT64** 终端中执行(`echo $MSYSTEM_PREFIX` 应为 `/ucrt64`),以及 `ldd staging/bin/filecollector.exe` 能正常解析。 @@ -45,6 +51,25 @@ dist/filecollector-windows--x64.zip dist/filecollector-windows--x64.zip.sha256 ``` +## LibreOffice 转换器包(可选) + +主包不捆绑 LibreOffice(体积原因)。Office 文档的**视觉转换**链路(soffice → PDF → pdftoppm → VLM)依赖 LibreOffice,通过独立的转换器包交付: + +```bash +tools/build-converter-package.sh +``` + +脚本下载官方 LibreOffice MSI(版本固定于脚本内 `LO_VERSION`),用 `msiexec /a` 管理员解包(不写注册表、不进 Program Files),打包为: + +```text +dist/filecollector-converter-windows--x64.zip +dist/filecollector-converter-windows--x64.zip.sha256 +``` + +布局为 `tools/LibreOffice/program/soffice.exe`,解压到便携包根目录(与 `bin/` 同级)即生效;应用通过 `BinaryConverter.resolve_soffice()` 自动探测。CI 中该包在 main/tag 推送时构建,tag 推送时随 Release 发布。 + +注意:`msiexec` 的 `/a` 参数从 bash 直接传入会被 MSYS2 改写为路径(与 `--prefix=/usr` 同类陷阱),脚本通过 `cmd //c` 复合字符串绕过。 + ## 试运行 将 ZIP 解压后,始终通过下列启动器启动: diff --git a/meson.build b/meson.build index 262a307..d909bc9 100644 --- a/meson.build +++ b/meson.build @@ -128,6 +128,7 @@ sources = files( 'src/services/binary_converter.vala', 'src/services/token_estimator.vala', 'src/services/binary_preprocessor.vala', + 'src/services/office_text_extractor.vala', 'src/services/config_manager.vala', 'src/services/file_content_reader.vala', 'src/services/file_generator.vala', @@ -163,8 +164,10 @@ sources = files( ) # win32_dpapi_shim.c 仅 Windows 构建需要 (DPAPI 封装), 其它平台不存在该文件。 +# win32_console_shim.c 仅 Windows 构建需要 (GUI 子系统下的 CLI 控制台接管)。 if host_machine.system() == 'windows' sources += files('src/win32_dpapi_shim.c') + sources += files('src/win32_console_shim.c') endif # macos_keychain_shim.c 仅 macOS 构建需要 (SecItem* 封装), 其它平台不存在该文件。 diff --git a/po/POTFILES b/po/POTFILES index 69cdfb9..5a8a1c6 100644 --- a/po/POTFILES +++ b/po/POTFILES @@ -13,6 +13,7 @@ src/services/ai_client.vala src/services/ai_types.vala src/services/binary_converter.vala src/services/binary_preprocessor.vala +src/services/office_text_extractor.vala src/services/config_manager.vala src/services/file_content_reader.vala src/services/file_generator.vala diff --git a/po/zh_CN.po b/po/zh_CN.po index 3a9b1eb..9ec27e0 100644 --- a/po/zh_CN.po +++ b/po/zh_CN.po @@ -2693,3 +2693,135 @@ msgstr "正在预处理 0/0 个文件..." #: src/widgets/vlm_queue_controller.vala:146 msgid "Cancel All" msgstr "取消全部" + +# ── 二进制预处理: 双模式 + 失败原因透出 (2026-11 新增) ── +# 注: 本批次由人工补录 (本机无 msgmerge), 后续重新生成 .pot 时请逐条核对 + +#: src/services/binary_converter.vala +msgid "Failed to create tmp dir for PDF conversion" +msgstr "创建 PDF 转换临时目录失败" + +#: src/services/binary_converter.vala +#, c-format +msgid "LibreOffice conversion failed (exit %d): %s" +msgstr "LibreOffice 转换失败 (退出码 %d): %s" + +#: src/services/binary_converter.vala +msgid "LibreOffice is required to convert Office documents. Download the converter package (see README) or install LibreOffice, then retry." +msgstr "转换 Office 文档需要 LibreOffice:请下载转换器包(见 README)或安装 LibreOffice 后重试。" + +#: src/services/binary_converter.vala +#, c-format +msgid "pdftoppm failed: %s" +msgstr "pdftoppm 失败: %s" + +#: src/services/binary_preprocessor.vala +msgid "Multimodal AI is disabled or credential is empty, cannot convert binary files" +msgstr "多模态 AI 未启用或凭据为空, 无法转换二进制文件" + +#: src/services/vlm_task_runner.vala +msgid "Multimodal AI is disabled in settings" +msgstr "多模态 AI 在设置中未启用" + +#: src/services/vlm_task_runner.vala +msgid "Multimodal AI credential is empty, cannot convert binary files" +msgstr "多模态 AI 凭据为空, 无法转换二进制文件" + +#: src/services/vlm_task_runner.vala +#, c-format +msgid "Image load failed: %s" +msgstr "图片加载失败: %s" + +#: src/services/vlm_task_runner.vala +#, c-format +msgid "Document render failed: %s" +msgstr "文档渲染失败: %s" + +#: src/services/office_text_extractor.vala +#, c-format +msgid "File too large for text extraction: %s" +msgstr "文件过大, 无法进行文本抽取: %s" + +#: src/services/office_text_extractor.vala +msgid "Unsupported format for text extraction" +msgstr "该格式不支持文本抽取" + +#: src/services/office_text_extractor.vala +msgid "Not a valid DOCX (missing word/document.xml)" +msgstr "不是有效的 DOCX (缺少 word/document.xml)" + +#: src/services/office_text_extractor.vala +msgid "Not a valid XLSX (no worksheet content)" +msgstr "不是有效的 XLSX (无工作表内容)" + +#: src/services/office_text_extractor.vala +msgid "Not a valid PPTX (no slides)" +msgstr "不是有效的 PPTX (无幻灯片)" + +#: src/services/office_text_extractor.vala +msgid "Not a valid ODF document (missing content.xml)" +msgstr "不是有效的 ODF 文档 (缺少 content.xml)" + +#: src/services/office_text_extractor.vala +#, c-format +msgid "pdftotext failed: %s" +msgstr "pdftotext 失败: %s" + +#: src/services/office_text_extractor.vala +msgid "PDF appears to be scanned (no extractable text)" +msgstr "PDF 疑似扫描件 (无可抽取文本)" + +#: src/services/office_text_extractor.vala +msgid "Not a ZIP container (EOCD not found)" +msgstr "不是 ZIP 容器 (未找到 EOCD)" + +#: src/services/office_text_extractor.vala +msgid "ZIP64 archives are not supported" +msgstr "不支持 ZIP64 归档" + +#: src/services/office_text_extractor.vala +msgid "Encrypted ZIP entries are not supported" +msgstr "不支持加密的 ZIP 条目" + +#: src/services/office_text_extractor.vala +#, c-format +msgid "Unsupported ZIP compression method %u" +msgstr "不支持的 ZIP 压缩方法 %u" + +#: src/services/office_text_extractor.vala +msgid "ZIP deflate decompression failed" +msgstr "ZIP deflate 解压失败" + +#: src/services/office_text_extractor.vala +msgid "Corrupted ZIP container" +msgstr "ZIP 容器已损坏" + +#: src/services/office_text_extractor.vala +#, c-format +msgid "Text truncated (over %s)" +msgstr "文本已截断 (超过 %s)" + +#: src/cli.vala +msgid "Text extracted locally" +msgstr "本地文本抽取完成" + +#: src/widgets/preferences_dialog.vala +msgid "Processing Mode" +msgstr "处理方式" + +#: src/widgets/preferences_dialog.vala +msgid "Visual conversion (keeps charts, formulas, tables)" +msgstr "视觉转换 (保留图表/公式/表格)" + +#: src/widgets/preferences_dialog.vala +msgid "Text extraction (fast, offline, text only)" +msgstr "文本抽取 (快速离线, 仅文本)" + +#: src/widgets/preferences_dialog.vala +msgid "Visual conversion renders documents to images and needs LibreOffice for Office files; text extraction parses Office/PDF locally without AI credentials." +msgstr "视觉转换把文档渲染为图片, Office 文件需要 LibreOffice; 文本抽取在本地直接解析 Office/PDF, 无需 AI 凭据。" + +#: src/window.vala +#, c-format +msgid "Reason: %s" +msgstr "原因: %s" diff --git a/src/cli.vala b/src/cli.vala index a1d062e..4b3f8a3 100644 --- a/src/cli.vala +++ b/src/cli.vala @@ -322,8 +322,22 @@ public class CliController : GLib.Object { return true; } + // 输入路径解析 (--select-file / --load): 相对路径先按进程 cwd 查找 + // (旧行为, 兼容既有脚本), 找不到且设置了 --work-dir 时再按 work_dir + // 解析 (README 完整工作流示例的约定)。输出路径 (--save / --export) + // 不在此列, 保持相对 cwd 不变。 + private string resolve_input_path (string path) { + if (Path.is_absolute (path) || work_dir == null) return path; + if (FileUtils.test (path, FileTest.EXISTS)) return path; + try { + var resolved = work_dir.resolve_relative_path (path); + if (resolved.query_exists ()) return resolved.get_path (); + } catch (Error e) { } + return path; + } + private bool add_file (string path) { - var file = File.new_for_path (path); + var file = File.new_for_path (resolve_input_path (path)); if (!file.query_exists ()) { stderr.printf (_("Error: File does not exist: %s\n"), path); return false; @@ -348,13 +362,18 @@ public class CliController : GLib.Object { item.preprocessed_content = md; item.preprocess_status = PreprocessStatus.COMPLETED; item.from_cache = from_cache; + item.preprocess_error = null; stdout.printf (_(" ↳ %s: %s\n"), - from_cache ? _("Reused from cache") : _("VLM conversion called"), + from_cache ? _("Reused from cache") + : (ConfigManager.get_binary_process_mode () == ConfigManager.BINARY_MODE_TEXT) + ? _("Text extracted locally") + : _("VLM conversion called"), file.get_basename ()); } catch (Error e) { stderr.printf (_(" ⚠ Preprocessing %s failed: %s\n"), file.get_basename (), e.message); item.preprocess_status = PreprocessStatus.FAILED; + item.preprocess_error = e.message; } } return true; @@ -441,7 +460,8 @@ public class CliController : GLib.Object { } private bool load_project (string path) { - var file = File.new_for_path (path); + string resolved = resolve_input_path (path); + var file = File.new_for_path (resolved); if (!file.query_exists ()) { stderr.printf (_("Error: Project file does not exist: %s\n"), path); return false; @@ -450,7 +470,7 @@ public class CliController : GLib.Object { try { string content; size_t len; - FileUtils.get_contents (path, out content, out len); + FileUtils.get_contents (resolved, out content, out len); var parser = new Json.Parser (); parser.load_from_data (content); diff --git a/src/controllers/ai_controller.vala b/src/controllers/ai_controller.vala index aafe517..bd2f161 100644 --- a/src/controllers/ai_controller.vala +++ b/src/controllers/ai_controller.vala @@ -903,8 +903,13 @@ public class AIController : GLib.Object { // ─── 静态辅助方法 ──────────────────────────────────────────────── private static string rel_path (string root, string full) { - if (full.has_prefix (root)) { - string r = full.substring (root.length); + // 归一分隔符为 "/": Windows 反斜杠路径下 substring 会产出 + // "\src\main.vala" 形式, 回传给 AI 后再解析会被 Path.is_absolute + // 误判为绝对路径 (Windows 下 "\x" 也算绝对) + string r = full.replace ("\\", "/"); + string base_r = root.replace ("\\", "/"); + if (r.has_prefix (base_r)) { + r = r.substring (base_r.length); while (r.has_prefix ("/")) r = r.substring (1); return r.length > 0 ? r : full; } @@ -912,8 +917,11 @@ public class AIController : GLib.Object { } private static string normalize_path (string path) { + // 统一分隔符为 "/": Windows 上 get_path() 返回反斜杠路径, 若不归一, + // 沙箱前缀比较 (allowed + "/") 永远不命中, AI 文件工具全部误判越界 + string p = path.replace ("\\", "/"); var stack = new Gee.ArrayList (); - foreach (unowned string part in path.split ("/")) { + foreach (unowned string part in p.split ("/")) { if (part == "" || part == ".") continue; if (part == "..") { if (stack.size > 0) stack.remove_at (stack.size - 1); @@ -922,7 +930,7 @@ public class AIController : GLib.Object { } } string joined = string.joinv ("/", (string[]) stack.to_array ()); - return path.has_prefix ("/") ? "/" + joined : joined; + return p.has_prefix ("/") ? "/" + joined : joined; } private string? resolve_ai_path (string path) { diff --git a/src/main.vala b/src/main.vala index 2180871..f954029 100644 --- a/src/main.vala +++ b/src/main.vala @@ -438,9 +438,19 @@ public class FileCollectorApp : Adw.Application { } #endif +#if WINDOWS + [CCode (cname = "win32_console_init")] + private static extern void win32_console_init (); +#endif + public static int main (string[] args) { #if MACOS setup_macos_bundle_env (); +#endif +#if WINDOWS + // GUI 子系统 exe 在 cmd/PowerShell 里跑 CLI 时 stdout 无句柄, + // 先检测并接管父进程控制台 (管道环境下为空操作) + win32_console_init (); #endif var app = new FileCollectorApp (); int ret = app.run (args); diff --git a/src/models/item_data.vala b/src/models/item_data.vala index eeedad0..6c314db 100644 --- a/src/models/item_data.vala +++ b/src/models/item_data.vala @@ -18,6 +18,8 @@ public class ItemData : GLib.Object { public PreprocessStatus preprocess_status { get; set; default = PreprocessStatus.NONE; } public string? preprocessed_content { get; set; } public bool from_cache { get; set; default = false; } + // 预处理失败原因 (FAILED 状态下展示); 重试/状态重置时必须一并清空 + public string? preprocess_error { get; set; } public int start_line { get; set; default = 0; } public int end_line { get; set; default = 0; } diff --git a/src/services/binary_converter.vala b/src/services/binary_converter.vala index 8cea0c7..04e5fef 100644 --- a/src/services/binary_converter.vala +++ b/src/services/binary_converter.vala @@ -1,6 +1,35 @@ public class BinaryConverter : GLib.Object { public const int MAX_IMAGE_DIMENSION = 2048; + // ─── 捆绑工具解析 ──────────────────────────────────────────────── + // resolve_bundled_tool / new_poppler_launcher 位于 Platform (无 GDK 依赖, + // 便于单元测试链接); 这里仅保留 soffice 探测链 (纯 FileUtils 逻辑)。 + + // 解析 soffice 可执行文件。探测顺序: + // 1. 转换器包约定布局: /../tools/LibreOffice/program/soffice.exe + // 2. Windows 常见安装路径 (官方安装器默认目录, soffice.exe 不进 PATH) + // 3. null → 调用方回退裸名 PATH 搜索 (Linux/macOS 现状) + public static string? resolve_soffice () { + string exe = Platform.get_executable_path (); + if (exe != "." && exe.length > 0) { + var portable = Path.build_filename ( + Path.get_dirname (exe), "..", "tools", "LibreOffice", + "program", "soffice.exe"); + if (FileUtils.test (portable, FileTest.EXISTS)) return portable; + } + string[] roots = { + GLib.Environment.get_variable ("ProgramFiles"), + GLib.Environment.get_variable ("ProgramFiles(x86)") + }; + foreach (unowned string? root in roots) { + if (root == null || root.length == 0) continue; + var candidate = Path.build_filename ( + root, "LibreOffice", "program", "soffice.exe"); + if (FileUtils.test (candidate, FileTest.EXISTS)) return candidate; + } + return null; + } + // 复用临时基目录, 避免每次转换都创建/销毁顶层临时目录. // 子目录以互斥锁保护的自增计数器命名, 保证 VLM 队列并发调用时不冲突. private static string? temp_base_dir = null; @@ -75,14 +104,15 @@ public class BinaryConverter : GLib.Object { return "image/png"; } - public static string[]? convert_to_base64_images (string path) { + public static string[]? convert_to_base64_images (string path) throws Error { string pdf_path = path; string lower = path.down (); string? tmp_pdf = null; if (!lower.has_suffix (".pdf")) { + // soffice 缺失/转换失败会抛出带指引的错误 (NOT_FOUND/FAILED), + // 由调用方捕获后作为失败原因展示 tmp_pdf = convert_office_to_pdf (path); - if (tmp_pdf == null) return null; pdf_path = tmp_pdf; } @@ -97,7 +127,7 @@ public class BinaryConverter : GLib.Object { return result; } - private static string? convert_office_to_pdf (string src) { + private static string convert_office_to_pdf (string src) throws Error { // 复用基目录, 在其中创建本次任务的子目录, 避免并发冲突. // 子目录名使用互斥锁保护的自增计数器, 保证 VLM 队列多线程下唯一. string? tmp_base = get_temp_base (); @@ -109,40 +139,45 @@ public class BinaryConverter : GLib.Object { // 子目录创建失败时回退到独立 mkdtemp, 保证转换流程能继续 task_dir = DirUtils.make_tmp ("fc_pdf_XXXXXX"); if (task_dir == null) { - warning ("Failed to create tmp dir for PDF"); - return null; + throw new IOError.FAILED (_("Failed to create tmp dir for PDF conversion")); } } } else { // 基目录创建失败时回退到独立 mkdtemp task_dir = DirUtils.make_tmp ("fc_pdf_XXXXXX"); if (task_dir == null) { - warning ("Failed to create tmp dir for PDF"); - return null; + throw new IOError.FAILED (_("Failed to create tmp dir for PDF conversion")); } } - string[] argv = {"soffice", "--headless", "--convert-to", "pdf", "--outdir", task_dir, src}; try { + // 转换器包 / 官方安装路径命中 → 绝对路径; 否则裸名 PATH 搜索 + string[] argv = { resolve_soffice () ?? "soffice", "--headless", + "--convert-to", "pdf", "--outdir", task_dir, src }; int status; string stdout_str, stderr_str; Process.spawn_sync (null, argv, null, SpawnFlags.SEARCH_PATH, null, out stdout_str, out stderr_str, out status); - if (status == 0) { - string basename = Path.get_basename (src); - int dot = basename.last_index_of ("."); - string pdf_name = (dot > 0 ? basename.substring (0, dot) : basename) + ".pdf"; - return Path.build_filename (task_dir, pdf_name); - } else { + if (status != 0) { // 捕获 soffice 的 stderr 输出, 便于诊断转换失败原因 // (如文件损坏/格式不支持/LibreOffice 未安装等) - warning ("LibreOffice conversion failed (status %d): %s", - status, (stderr_str ?? "").strip ()); + throw new IOError.FAILED (_("LibreOffice conversion failed (exit %d): %s").printf ( + status, (stderr_str ?? "").strip ())); } + string basename = Path.get_basename (src); + int dot = basename.last_index_of ("."); + string pdf_name = (dot > 0 ? basename.substring (0, dot) : basename) + ".pdf"; + return Path.build_filename (task_dir, pdf_name); + } catch (SpawnError e) { + // 裸名搜索也找不到: Windows 上既没有转换器包也没装 LibreOffice。 + // 失败路径必须清理本次的 task_dir, 否则每次尝试都泄漏一个目录。 + cleanup_dir (task_dir); + throw new IOError.NOT_FOUND (_( + "LibreOffice is required to convert Office documents. Download the converter package (see README) or install LibreOffice, then retry.")); } catch (Error e) { - warning ("LibreOffice conversion failed: %s", e.message); + cleanup_dir (task_dir); + throw e; } - return null; } private static string[]? render_pdf_to_base64_images (string pdf_path) { @@ -153,22 +188,24 @@ public class BinaryConverter : GLib.Object { } string prefix = Path.build_filename (tmp_dir, "page"); - string[] argv = {"pdftoppm", "-png", "-r", "200", pdf_path, prefix}; try { - int status; - string stdout_str, stderr_str; // 捕获 stderr: pdftoppm 失败时 (如 PDF 损坏/poppler 未安装) 会把 // 诊断信息写到 stderr, 原实现丢弃 stderr 只打印退出码, 无法诊断。 - Process.spawn_sync (null, argv, null, SpawnFlags.SEARCH_PATH, null, - out stdout_str, out stderr_str, out status); - if (status != 0) { - warning ("pdftoppm failed (status %d): %s", - status, (stderr_str ?? "").strip ()); + // Windows 便携包用捆绑的 pdftoppm (绝对路径 + POPPLER_DATADIR), + // 其余平台回退裸名 PATH 搜索。 + string? tool = Platform.resolve_bundled_tool ("pdftoppm.exe"); + string[] argv = { tool ?? "pdftoppm", "-png", "-r", "200", pdf_path, prefix }; + var launcher = Platform.new_poppler_launcher (); + var proc = launcher.spawnv (argv); + string out_str, err_str; + proc.communicate_utf8 (null, null, out out_str, out err_str); + if (!proc.get_successful ()) { + warning (_("pdftoppm failed: %s"), (err_str ?? "").strip ()); cleanup_dir (tmp_dir); return null; } } catch (Error e) { - warning ("pdftoppm failed: %s", e.message); + warning (_("pdftoppm failed: %s"), e.message); cleanup_dir (tmp_dir); return null; } @@ -216,9 +253,10 @@ public class BinaryConverter : GLib.Object { * 返回临时 PDF 路径, is_temp = true. 调用方处理完毕后须调用 * cleanup_dir(Path.get_dirname(result)) 清理临时目录. * - * 返回 null 表示无法解析/转换失败. + * soffice 缺失/转换失败时抛出带指引的错误 (NOT_FOUND/FAILED), + * 不再返回 null —— 调用方捕获后作为失败原因展示。 */ - public static string? resolve_upload_source (string path, out bool is_temp) { + public static string resolve_upload_source (string path, out bool is_temp) throws Error { is_temp = false; string lower = path.down (); @@ -228,13 +266,11 @@ public class BinaryConverter : GLib.Object { lower.has_suffix (".jpeg") || lower.has_suffix (".webp") || lower.has_suffix (".bmp") || lower.has_suffix (".tiff") || lower.has_suffix (".tif")) { - is_temp = false; return path; } // 其余 (Office 等): 转 PDF 后上传 - string? tmp_pdf = convert_office_to_pdf (path); - if (tmp_pdf == null) return null; + string tmp_pdf = convert_office_to_pdf (path); is_temp = true; return tmp_pdf; } diff --git a/src/services/binary_preprocessor.vala b/src/services/binary_preprocessor.vala index 42463df..735fab6 100644 --- a/src/services/binary_preprocessor.vala +++ b/src/services/binary_preprocessor.vala @@ -15,11 +15,15 @@ public class BinaryPreprocessor : GLib.Object { ) throws Error { from_cache = false; + // 0. 当前处理模式对应的缓存 flow 标记 (同一文件两种模式产物不同) + string flow = (ConfigManager.get_binary_process_mode () == ConfigManager.BINARY_MODE_TEXT) + ? "text" : "vlm"; + // 1. 先走轻量指纹 (size:mtime) 快速命中缓存, 避免对大文件 (PDF/图片) 计算 SHA256. // 命中即返回, 未命中 (含旧版无 quick 字段的缓存) 再回退到完整 SHA256 路径. string quick = PreprocessCache.compute_file_hash_fast (item.file_path); var cache = new PreprocessCache (work_dir_path); - string? cached = cache.get_cached_markdown_quick (item.file_path, quick); + string? cached = cache.get_cached_markdown_quick (item.file_path, quick, flow); if (cached != null) { from_cache = true; return cached; @@ -36,13 +40,23 @@ public class BinaryPreprocessor : GLib.Object { warning ("Hash computation failed for %s: %s", item.file_path, e.message); } if (hash_valid) { - cached = cache.get_cached_markdown (item.file_path, hash); + cached = cache.get_cached_markdown (item.file_path, hash, flow); if (cached != null) { from_cache = true; return cached; } } + // 2.5 "文本抽取" 模式: 支持的格式直接本地抽纯文本, 不需要 VLM 凭据与 + // 网络, 因此放在凭据检查之前。抽取失败 (含扫描版 PDF 抽空) 返回 + // null, 继续走下方视觉转换/OCR 流程兜底。vlm 模式下此函数直接 + // 返回 null, 不改变现有行为。 + string? text_md = try_extract_text_document ( + item.file_path, hash_valid ? hash : null, cache); + if (text_md != null) { + return text_md; + } + // 3. Load VLM settings var settings = ConfigManager.load_multimodal_ai_settings (); bool has_credential = (settings.provider == ConfigManager.PROVIDER_PADDLEOCR) @@ -50,7 +64,7 @@ public class BinaryPreprocessor : GLib.Object { : settings.api_key.length > 0; if (!settings.enabled || !has_credential) { throw new IOError.FAILED ( - "多模态 AI 未启用或凭据为空, 无法转换二进制文件" + _("Multimodal AI is disabled or credential is empty, cannot convert binary files") ); } @@ -59,10 +73,7 @@ public class BinaryPreprocessor : GLib.Object { if (settings.provider == ConfigManager.PROVIDER_PADDLEOCR) { // PaddleOCR 云端路径: 上传原文件, 轮询并合并 Markdown bool is_temp; - string? upload_source = BinaryConverter.resolve_upload_source (item.file_path, out is_temp); - if (upload_source == null) { - throw new IOError.FAILED ("准备上传源失败: " + item.file_path); - } + string upload_source = BinaryConverter.resolve_upload_source (item.file_path, out is_temp); try { var client = new PaddleOCRClient (settings.paddleocr_token, settings.timeout); // 仅在哈希有效时传入缓存, 使配图落盘到对应缓存子文件夹 @@ -109,7 +120,7 @@ public class BinaryPreprocessor : GLib.Object { // 7. Save to cache (仅在 hash 有效时保存, 避免空 hash 污染 manifest) if (hash_valid) { - cache.save_markdown (item.file_path, hash, md); + cache.save_markdown (item.file_path, hash, md, flow); } return md; @@ -122,13 +133,45 @@ public class BinaryPreprocessor : GLib.Object { ItemData item, string? work_dir_path ) throws Error { if (work_dir_path == null) return null; + // 按当前处理模式匹配 flow 标记 (两种模式的产物互不命中) + string flow = (ConfigManager.get_binary_process_mode () == ConfigManager.BINARY_MODE_TEXT) + ? "text" : "vlm"; // 先轻量指纹命中, 未命中回退 SHA256 路径 (兼容旧缓存) string quick = PreprocessCache.compute_file_hash_fast (item.file_path); var cache = new PreprocessCache (work_dir_path); - string? cached = cache.get_cached_markdown_quick (item.file_path, quick); + string? cached = cache.get_cached_markdown_quick (item.file_path, quick, flow); if (cached != null) return cached; string hash = PreprocessCache.compute_file_hash (item.file_path); - return cache.get_cached_markdown (item.file_path, hash); + return cache.get_cached_markdown (item.file_path, hash, flow); + } + + // "文本抽取" 模式入口: 支持的格式本地抽纯文本并写入缓存 (flow=text)。 + // 返回 null 表示未抽取成功 —— 模式非 text / 格式不支持 / 抽取失败 + // (含扫描版 PDF 抽空), 调用方应继续走视觉转换/OCR 流程兜底。 + // 本分支不需要 VLM 凭据与网络, 调用点必须放在凭据检查之前。 + public static string? try_extract_text_document ( + string file_path, string? hash, PreprocessCache? cache + ) { + if (ConfigManager.get_binary_process_mode () != ConfigManager.BINARY_MODE_TEXT) { + return null; + } + int dot = file_path.last_index_of ("."); + if (dot < 0) return null; + if (!OfficeTextExtractor.supports (file_path.substring (dot).down ())) { + return null; + } + try { + string md = OfficeTextExtractor.extract_markdown (file_path); + if (cache != null && hash != null) { + cache.save_markdown (file_path, hash, md, "text"); + } + return md; + } catch (Error e) { + // 抽取失败一律回退 (不抛出), 原因留日志供诊断 + warning ("Text extraction failed for %s, falling back to visual/OCR: %s", + file_path, e.message); + return null; + } } // 默认提示词 (基于 file_path 字符串, 不依赖 ItemData GObject) diff --git a/src/services/config_manager.vala b/src/services/config_manager.vala index d3be58a..e8e400a 100644 --- a/src/services/config_manager.vala +++ b/src/services/config_manager.vala @@ -122,6 +122,13 @@ public class ConfigManager : GLib.Object { public const string PROVIDER_OPENAI = "openai"; public const string PROVIDER_PADDLEOCR = "paddleocr"; + // 二进制文件处理模式: + // vlm = 视觉转换 (soffice → pdftoppm → VLM/OCR, 保图表/公式/表格, 默认) + // text = 文本抽取 (纯本地解析 OOXML/ODF + pdftotext, 离线且无需凭据, + // 但丢失图表/公式/嵌入图片) + public const string BINARY_MODE_VLM = "vlm"; + public const string BINARY_MODE_TEXT = "text"; + public struct AISettings { public bool enabled; public string profile_name; // 当前激活的 ai_models 配置方案名, 空串表示尚未选择 @@ -351,6 +358,39 @@ public class ConfigManager : GLib.Object { } } + // ─── 二进制文件处理模式 (视觉转换 / 文本抽取) ──────────────────────── + + public static string get_binary_process_mode () { + config_mutex.lock (); + try { + var root = load_settings_root_unlocked (); + if (root != null && root.has_member ("binary_process_mode")) { + string mode = root.get_string_member ("binary_process_mode"); + if (mode == BINARY_MODE_TEXT) return BINARY_MODE_TEXT; + } + } catch (Error e) { + warning ("Failed to load binary_process_mode: %s", e.message); + } finally { + config_mutex.unlock (); + } + return BINARY_MODE_VLM; + } + + public static void save_binary_process_mode (string mode) { + config_mutex.lock (); + try { + Json.Object root = load_settings_root_unlocked () ?? new Json.Object (); + // 非法值一律落回默认, 防止手改配置文件后两条管线都走不到 + root.set_string_member ("binary_process_mode", + mode == BINARY_MODE_TEXT ? BINARY_MODE_TEXT : BINARY_MODE_VLM); + write_settings_root_unlocked (root); + } catch (Error e) { + warning ("Failed to save binary_process_mode: %s", e.message); + } finally { + config_mutex.unlock (); + } + } + // ─── 多模态 AI 设置 ────────────────────────────────────────────────── public struct MultimodalAISettings { diff --git a/src/services/git_service.vala b/src/services/git_service.vala index db77ce9..1279b7d 100644 --- a/src/services/git_service.vala +++ b/src/services/git_service.vala @@ -30,6 +30,11 @@ public class GitService : GLib.Object { public static string run_git (string work_dir, string[] args) throws GLib.Error { string[] argv = { git_executable () }; + // 非 ASCII 文件名默认被 core.quotePath 转成 "\346\226\207..." 八进制 + // 转义, status/log 解析端按原样取路径后存在性检查必然失败, 中文 + // 文件会被静默漏掉。统一关闭转义, 输出保持原始 UTF-8。 + argv += "-c"; + argv += "core.quotePath=false"; foreach (var a in args) argv += a; var launcher = new SubprocessLauncher ( diff --git a/src/services/office_text_extractor.vala b/src/services/office_text_extractor.vala new file mode 100644 index 0000000..0cf2bd1 --- /dev/null +++ b/src/services/office_text_extractor.vala @@ -0,0 +1,522 @@ +using Gee; + +// 纯 Vala 的 Office 文档 / PDF 文本抽取器 (可选的 "文本抽取" 处理模式)。 +// +// 与视觉转换 (soffice → pdftoppm → VLM) 互补: 文本模式零网络、无需 VLM 凭据、 +// 秒级完成, 但只覆盖文字与表格结构 —— 图表/公式/嵌入图片会丢失, 因此默认 +// 模式仍为视觉转换 (ConfigManager.binary_process_mode)。 +// +// 覆盖格式: +// - OOXML (.docx/.xlsx/.pptx): ZIP 容器 + XML, 内置最小 ZIP 读取器解析 +// - ODF (.odt/.ods/.odp): 同为 ZIP + XML +// - PDF: 调用捆绑的 pdftotext (与 pdftoppm 同属 poppler, Windows 包已捆绑; +// 其余平台回退 PATH 搜索)。抽不出文本 (扫描版) 抛 NOT_SUPPORTED, +// 调用方据此回退视觉/OCR 流程。 +// +// ZIP 解析范围: PKZIP 经典布局 (EOCD + central directory + local header), +// 支持 STORED 与 DEFLATE; ZIP64 / 加密条目明确报错而不是静默给出错误结果。 +public class OfficeTextExtractor : GLib.Object { + + // 单文件守卫: 防止超大容器把内存吃爆 (文本抽取场景下 50MB 已极宽裕) + private const int MAX_FILE_SIZE = 50 * 1024 * 1024; + // 抽出文本的上限, 超出截断 (导出与 token 估算不需要无限正文) + private const int MAX_TEXT_SIZE = 8 * 1024 * 1024; + + private const string[] SUPPORTED_EXTS = { + ".docx", ".xlsx", ".pptx", ".odt", ".ods", ".odp", ".pdf" + }; + + // ext 为带前导点的小写扩展名 (与 ItemData.is_allowed_binary_target 口径一致) + public static bool supports (string ext_lower) { + foreach (unowned string e in SUPPORTED_EXTS) { + if (ext_lower == e) return true; + } + return false; + } + + public static string extract_markdown (string path) throws Error { + var f = File.new_for_path (path); + var info = f.query_info (FileAttribute.STANDARD_SIZE, FileQueryInfoFlags.NONE); + if (info.get_size () > MAX_FILE_SIZE) { + throw new IOError.FAILED (_("File too large for text extraction: %s").printf ( + GLib.format_size ((int64) info.get_size ()))); + } + + string md; + string lower = path.down (); + if (lower.has_suffix (".pdf")) { + md = extract_pdf (path); + } else { + uint8[] data; + FileUtils.get_data (path, out data); + var zip = new ZipReader (data); + if (lower.has_suffix (".docx")) { + md = extract_docx (zip); + } else if (lower.has_suffix (".xlsx")) { + md = extract_xlsx (zip); + } else if (lower.has_suffix (".pptx")) { + md = extract_pptx (zip); + } else if (lower.has_suffix (".ods")) { + md = extract_odf (zip, true); + } else if (lower.has_suffix (".odt") || lower.has_suffix (".odp")) { + md = extract_odf (zip, false); + } else { + throw new IOError.NOT_SUPPORTED (_("Unsupported format for text extraction")); + } + } + + if (md.length > MAX_TEXT_SIZE) { + md = md.substring (0, MAX_TEXT_SIZE) + "\n\n" + + _("Text truncated (over %s)").printf (GLib.format_size (MAX_TEXT_SIZE)); + } + return md; + } + + // ─── 各格式抽取 ─────────────────────────────────────────────────── + + private static string extract_docx (ZipReader zip) throws Error { + var entry = zip.find ("word/document.xml"); + if (entry == null) { + throw new IOError.NOT_SUPPORTED (_("Not a valid DOCX (missing word/document.xml)")); + } + string xml = bytes_to_utf8 (entry.data); + // 表格单元格 → \t, 行/段落 → \n, 软换行/制表符保留; 其余标签剥离 + string t = xml.replace ("", "\t") + .replace ("", "\n") + .replace ("", "\n") + .replace ("", "\t") + .replace ("", "\n"); + return tidy_text (decode_entities (strip_tags (t))); + } + + private static string extract_xlsx (ZipReader zip) throws Error { + var shared = new Gee.ArrayList (); + var sst = zip.find ("xl/sharedStrings.xml"); + if (sst != null) { + string xml = bytes_to_utf8 (sst.data); + foreach (string si in extract_blocks (xml, "si")) { + // 内可能是单个 或富文本 .. 多段: 拼接全部 + var sb2 = new StringBuilder (); + foreach (string t in extract_tag_contents (si, "t")) { + sb2.append (t); + } + shared.add (decode_entities (sb2.str)); + } + } + + // 工作表按文件名中的数字自然排序 (sheet1, sheet2, ... sheet10) + var sheets = zip.entries_matching ("xl/worksheets/"); + sheets.sort ((a, b) => { + return num_after (a.name, "sheet") - num_after (b.name, "sheet"); + }); + if (sheets.size == 0) { + throw new IOError.NOT_SUPPORTED (_("Not a valid XLSX (no worksheet content)")); + } + + var sb = new StringBuilder (); + int sheet_idx = 0; + foreach (var sheet in sheets) { + sheet_idx++; + if (sb.len > 0) sb.append ("\n"); + sb.append ("## Sheet ").append (sheet_idx.to_string ()).append ("\n"); + string xml = bytes_to_utf8 (sheet.data); + foreach (string row in extract_blocks (xml, "row")) { + var cells = new Gee.ArrayList (); + foreach (string cell in extract_blocks (row, "c")) { + // t="s" → sharedStrings 索引; 其余读 原值 + var vs = extract_tag_contents (cell, "v"); + string val = vs.size > 0 ? vs.get (0) : ""; + if (val.length > 0 && cell.contains ("t=\"s\"")) { + int idx = int.parse (val); + val = (idx >= 0 && idx < shared.size) ? shared.get (idx) : ""; + } else { + val = decode_entities (val); + } + cells.add (val); + } + if (cells.size == 0) continue; + sb.append (string.joinv ("\t", cells.to_array ())); + sb.append ("\n"); + } + } + return sb.str; + } + + private static string extract_pptx (ZipReader zip) throws Error { + var slides = zip.entries_matching ("ppt/slides/"); + slides.sort ((a, b) => { + return num_after (a.name, "slide") - num_after (b.name, "slide"); + }); + if (slides.size == 0) { + throw new IOError.NOT_SUPPORTED (_("Not a valid PPTX (no slides)")); + } + var sb = new StringBuilder (); + int n = 0; + foreach (var slide in slides) { + n++; + if (sb.len > 0) sb.append ("\n"); + sb.append ("## Slide ").append (n.to_string ()).append ("\n"); + string xml = bytes_to_utf8 (slide.data); + string t = xml.replace ("", "\t") + .replace ("", "\n") + .replace ("", "\n"); + sb.append (tidy_text (decode_entities (strip_tags (t)))); + } + return sb.str; + } + + // ODF 家族共用 content.xml; 电子表格额外处理单元格/行边界 + private static string extract_odf (ZipReader zip, bool spreadsheet) throws Error { + var entry = zip.find ("content.xml"); + if (entry == null) { + throw new IOError.NOT_SUPPORTED (_("Not a valid ODF document (missing content.xml)")); + } + string xml = bytes_to_utf8 (entry.data); + string t = xml.replace ("", "\n") + .replace ("", "\n") + .replace ("", "\t") + .replace ("", "\n"); + if (spreadsheet) { + t = t.replace ("", "\t") + .replace ("", "\n"); + } + return tidy_text (decode_entities (strip_tags (t))); + } + + private static string extract_pdf (string path) throws Error { + string? tool = Platform.resolve_bundled_tool ("pdftotext.exe"); + string[] argv = { tool ?? "pdftotext", "-layout", "-enc", "UTF-8", path, "-" }; + var launcher = Platform.new_poppler_launcher (); + var proc = launcher.spawnv (argv); + string out_str, err_str; + proc.communicate_utf8 (null, null, out out_str, out err_str); + if (!proc.get_successful ()) { + throw new IOError.FAILED (_("pdftotext failed: %s").printf ((err_str ?? "").strip ())); + } + string text = (out_str ?? "").strip (); + if (text.length == 0) { + // 扫描版 PDF (纯图): 文本抽取无价值, 调用方据此回退视觉/OCR 流程 + throw new IOError.NOT_SUPPORTED (_("PDF appears to be scanned (no extractable text)")); + } + return tidy_text (text); + } + + // ─── 文本收尾 ───────────────────────────────────────────────────── + + // 连续空行压缩到最多一个, 行尾空白清理 (顺带处理 CRLF 残留) + private static string tidy_text (string text) { + var sb = new StringBuilder (); + int blank = 0; + foreach (unowned string line in text.split ("\n")) { + string trimmed = line.strip (); + if (trimmed.length == 0) { + blank++; + if (blank <= 2) sb.append ("\n"); + continue; + } + blank = 0; + sb.append (trimmed); + sb.append ("\n"); + } + return sb.str; + } + + // ─── XML 处理 ───────────────────────────────────────────────────── + + // 剥掉全部标签 (含注释); 边界标签已在调用方 replace 为 \n/\t + private static string strip_tags (string xml) { + var sb = new GLib.StringBuilder.sized (xml.length); + int pos = 0; + while (pos < xml.length) { + int lt = xml.index_of ("<", pos); + if (lt < 0) { + sb.append (xml.substring (pos)); + break; + } + if (lt > pos) sb.append (xml.substring (pos, lt - pos)); + if (lt + 3 < xml.length && xml[lt + 1] == '!' && xml[lt + 2] == '-' + && xml[lt + 3] == '-') { + int end = xml.index_of ("-->", lt + 4); + pos = (end >= 0) ? end + 3 : xml.length; + } else { + int gt = xml.index_of (">", lt); + pos = (gt >= 0) ? gt + 1 : xml.length; + } + } + return sb.str; + } + + private static string decode_entities (string s) { + if (!s.contains ("&")) return s; + var sb = new GLib.StringBuilder.sized (s.length); + int pos = 0; + while (pos < s.length) { + int amp = s.index_of ("&", pos); + if (amp < 0) { + sb.append (s.substring (pos)); + break; + } + if (amp > pos) sb.append (s.substring (pos, amp - pos)); + int semi = s.index_of (";", amp); + if (semi < 0 || semi - amp > 12) { + // 不是实体: 原样输出 '&' + sb.append ("&"); + pos = amp + 1; + continue; + } + string ent = s.substring (amp + 1, semi - amp - 1); + string? decoded = null; + switch (ent) { + case "amp": decoded = "&"; break; + case "lt": decoded = "<"; break; + case "gt": decoded = ">"; break; + case "quot": decoded = "\""; break; + case "apos": decoded = "'"; break; + default: + if (ent.has_prefix ("#x") || ent.has_prefix ("#X")) { + int v = hex_val (ent.substring (2)); + decoded = (v > 0 && v <= 0x10FFFF) ? ((unichar) v).to_string () : null; + } else if (ent.has_prefix ("#")) { + int v = int.parse (ent.substring (1)); + decoded = (v > 0 && v <= 0x10FFFF) ? ((unichar) v).to_string () : null; + } + break; + } + if (decoded != null) { + sb.append (decoded); + } else { + sb.append (s.substring (amp, semi - amp + 1)); + } + pos = semi + 1; + } + return sb.str; + } + + private static int hex_val (string hex) { + if (hex.length == 0 || hex.length > 6) return -1; + int v = 0; + for (int i = 0; i < hex.length; i++) { + char c = hex[i]; + v <<= 4; + if (c >= '0' && c <= '9') v += c - '0'; + else if (c >= 'a' && c <= 'f') v += c - 'a' + 10; + else if (c >= 'A' && c <= 'F') v += c - 'A' + 10; + else return -1; + } + return v; + } + + // 提取 ... 整块 (含标签); start 精确匹配 "' / '/' / 空白, 排除 这类前缀碰撞 + private static Gee.ArrayList extract_blocks (string xml, string tag) { + var result = new Gee.ArrayList (); + string open = "<" + tag; + string close = ""; + int pos = 0; + while (true) { + int s = find_exact_open (xml, open, pos); + if (s < 0) break; + int e = xml.index_of (close, s); + if (e < 0) break; + result.add (xml.substring (s, e - s + close.length)); + pos = e + close.length; + } + return result; + } + + // 提取 内容 的内容 (不含标签); 空元素 记为 "" + private static Gee.ArrayList extract_tag_contents (string xml, string tag) { + var result = new Gee.ArrayList (); + string open = "<" + tag; + string close = ""; + int pos = 0; + while (true) { + int s = find_exact_open (xml, open, pos); + if (s < 0) break; + int gt = xml.index_of (">", s); + if (gt < 0) break; + if (gt > s && xml[gt - 1] == '/') { + result.add (""); + pos = gt + 1; + continue; + } + int e = xml.index_of (close, gt); + if (e < 0) break; + result.add (xml.substring (gt + 1, e - gt - 1)); + pos = e + close.length; + } + return result; + } + + // 查找 "' / '/' / 空白的出现位置; 找不到返回 -1 + private static int find_exact_open (string xml, string open, int from) { + int s = xml.index_of (open, from); + while (s >= 0) { + char nxt = (s + open.length < xml.length) ? xml[s + open.length] : '\0'; + if (nxt == '>' || nxt == '/' || nxt == ' ' + || nxt == '\t' || nxt == '\n' || nxt == '\r') { + return s; + } + s = xml.index_of (open, s + 1); + } + return -1; + } + + // 文件名中 prefix 之后的连续数字 (sheet1.xml → 1); 无数字排到最后 + private static int num_after (string name, string prefix) { + int p = name.index_of (prefix); + if (p < 0) return int.MAX; + int v = 0; + bool any = false; + for (int i = p + prefix.length; i < name.length; i++) { + char c = name[i]; + if (c >= '0' && c <= '9') { + v = v * 10 + (c - '0'); + any = true; + } else if (any) { + break; + } + } + return any ? v : int.MAX; + } + + private static string bytes_to_utf8 (uint8[] data) { + var sb = new GLib.StringBuilder.sized (data.length); + sb.append_len ((string) data, data.length); + return sb.str; + } + + // ─── 最小 ZIP 读取器 ────────────────────────────────────────────── + + private class ZipEntry : GLib.Object { + public string name; + public uint8[] data; + public ZipEntry (string name, uint8[] data) { + this.name = name; + this.data = data; + } + } + + private class ZipReader : GLib.Object { + private Gee.HashMap map = + new Gee.HashMap (); + + public ZipReader (uint8[] data) throws Error { + parse (data); + } + + public ZipEntry? find (string name) { + return map[name]; + } + + public Gee.ArrayList entries_matching (string prefix) { + var result = new Gee.ArrayList (); + foreach (var e in map.values) { + if (e.name.has_prefix (prefix)) result.add (e); + } + return result; + } + + private void parse (uint8[] data) throws Error { + // 1. 从尾部扫描 EOCD (0x06054b50): 结构 22 字节 + 注释最长 65535 + int eocd = -1; + int min_scan = int.max (0, data.length - 22 - 65535); + for (int i = data.length - 22; i >= min_scan; i--) { + if (le32 (data, i) == 0x06054b50u) { + eocd = i; + break; + } + } + if (eocd < 0) { + throw new IOError.NOT_SUPPORTED (_("Not a ZIP container (EOCD not found)")); + } + uint16 total = le16 (data, eocd + 10); + uint32 cd_size = le32 (data, eocd + 12); + uint32 cd_offset = le32 (data, eocd + 16); + if (total == 0xFFFFu || cd_offset == 0xFFFFFFFFu || cd_size == 0xFFFFFFFFu) { + throw new IOError.NOT_SUPPORTED (_("ZIP64 archives are not supported")); + } + if ((int) cd_offset + (int) cd_size > data.length) { + throw corrupted (); + } + // 2. 遍历 central directory + int p = (int) cd_offset; + for (uint16 i = 0; i < total; i++) { + if (p + 46 > data.length || le32 (data, p) != 0x02014b50u) { + throw corrupted (); + } + uint16 flags = le16 (data, p + 8); + uint16 method = le16 (data, p + 10); + uint32 comp_size = le32 (data, p + 20); + uint16 name_len = le16 (data, p + 28); + uint16 extra_len = le16 (data, p + 30); + uint16 comment_len = le16 (data, p + 32); + uint32 local_offset = le32 (data, p + 42); + if ((flags & 0x1u) != 0) { + throw new IOError.NOT_SUPPORTED (_("Encrypted ZIP entries are not supported")); + } + if (p + 46 + name_len > data.length) { + throw corrupted (); + } + string name = bytes_to_utf8 (data[p + 46 : p + 46 + name_len]); + // 3. local header 定位数据区 (local 与 central 的 name/extra 长度可能不同) + if (local_offset + 30 > (uint32) data.length + || le32 (data, (int) local_offset) != 0x04034b50u) { + throw corrupted (); + } + uint16 lname_len = le16 (data, (int) local_offset + 26); + uint16 lextra_len = le16 (data, (int) local_offset + 28); + int start = (int) local_offset + 30 + lname_len + lextra_len; + if (start < 0 || start + (int) comp_size > data.length) { + throw corrupted (); + } + uint8[] content; + if (method == 0) { + content = data[start : start + (int) comp_size]; + } else if (method == 8) { + content = inflate (data[start : start + (int) comp_size]); + } else { + throw new IOError.NOT_SUPPORTED ( + _("Unsupported ZIP compression method %u").printf (method)); + } + map[name] = new ZipEntry (name, content); + p += 46 + name_len + extra_len + comment_len; + } + } + + private uint8[] inflate (uint8[] raw) throws Error { + // 注意: vala 0.56 的 glib vapi 没有 GLib.BytesBuilder, 用 + // GLib.ByteArray 累积; ZlibDecompressor 构造参数复用 + // ZlibCompressorFormat 枚举 (0.56 vapi 无独立的 + // ZlibDecompressorFormat 绑定)。 + var mins = new MemoryInputStream.from_data (raw); + var decomp = new GLib.ZlibDecompressor (GLib.ZlibCompressorFormat.RAW); + var cis = new GLib.ConverterInputStream (mins, decomp); + // Gee 泛型不支持数组类型参数, 二进制累积用 GLib.ByteArray + var out_buf = new GLib.ByteArray (); + var buf = new uint8[64 * 1024]; + while (true) { + ssize_t n = cis.read (buf); + if (n < 0) { + throw new IOError.FAILED (_("ZIP deflate decompression failed")); + } + if (n == 0) break; + out_buf.append (buf[0 : (int) n]); + } + return out_buf.steal (); + } + + private static Error corrupted () { + return new IOError.FAILED (_("Corrupted ZIP container")); + } + + private static uint16 le16 (uint8[] d, int off) { + return (uint16) (d[off] | (d[off + 1] << 8)); + } + + private static uint32 le32 (uint8[] d, int off) { + return ((uint32) d[off]) | ((uint32) d[off + 1] << 8) + | ((uint32) d[off + 2] << 16) | ((uint32) d[off + 3] << 24); + } + } +} diff --git a/src/services/preprocess_cache.vala b/src/services/preprocess_cache.vala index 5bf72aa..b3a72cb 100644 --- a/src/services/preprocess_cache.vala +++ b/src/services/preprocess_cache.vala @@ -70,7 +70,10 @@ public class PreprocessCache : GLib.Object { } } - public string? get_cached_markdown (string abs_path, string current_hash) { + // flow: 产生该结果的处理管线 ("vlm" 视觉转换 / "text" 文本抽取)。 + // 同一文件两种模式的产物不同, 命中条件必须匹配; 旧缓存条目无 flow 字段 + // 视为 "vlm", 保证向后兼容。 + public string? get_cached_markdown (string abs_path, string current_hash, string flow = "vlm") { cache_mutex.lock (); string? result = null; string rel_path = get_rel_path (abs_path); @@ -78,7 +81,7 @@ public class PreprocessCache : GLib.Object { if (manifest.has_member (rel_path)) { var entry = manifest.get_object_member (rel_path); string cached_hash = entry.get_string_member_with_default ("hash", ""); - if (cached_hash == current_hash) { + if (cached_hash == current_hash && entry_flow (entry) == flow) { string? md_path = resolve_md_path (entry, cached_hash); if (md_path != null && FileUtils.test (md_path, FileTest.EXISTS)) { try { @@ -112,7 +115,7 @@ public class PreprocessCache : GLib.Object { * 旧版缓存条目未记录 quick 字段时会 miss, 此时调用方回退到 get_cached_markdown * (走 SHA256 路径) 以兼容既有磁盘缓存. */ - public string? get_cached_markdown_quick (string abs_path, string quick_fp) { + public string? get_cached_markdown_quick (string abs_path, string quick_fp, string flow = "vlm") { cache_mutex.lock (); string? result = null; string rel_path = get_rel_path (abs_path); @@ -120,7 +123,7 @@ public class PreprocessCache : GLib.Object { if (manifest.has_member (rel_path)) { var entry = manifest.get_object_member (rel_path); string cached_quick = entry.get_string_member_with_default ("quick", ""); - if (cached_quick == quick_fp) { + if (cached_quick == quick_fp && entry_flow (entry) == flow) { string? md_path = resolve_md_path (entry, entry.get_string_member_with_default ("hash", "")); if (md_path != null && FileUtils.test (md_path, FileTest.EXISTS)) { try { @@ -136,7 +139,7 @@ public class PreprocessCache : GLib.Object { return result; } - public void save_markdown (string abs_path, string current_hash, string markdown_content) { + public void save_markdown (string abs_path, string current_hash, string markdown_content, string flow = "vlm") { string rel_path = get_rel_path (abs_path); // 扁平结构: 以 hash 命名的独立子文件夹 cache_dir/{hash}/, 内含 content.md 与 // (可选) imgs/; markdown 内相对路径 "imgs/xxx.jpg" 相对 {hash}/ 基点, 可正确 @@ -154,6 +157,8 @@ public class PreprocessCache : GLib.Object { entry.set_string_member ("quick", compute_file_hash_fast (abs_path)); // 新结构标记: 缓存子文件夹名 (= hash), 读取时据此定位独立目录 entry.set_string_member ("cache_subdir", current_hash); + // 产生该结果的处理管线 ("vlm"/"text"), 命中条件按 flow 匹配 + entry.set_string_member ("flow", flow); entry.set_boolean_member ("has_images", has_imgs_for (current_hash)); entry.set_int_member ("timestamp", new DateTime.now_utc ().to_unix ()); manifest.set_member (rel_path, AI.SchemaHelper.obj_to_node (entry)); @@ -164,6 +169,11 @@ public class PreprocessCache : GLib.Object { cache_mutex.unlock (); } + // 条目的 flow 归一: 旧条目无该字段视为 "vlm" (历史行为即视觉转换) + private static string entry_flow (Json.Object entry) { + return entry.get_string_member_with_default ("flow", "vlm"); + } + // 将缓存图片落盘到 cache_dir/{hash}/imgs/, 供 markdown 内相对路径引用. // 调用时机: PaddleOCR 解析完成后, 逐张下载并写盘. 同时把 has_images 标记写回 manifest. public void save_image (string current_hash, string relpath, uint8[] data) { diff --git a/src/services/vlm_queue.vala b/src/services/vlm_queue.vala index fe088c7..38e3757 100644 --- a/src/services/vlm_queue.vala +++ b/src/services/vlm_queue.vala @@ -11,11 +11,13 @@ public class VLMQueueManager : GLib.Object { public signal void state_changed (bool has_tasks); // 工作线程的结果回送信号. 主线程接收后, 在 items 中按 file_path 查找 ItemData // 并更新其属性. md 为 null 表示无内容更新 (PROCESSING/FAILED 阶段). + // error_message 仅在 status == FAILED 时有意义, 为用户可读的失败原因. public signal void task_completed ( string file_path, string? preprocessed_content, PreprocessStatus status, - bool from_cache + bool from_cache, + string? error_message ); private Gee.LinkedList pending_queue; @@ -182,8 +184,9 @@ public class VLMQueueManager : GLib.Object { } } catch (ThreadError e) { warning ("Failed to create VLM thread: %s", e.message); + string err = e.message; Idle.add (() => { - task_completed (file_path, null, PreprocessStatus.FAILED, false); + task_completed (file_path, null, PreprocessStatus.FAILED, false, err); finish_task (file_path); return Source.REMOVE; }); diff --git a/src/services/vlm_task_runner.vala b/src/services/vlm_task_runner.vala index ecc4632..a4c4cac 100644 --- a/src/services/vlm_task_runner.vala +++ b/src/services/vlm_task_runner.vala @@ -104,7 +104,9 @@ public class VLMTaskRunner : GLib.Object { // 更新了 vlm_runner.work_dir, local_work_dir 持有的 GFile 也不会被释放. File? local_work_dir = work_dir; - // 1. 检查缓存 + // 1. 检查缓存 (flow 匹配当前处理模式: vlm/text 两种产物互不命中) + string flow = (ConfigManager.get_binary_process_mode () == ConfigManager.BINARY_MODE_TEXT) + ? "text" : "vlm"; string? cached_md = null; string hash = ""; bool hash_valid = false; @@ -113,11 +115,11 @@ public class VLMTaskRunner : GLib.Object { string quick = PreprocessCache.compute_file_hash_fast (file_path); if (local_work_dir != null) { var cache = new PreprocessCache (local_work_dir.get_path ()); - cached_md = cache.get_cached_markdown_quick (file_path, quick); + cached_md = cache.get_cached_markdown_quick (file_path, quick, flow); if (cached_md == null) { hash = PreprocessCache.compute_file_hash (file_path); hash_valid = true; - cached_md = cache.get_cached_markdown (file_path, hash); + cached_md = cache.get_cached_markdown (file_path, hash, flow); } } else { hash = PreprocessCache.compute_file_hash (file_path); @@ -135,7 +137,7 @@ public class VLMTaskRunner : GLib.Object { if (cached_md != null) { string md = cached_md; Idle.add (() => { - manager.task_completed (file_path, md, PreprocessStatus.COMPLETED, true); + manager.task_completed (file_path, md, PreprocessStatus.COMPLETED, true, null); return Source.REMOVE; }); manager.notify_finished (file_path); @@ -144,10 +146,28 @@ public class VLMTaskRunner : GLib.Object { // 2. 标记为 PROCESSING (主线程接收信号后更新 UI) Idle.add (() => { - manager.task_completed (file_path, null, PreprocessStatus.PROCESSING, false); + manager.task_completed (file_path, null, PreprocessStatus.PROCESSING, false, null); return Source.REMOVE; }); + // 1.5 "文本抽取" 模式: 支持的格式直接本地抽纯文本, 不需要 VLM 凭据与 + // 网络。内部自行判断模式 (vlm 模式直接返回 null); 抽取失败 (含 + // 扫描版 PDF 抽空) 返回 null, 继续走下方视觉转换/OCR 流程兜底。 + { + PreprocessCache? cache = (local_work_dir != null) + ? new PreprocessCache (local_work_dir.get_path ()) : null; + string? text_md = BinaryPreprocessor.try_extract_text_document ( + file_path, hash_valid ? hash : null, cache); + if (text_md != null) { + Idle.add (() => { + manager.task_completed (file_path, text_md, PreprocessStatus.COMPLETED, false, null); + return Source.REMOVE; + }); + manager.notify_finished (file_path); + return; + } + } + var settings = ConfigManager.load_multimodal_ai_settings (); // 校验启用状态与对应服务商所需凭据 @@ -155,8 +175,11 @@ public class VLMTaskRunner : GLib.Object { ? settings.paddleocr_token.length > 0 : settings.api_key.length > 0; if (!settings.enabled || !has_credential) { + string why = (!settings.enabled) + ? _("Multimodal AI is disabled in settings") + : _("Multimodal AI credential is empty, cannot convert binary files"); Idle.add (() => { - manager.task_completed (file_path, null, PreprocessStatus.FAILED, false); + manager.task_completed (file_path, null, PreprocessStatus.FAILED, false, why); return Source.REMOVE; }); manager.notify_finished (file_path); @@ -170,10 +193,7 @@ public class VLMTaskRunner : GLib.Object { // ── PaddleOCR 云端路径 ── // 上传原文件 (PDF/图片直传, Office 先转 PDF), 轮询并合并 Markdown. bool is_temp; - string? upload_source = BinaryConverter.resolve_upload_source (file_path, out is_temp); - if (upload_source == null) { - throw new IOError.FAILED (_("Failed to prepare upload source for %s").printf (file_path)); - } + string upload_source = BinaryConverter.resolve_upload_source (file_path, out is_temp); try { // 仅在哈希有效且有工作目录时构造缓存实例, 供配图落盘与 md 保存复用 PreprocessCache? pd_cache = (local_work_dir != null && hash_valid) @@ -197,12 +217,13 @@ public class VLMTaskRunner : GLib.Object { if (ItemData.is_image_file (file_path)) { string? b64 = BinaryConverter.convert_image_to_base64 (file_path); - if (b64 == null) throw new IOError.FAILED ("Image load failed"); + if (b64 == null) throw new IOError.FAILED (_("Image load failed: %s").printf (file_path)); base64_images = { b64 }; mime_types = { BinaryConverter.get_output_mime_for_image (file_path) }; } else { + // soffice/pdftoppm 缺失或转换失败时抛出带指引的错误 string[]? images = BinaryConverter.convert_to_base64_images (file_path); - if (images == null) throw new IOError.FAILED ("Document render failed"); + if (images == null) throw new IOError.FAILED (_("Document render failed: %s").printf (file_path)); base64_images = images; mime_types = new string[images.length]; for (int i = 0; i < images.length; i++) mime_types[i] = "image/png"; @@ -230,13 +251,15 @@ public class VLMTaskRunner : GLib.Object { } Idle.add (() => { - manager.task_completed (file_path, md, PreprocessStatus.COMPLETED, false); + manager.task_completed (file_path, md, PreprocessStatus.COMPLETED, false, null); return Source.REMOVE; }); } catch (Error e) { warning ("VLM Task failed: %s", e.message); + // 失败原因透传给 UI: 队列行/预览占位/tooltip 展示, 不再丢弃 e.message + string fail_msg = e.message; Idle.add (() => { - manager.task_completed (file_path, null, PreprocessStatus.FAILED, false); + manager.task_completed (file_path, null, PreprocessStatus.FAILED, false, fail_msg); return Source.REMOVE; }); } diff --git a/src/services/zip_exporter.vala b/src/services/zip_exporter.vala index 5999cfd..b13a4aa 100644 --- a/src/services/zip_exporter.vala +++ b/src/services/zip_exporter.vala @@ -10,7 +10,8 @@ using Gee; // // 设计上走外部 `zip` 命令而不是 libzip 是因为: // - 项目已有 git/soffice/pdftoppm 等用 Process.spawn_sync 的先例 -// - 零额外依赖 (系统装 libreoffice 时通常也带了 zip) +// - 零额外依赖 (Linux 上系统装 libreoffice 时通常也带了 zip; +// Windows 便携包由 tools/build-windows-package.sh 捆绑 zip.exe) // - 不会因为 libzip 版本差异出 ABI 兼容问题 public class ZipExporter : GLib.Object { @@ -135,16 +136,27 @@ public class ZipExporter : GLib.Object { } private static string compute_relative_path (string abs_path, File? work_dir) { + if (work_dir != null && work_dir.get_path () != null) { + // GIO 的相对路径计算按规范路径比较, Windows 反斜杠与 Unix 正斜杠 + // 皆可 (原实现 has_prefix (wd + "/") 在 Windows 上恒不命中, + // 导致所有文件都落入 _external/) + var rel = work_dir.get_relative_path (File.new_for_path (abs_path)); + if (rel != null) { + return rel.replace ("\\", "/"); + } + } if (work_dir == null) { - // 没有工作目录: 用去掉前导 / 的全路径, 避免同名冲突 - var p = abs_path; + // 没有工作目录: 用去掉前导分隔符的全路径, 避免同名冲突。 + // Windows 的盘符冒号必须一并去掉, 否则 "C:/..." 会被 + // Path.build_filename 当作绝对路径 (最后的绝对分量胜出), + // dest 直接指回源文件, 复制自身必然失败。 + var p = abs_path.replace ("\\", "/"); while (p.has_prefix ("/")) p = p.substring (1); +#if WINDOWS + p = p.replace (":", ""); +#endif return p; } - var wd = work_dir.get_path (); - if (wd != null && abs_path.has_prefix (wd + "/")) { - return abs_path.substring (wd.length + 1); - } // 文件在工作目录外: 用 _external/- 隔离 // 注释之前写"-"但代码未实现 hash, 同名外部文件 // (如 /tmp/notes.txt 与 /home/notes.txt) 会相互覆盖。 diff --git a/src/utils/glob_helper.vala b/src/utils/glob_helper.vala index 5133afa..08a547e 100644 --- a/src/utils/glob_helper.vala +++ b/src/utils/glob_helper.vala @@ -3,11 +3,13 @@ using Gee; public class GlobHelper : GLib.Object { public static bool match_glob (string pattern, string text) { - if (!pattern.contains ("**")) { - var spec = new PatternSpec (pattern.down ()); + // 归一分隔符: Windows 上 AI 给出的模式与路径可能是反斜杠形式 + string norm_pattern = pattern.replace ("\\", "/"); + if (!norm_pattern.contains ("**")) { + var spec = new PatternSpec (norm_pattern.down ()); return spec.match_string (text.down ()); } - return match_glob_recursive (pattern.down (), text.down ()); + return match_glob_recursive (norm_pattern.down (), text.down ()); } private static bool match_glob_recursive (string pattern, string text) { @@ -99,7 +101,9 @@ public class GlobHelper : GLib.Object { } } var child_path = Path.build_filename (current_dir, info.get_name ()); - string rel = child_path.substring (base_dir.length); + // 归一分隔符: Windows 上 build_filename 产出反斜杠路径, 前导 "\" + // 剥不掉, 且 "**/*.py" 这类按 "/" 分段的模式永不命中 + string rel = child_path.substring (base_dir.length).replace ("\\", "/"); if (rel.has_prefix ("/")) rel = rel.substring (1); if (match_glob (pattern, rel)) { diff --git a/src/utils/platform.vala b/src/utils/platform.vala index 8c40c7a..ba963c7 100644 --- a/src/utils/platform.vala +++ b/src/utils/platform.vala @@ -65,6 +65,39 @@ namespace Platform { private extern static int _NSGetExecutablePath ([CCode (array_length = false)] uint8[] buf, ref uint32 bufsize); #endif + // ─── 捆绑外部工具解析 ──────────────────────────────────────────── + // Windows 便携包把 poppler 工具 (pdftoppm/pdftotext) 放在主程序同目录 + // (bin/); 绝对路径优先, 找不到回退裸名 PATH 搜索 (Linux/macOS 现状)。 + // 放在 Platform 是为了让不依赖 GDK 的单元测试 (OfficeTextExtractor) + // 也能链接这两个解析逻辑, 避免 binary_converter 的 Gdk.Pixbuf 依赖 + // 把 gtk4 拉进测试闭包。 + + // 解析捆绑的外部工具, 找不到返回 null (调用方回退裸名 + SpawnFlags.SEARCH_PATH)。 + public static string? resolve_bundled_tool (string exe_name) { + string exe = get_executable_path (); + if (exe == "." || exe.length == 0) return null; + var candidate = Path.build_filename (Path.get_dirname (exe), exe_name); + return FileUtils.test (candidate, FileTest.EXISTS) ? candidate : null; + } + + // poppler 工具专用 launcher: 中文 PDF 需要 CMap 数据, 运行时把 + // POPPLER_DATADIR 指向便携包内 share/poppler (打包脚本已捆绑)。 + public static SubprocessLauncher new_poppler_launcher () throws Error { + var launcher = new SubprocessLauncher ( + SubprocessFlags.STDOUT_PIPE | SubprocessFlags.STDERR_PIPE + ); + string exe = get_executable_path (); + if (exe != "." && exe.length > 0) { + var data_dir = Path.build_filename ( + Path.get_dirname (exe), "..", "share", "poppler"); + if (FileUtils.test (data_dir, FileTest.IS_DIR)) { + launcher.setenv ("POPPLER_DATADIR", data_dir, true); + } + } + return launcher; + } + + /** * 应用数据目录 (gtksourceview 主题等)。 * 解析优先级: diff --git a/src/widgets/context_menus.vala b/src/widgets/context_menus.vala index 0aab322..9745700 100644 --- a/src/widgets/context_menus.vala +++ b/src/widgets/context_menus.vala @@ -83,7 +83,9 @@ public class ContextMenus : GLib.Object { can_retry_vlm = false; } if (it.item_type == "file" && work_dir != null && it.file_path != null) { - if (!it.file_path.has_prefix (work_dir.get_path () + "/")) { + // GIO 的 File.has_prefix 按规范路径比较, 跨平台兼容两种分隔符 + // (原 has_prefix (wd + "/") 字符串手术在 Windows 上恒不命中) + if (!File.new_for_path (it.file_path).has_prefix (work_dir)) { has_external = true; } } diff --git a/src/widgets/preferences_dialog.vala b/src/widgets/preferences_dialog.vala index 99eb5bc..ac53a37 100644 --- a/src/widgets/preferences_dialog.vala +++ b/src/widgets/preferences_dialog.vala @@ -28,8 +28,10 @@ public class PreferencesDialog : GLib.Object { private Gee.ArrayList vlm_profiles; private bool switching_vlm_profile = false; + private bool switching_mm_mode = false; private Switch chk_mm_enabled; + private Adw.ComboRow combo_mm_process_mode; private Adw.ComboRow combo_mm_provider; private Adw.ComboRow combo_vlm_profile; private EntryRow edit_mm_base_url; @@ -228,6 +230,31 @@ public class PreferencesDialog : GLib.Object { mm_enabled_row.set_activatable_widget (chk_mm_enabled); mm_group.add (mm_enabled_row); + // ── 二进制处理方式下拉: 视觉转换 (默认, 保图表/公式/表格) / 文本抽取 + // (快速离线, 无需凭据, 但丢失图表/公式/嵌入图片)。立即持久化, + // 两条管线在下次预处理时读取。 + var process_mode_model = new StringList (new string[] { + _("Visual conversion (keeps charts, formulas, tables)"), + _("Text extraction (fast, offline, text only)") + }); + combo_mm_process_mode = new Adw.ComboRow (); + combo_mm_process_mode.set_title (_("Processing Mode")); + combo_mm_process_mode.set_subtitle (_( + "Visual conversion renders documents to images and needs LibreOffice for Office files; text extraction parses Office/PDF locally without AI credentials.")); + combo_mm_process_mode.set_model (process_mode_model); + mm_group.add (combo_mm_process_mode); + switching_mm_mode = true; + combo_mm_process_mode.set_selected ( + ConfigManager.get_binary_process_mode () == ConfigManager.BINARY_MODE_TEXT ? 1 : 0); + switching_mm_mode = false; + combo_mm_process_mode.notify["selected"].connect (() => { + if (switching_mm_mode) return; + ConfigManager.save_binary_process_mode ( + combo_mm_process_mode.get_selected () == 1 + ? ConfigManager.BINARY_MODE_TEXT : ConfigManager.BINARY_MODE_VLM); + ai_settings_changed (); + }); + // ── VLM 模型配置方案下拉: 预置存于配置文件 vlm_models 数组, 双向同步 var mm_profile_row = new ComboRow (); mm_profile_row.set_title (_("Model Profile")); diff --git a/src/widgets/queue_list_factory.vala b/src/widgets/queue_list_factory.vala index f752818..8914f6b 100644 --- a/src/widgets/queue_list_factory.vala +++ b/src/widgets/queue_list_factory.vala @@ -206,11 +206,19 @@ public class QueueListFactory : GLib.Object { } }); + // 监听失败原因变化: FAILED 状态下行内显示简短原因 + ulong error_handler_id = data.notify["preprocess-error"].connect (() => { + if (list_item != null && list_item.get_item () != null) { + render_row (list_item, data, label, icon); + } + }); + // 将句柄 ID 与所监视的数据模型指针弱挂载到 ListItem 容器上, // 供 unbind 时双重校验安全剥离信号 list_item.set_data ("content_notify_id", handler_id); list_item.set_data ("status_notify_id", status_handler_id); list_item.set_data ("cache_notify_id", cache_handler_id); + list_item.set_data ("error_notify_id", error_handler_id); list_item.set_data ("position_notify_id", pos_handler); list_item.set_data ("monitored_data_ptr", data); }); @@ -222,6 +230,7 @@ public class QueueListFactory : GLib.Object { ulong handler_id = list_item.get_data ("content_notify_id"); ulong status_handler_id = list_item.get_data ("status_notify_id"); ulong cache_handler_id = list_item.get_data ("cache_notify_id"); + ulong error_handler_id = list_item.get_data ("error_notify_id"); ulong pos_handler = list_item.get_data ("position_notify_id"); var data = list_item.get_data ("monitored_data_ptr"); @@ -235,6 +244,9 @@ public class QueueListFactory : GLib.Object { if (cache_handler_id != 0 && data != null && GLib.SignalHandler.is_connected (data, cache_handler_id)) { GLib.SignalHandler.disconnect (data, cache_handler_id); } + if (error_handler_id != 0 && data != null && GLib.SignalHandler.is_connected (data, error_handler_id)) { + GLib.SignalHandler.disconnect (data, error_handler_id); + } if (pos_handler != 0 && GLib.SignalHandler.is_connected (list_item, pos_handler)) { GLib.SignalHandler.disconnect (list_item, pos_handler); } @@ -243,6 +255,7 @@ public class QueueListFactory : GLib.Object { list_item.set_data ("content_notify_id", null); list_item.set_data ("status_notify_id", null); list_item.set_data ("cache_notify_id", null); + list_item.set_data ("error_notify_id", null); list_item.set_data ("position_notify_id", null); list_item.set_data ("monitored_data_ptr", null); var unbind_box = list_item.get_child () as Gtk.Box; @@ -299,6 +312,12 @@ public class QueueListFactory : GLib.Object { break; case PreprocessStatus.FAILED: display_name += _(" [Conversion failed]"); + if (data.preprocess_error != null && data.preprocess_error.length > 0) { + // 队列行空间有限: 单行 + 截断, 完整原因在预览/tooltip + string reason = data.preprocess_error.replace ("\n", " ").strip (); + if (reason.length > 50) reason = reason.substring (0, 50) + "…"; + display_name += " " + reason; + } break; } } diff --git a/src/widgets/vlm_queue_controller.vala b/src/widgets/vlm_queue_controller.vala index a36783e..6b09cc5 100644 --- a/src/widgets/vlm_queue_controller.vala +++ b/src/widgets/vlm_queue_controller.vala @@ -102,7 +102,9 @@ public class VlmQueueController : GLib.Object { // ─── 内部 ────────────────────────────────────────────────────────── - private void on_task_completed (string file_path, string? md, PreprocessStatus status, bool from_cache) { + private void on_task_completed ( + string file_path, string? md, PreprocessStatus status, bool from_cache, string? error_message + ) { var items = app_state.items; for (int i = 0; i < items.size; i++) { var it = items.get (i); @@ -110,6 +112,8 @@ public class VlmQueueController : GLib.Object { if (md != null) it.preprocessed_content = md; it.preprocess_status = status; it.from_cache = from_cache; + // FAILED 时带用户可读原因; 其它状态清空旧原因防止残留 + it.preprocess_error = (status == PreprocessStatus.FAILED) ? error_message : null; // 合并刷新: 批量预处理完成时每个文件完成都会触发, 直接 refresh_list // 会形成 N 次 O(n) 扫描 + 末尾预览级联. 委托给 Window 合并到 150ms 窗口 // 只刷新一次. diff --git a/src/win32_console_shim.c b/src/win32_console_shim.c new file mode 100644 index 0000000..10342f4 --- /dev/null +++ b/src/win32_console_shim.c @@ -0,0 +1,24 @@ +#include "win32_console_shim.h" + +#include + +void win32_console_init (void) { + /* stdout 已有有效句柄时 (Git Bash / CI 的管道、文件重定向、已附加的 + * 控制台) 不做任何事, 保住管道输出。GUI 子系统直启时 std 句柄为空, + * GetFileType 返回 FILE_TYPE_UNKNOWN 并置 ERROR_INVALID_HANDLE。 */ + SetLastError (0); + DWORD out_type = GetFileType (GetStdHandle (STD_OUTPUT_HANDLE)); + if (!(out_type == FILE_TYPE_UNKNOWN && GetLastError () == ERROR_INVALID_HANDLE)) { + return; + } + + /* 父进程没有控制台 (双击 / start 启动) 时附加失败, 直接返回, GUI 照常 + * 运行; 有控制台 (cmd / PowerShell 跑 CLI) 时接管 std 流。 */ + if (!AttachConsole (ATTACH_PARENT_PROCESS)) { + return; + } + freopen ("CONOUT$", "w", stdout); + freopen ("CONOUT$", "w", stderr); + /* 程序输出恒为 UTF-8, 控制台默认码页 (中文系统 CP936) 下会乱码 */ + SetConsoleOutputCP (CP_UTF8); +} diff --git a/src/win32_console_shim.h b/src/win32_console_shim.h new file mode 100644 index 0000000..711f3f4 --- /dev/null +++ b/src/win32_console_shim.h @@ -0,0 +1,11 @@ +#ifndef FILECOLLECTOR_WIN32_CONSOLE_SHIM_H +#define FILECOLLECTOR_WIN32_CONSOLE_SHIM_H + +#include + +/* CLI 输出接管: exe 为 GUI 子系统 (GUI 启动不弹黑框), 在 cmd/PowerShell + * 里直接跑 CLI 时 stdout 无句柄, 输出完全不可见。此函数在检测到该场景时 + * 附加父进程控制台并接管 std 流。Git Bash / CI 等管道环境不做任何事。 */ +void win32_console_init (void); + +#endif diff --git a/src/window.vala b/src/window.vala index f85b989..31b86e0 100644 --- a/src/window.vala +++ b/src/window.vala @@ -2683,12 +2683,17 @@ public class FileCollectorWindow : Adw.ApplicationWindow { // 确保指定文件路径的所有父目录都已加载到树中 private void ensure_path_loaded (string abs_path) { - if (work_dir == null || !abs_path.has_prefix (work_dir.get_path () + "/")) return; + if (work_dir == null) return; + // Windows 上 get_path() 返回反斜杠路径, 归一为 "/" 后再做前缀比较 + // 与层级切分, 否则懒加载判断恒不命中 + string wd_path = (work_dir.get_path () ?? "").replace ("\\", "/"); + string norm = abs_path.replace ("\\", "/"); + if (!norm.has_prefix (wd_path + "/")) return; if (root_store.get_n_items () == 0) return; ensure_path_token++; uint my_token = ensure_path_token; - string rel = abs_path.substring (work_dir.get_path ().length + 1); + string rel = norm.substring (wd_path.length + 1); string[] parts = rel.split ("/"); var current = (DirectoryItem) root_store.get_item (0); @@ -3552,6 +3557,11 @@ public class FileCollectorWindow : Adw.ApplicationWindow { case PreprocessStatus.FAILED: show_action_bar = true; btn_retry_preprocess.tooltip_text = _("AI conversion failed\nClick to retry conversion"); + if (item.preprocess_error != null && item.preprocess_error.length > 0) { + string reason = item.preprocess_error.replace ("\n", " "); + if (reason.length > 120) reason = reason.substring (0, 120) + "…"; + btn_retry_preprocess.tooltip_text += "\n\n" + reason; + } break; case PreprocessStatus.PROCESSING: btn_retry_preprocess.tooltip_text = _("Processing..."); @@ -3585,7 +3595,15 @@ public class FileCollectorWindow : Adw.ApplicationWindow { apply_preview_content_full (item, _("Processing...")); break; case PreprocessStatus.FAILED: - apply_preview_content_full (item, _("[AI conversion failed; click the retry button on the toolbar to reconvert]")); + { + string placeholder = _("[AI conversion failed; click the retry button on the toolbar to reconvert]"); + if (item.preprocess_error != null && item.preprocess_error.length > 0) { + string reason = item.preprocess_error.replace ("\n", " "); + if (reason.length > 200) reason = reason.substring (0, 200) + "…"; + placeholder += "\n" + _("Reason: %s").printf (reason); + } + apply_preview_content_full (item, placeholder); + } break; case PreprocessStatus.CHECKING: apply_preview_content_full (item, _("[Checking cache...]")); @@ -3986,6 +4004,7 @@ public class FileCollectorWindow : Adw.ApplicationWindow { item.preprocessed_content = null; item.from_cache = false; + item.preprocess_error = null; item.preprocess_status = PreprocessStatus.PENDING; refresh_list (); @@ -4019,6 +4038,7 @@ public class FileCollectorWindow : Adw.ApplicationWindow { item.preprocess_status = PreprocessStatus.NONE; item.preprocessed_content = null; item.from_cache = false; + item.preprocess_error = null; } } @@ -4057,9 +4077,11 @@ public class FileCollectorWindow : Adw.ApplicationWindow { if (ctx_item == null || ctx_item.file_path == null) return; string path_to_copy = ctx_item.file_path; if (!ctx_item.force_absolute && work_dir != null && !use_absolute) { - string wd = work_dir.get_path () + "/"; - if (path_to_copy.has_prefix (wd)) { - path_to_copy = path_to_copy.substring (wd.length); + // 归一分隔符: Windows 反斜杠路径对 wd + "/" 前缀恒不命中 + string wd = (work_dir.get_path () ?? "").replace ("\\", "/") + "/"; + string norm = path_to_copy.replace ("\\", "/"); + if (norm.has_prefix (wd)) { + path_to_copy = norm.substring (wd.length); } } get_clipboard ().set_text (path_to_copy); @@ -4149,9 +4171,10 @@ public class FileCollectorWindow : Adw.ApplicationWindow { if (ctx_tree_item == null) return; string path_to_copy = ctx_tree_item.path; if (work_dir != null) { - string wd = work_dir.get_path () + "/"; - if (path_to_copy.has_prefix (wd)) { - path_to_copy = path_to_copy.substring (wd.length); + string wd = (work_dir.get_path () ?? "").replace ("\\", "/") + "/"; + string norm = path_to_copy.replace ("\\", "/"); + if (norm.has_prefix (wd)) { + path_to_copy = norm.substring (wd.length); } } get_clipboard ().set_text (path_to_copy); @@ -4329,9 +4352,10 @@ public class FileCollectorWindow : Adw.ApplicationWindow { if (ctx_preview_item == null || ctx_preview_item.file_path == null) return; string path_to_copy = ctx_preview_item.file_path; if (work_dir != null) { - string wd = work_dir.get_path () + "/"; - if (path_to_copy.has_prefix (wd)) { - path_to_copy = path_to_copy.substring (wd.length); + string wd = (work_dir.get_path () ?? "").replace ("\\", "/") + "/"; + string norm = path_to_copy.replace ("\\", "/"); + if (norm.has_prefix (wd)) { + path_to_copy = norm.substring (wd.length); } } get_clipboard ().set_text (path_to_copy); @@ -5100,8 +5124,10 @@ public class FileCollectorWindow : Adw.ApplicationWindow { private string get_display_path (ItemData data) { if (data.force_absolute || use_absolute || work_dir == null) return data.file_path; - var wd_path = work_dir.get_path () + "/"; - if (data.file_path.has_prefix (wd_path)) return data.file_path.substring (wd_path.length); + // 归一分隔符, Windows 反斜杠路径下前缀判断恒不命中 + var wd_path = (work_dir.get_path () ?? "").replace ("\\", "/") + "/"; + var norm = data.file_path.replace ("\\", "/"); + if (norm.has_prefix (wd_path)) return norm.substring (wd_path.length); return data.file_path; } diff --git a/tests/meson.build b/tests/meson.build index 9990422..3b5d2e5 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -170,3 +170,16 @@ test_search_query_parser_exe = executable('test_search_query_parser', dependencies: [glib_dep, gio_dep, gee_dep], ) test('test_search_query_parser', test_search_query_parser_exe) + +# ─── 12. OfficeTextExtractor (纯 Vala ZIP/XML 解析; PDF 分支依赖外部 +# pdftotext, 属集成验证不在此覆盖). poppler 工具解析位于 platform +# (无 GDK 依赖), 因此测试闭包不含 binary_converter / gtk4. +office_text_extractor_src = files(join_paths(src_dir, 'services', 'office_text_extractor.vala')) +platform_src = files(join_paths(src_dir, 'utils', 'platform.vala')) +test_office_text_extractor_exe = executable('test_office_text_extractor', + 'test_office_text_extractor.vala', + office_text_extractor_src, + platform_src, + dependencies: [glib_dep, gio_dep, gee_dep], +) +test('test_office_text_extractor', test_office_text_extractor_exe) diff --git a/tests/test_office_text_extractor.vala b/tests/test_office_text_extractor.vala new file mode 100644 index 0000000..d98ed71 --- /dev/null +++ b/tests/test_office_text_extractor.vala @@ -0,0 +1,323 @@ +// OfficeTextExtractor 单元测试: 内存构造 ZIP fixture, 覆盖 +// STORED/DEFLATE 两种压缩、docx/xlsx/pptx/odt 抽取、实体解码、 +// 工作表/幻灯片自然排序、EOCD 注释容错、ZIP64/加密/垃圾数据的明确报错。 +// PDF 分支依赖外部 pdftotext, 不在此覆盖 (属于集成验证)。 +// +// fixture 侧的 ZIP 写入器是测试专用的最小实现: 只写 reader 关心的字段, +// CRC 一律置 0 (OfficeTextExtractor.ZipReader 不校验 CRC)。 + +using Gee; + +static int failures = 0; + +static void check (bool cond, string what) { + if (cond) { + print ("PASS: %s\n", what); + } else { + print ("FAIL: %s\n", what); + failures++; + } +} + +delegate void TrialFunc () throws Error; + +static void run (string name, TrialFunc f) { + try { + f (); + } catch (Error e) { + print ("FAIL: %s (unexpected throw: %s)\n", name, e.message); + failures++; + } +} + +// ─── ZIP fixture 写入器 ────────────────────────────────────────────── + +class ZipBuilder : GLib.Object { + // vala 0.56 的 glib vapi 没有 GLib.BytesBuilder: 用 GLib.ByteArray 累积 + private GLib.ByteArray body = new GLib.ByteArray (); + private uint32 written = 0; + private Gee.ArrayList entries = new Gee.ArrayList (); + + private class EntryMeta : GLib.Object { + public string name; + public uint16 flags; + public uint16 method; + public uint32 comp_size; + public uint32 uncomp_size; + public uint32 offset; + } + + public void add_stored (string name, uint8[] content, uint16 flags = 0) { + add_entry_raw (name, content, content.length, 0, flags); + } + + public void add_deflated (string name, uint8[] content) throws Error { + uint8[] comp = deflate_bytes (content); + add_entry_raw (name, comp, content.length, 8, 0); + } + + private void add_entry_raw (string name, uint8[] stored_data, + uint32 uncomp_size, uint16 method, uint16 flags) { + var meta = new EntryMeta (); + meta.name = name; + meta.flags = flags; + meta.method = method; + meta.comp_size = (uint32) stored_data.length; + meta.uncomp_size = uncomp_size; + meta.offset = written; + entries.add (meta); + + uint8[] nb = name.data; + u32 (0x04034b50u); + u16 (20); // version needed + u16 (flags); + u16 (method); + u16 (0); u16 (0); // time, date + u32 (0); // crc (reader 不校验) + u32 (meta.comp_size); + u32 (uncomp_size); + u16 ((uint16) nb.length); + u16 (0); // extra len + push (nb); + push (stored_data); + } + + public uint8[] finish () { + uint32 cd_offset = written; + foreach (var m in entries) { + uint8[] nb = m.name.data; + u32 (0x02014b50u); + u16 (20); u16 (20); // version made by / needed + u16 (m.flags); + u16 (m.method); + u16 (0); u16 (0); + u32 (0); + u32 (m.comp_size); + u32 (m.uncomp_size); + u16 ((uint16) nb.length); + u16 (0); // extra + u16 (0); // comment + u16 (0); // disk start + u16 (0); // internal attrs + u32 (0); // external attrs + u32 (m.offset); + push (nb); + } + uint32 cd_size = written - cd_offset; + u32 (0x06054b50u); + u16 (0); u16 (0); // disk numbers + u16 ((uint16) entries.size); + u16 ((uint16) entries.size); + u32 (cd_size); + u32 (cd_offset); + u16 (0); // comment len + return body.steal (); + } + + private void push (uint8[] piece) { + body.append (piece); + written += (uint32) piece.length; + } + + private void u16 (uint16 v) { + push (new uint8[] { (uint8) (v & 0xFF), (uint8) ((v >> 8) & 0xFF) }); + } + + private void u32 (uint32 v) { + push (new uint8[] { + (uint8) (v & 0xFF), (uint8) ((v >> 8) & 0xFF), + (uint8) ((v >> 16) & 0xFF), (uint8) ((v >> 24) & 0xFF) + }); + } + + private static uint8[] deflate_bytes (uint8[] input) throws Error { + var comp = new GLib.ZlibCompressor (GLib.ZlibCompressorFormat.RAW); + var out_acc = new GLib.ByteArray (); + uint8[] out_buf = new uint8[input.length * 2 + 4096]; + size_t bytes_read; + size_t bytes_written; + // 第一次: 喂入全部输入 + comp.convert (input, out_buf, GLib.ConverterFlags.NONE, out bytes_read, out bytes_written); + out_acc.append (out_buf[0 : (int) bytes_written]); + // INPUT_AT_END 告知输入结束 -> zlib 执行 Z_FINISH 输出终止流并返回 FINISHED + // (FLUSH 只是同步刷新内部缓冲, 不会终止 deflate 流) + bool finished = false; + for (int i = 0; i < 4 && !finished; i++) { + var res = comp.convert (new uint8[0], out_buf, GLib.ConverterFlags.INPUT_AT_END, + out bytes_read, out bytes_written); + out_acc.append (out_buf[0 : (int) bytes_written]); + finished = (res == GLib.ConverterResult.FINISHED); + } + if (!finished) { + error ("fixture deflate not finished after flush"); + } + return out_acc.steal (); + } +} + +// ─── 工具 ──────────────────────────────────────────────────────────── + +static string write_fixture (string name, uint8[] data) throws Error { + string dir = DirUtils.make_tmp ("fc_ext_test_XXXXXX"); + string path = GLib.Path.build_filename (dir, name); + FileUtils.set_data (path, data); + return path; +} + +// ─── 测试用例 ──────────────────────────────────────────────────────── + +static void test_supports () { + check (OfficeTextExtractor.supports (".docx"), "supports .docx"); + check (OfficeTextExtractor.supports (".pdf"), "supports .pdf"); + check (OfficeTextExtractor.supports (".odp"), "supports .odp"); + check (!OfficeTextExtractor.supports (".doc"), "rejects legacy .doc"); + check (!OfficeTextExtractor.supports (".txt"), "rejects .txt"); +} + +static void test_docx_stored () throws Error { + var xml = """ +标题:项目说明 +带有 & 符号与 <标签> 的文本 +单元格A单元格B +结尾 +"""; + var builder = new ZipBuilder (); + builder.add_stored ("word/document.xml", xml.data); + builder.add_stored ("[Content_Types].xml", "".data); + string path = write_fixture ("t.docx", builder.finish ()); + string md = OfficeTextExtractor.extract_markdown (path); + check (md.contains ("标题:项目说明"), "docx: 中文段落提取"); + check (md.contains ("带有 & 符号与 <标签> 的文本"), "docx: 实体解码"); + check (md.contains ("单元格A\t单元格B"), "docx: 表格单元格按 \\t 分隔"); + check (md.contains ("结尾"), "docx: 尾段保留"); +} + +static void test_docx_deflate () throws Error { + var xml = "压缩条目正文"; + var builder = new ZipBuilder (); + builder.add_deflated ("word/document.xml", xml.data); + string path = write_fixture ("t2.docx", builder.finish ()); + string md = OfficeTextExtractor.extract_markdown (path); + check (md.contains ("压缩条目正文"), "docx: DEFLATE 条目可解压"); +} + +static void test_xlsx () throws Error { + var sst = """名称数值列第二张表最后一张"""; + var sheet1 = """ +01 +423.14 +"""; + var sheet2 = "2"; + var sheet10 = "3"; + var builder = new ZipBuilder (); + // 故意乱序加入, 验证 sheet1/sheet2/sheet10 按数字自然排序 + builder.add_stored ("xl/worksheets/sheet10.xml", sheet10.data); + builder.add_stored ("xl/worksheets/sheet2.xml", sheet2.data); + builder.add_stored ("xl/worksheets/sheet1.xml", sheet1.data); + builder.add_stored ("xl/sharedStrings.xml", sst.data); + string path = write_fixture ("t.xlsx", builder.finish ()); + string md = OfficeTextExtractor.extract_markdown (path); + check (md.contains ("名称\t数值列"), "xlsx: sharedStrings 引用解析"); + check (md.contains ("42\t3.14"), "xlsx: 数值单元格原样输出"); + check (md.contains ("") == false, "xlsx: rowBreaks 未被误当成 row"); + check (md.contains ("第二张表"), "xlsx: 第二张表存在"); + int pos_sheet2 = md.index_of ("第二张表"); + int pos_sheet10 = md.index_of ("最后一张"); + check (pos_sheet2 >= 0 && pos_sheet10 > pos_sheet2, "xlsx: sheet2 在 sheet10 之前 (自然排序)"); +} + +static void test_pptx () throws Error { + var s1 = "第一页要点副要点"; + var s2 = "第二页内容"; + var s10 = "第十页内容"; + var builder = new ZipBuilder (); + builder.add_stored ("ppt/slides/slide10.xml", s10.data); + builder.add_stored ("ppt/slides/slide1.xml", s1.data); + builder.add_stored ("ppt/slides/slide2.xml", s2.data); + string path = write_fixture ("t.pptx", builder.finish ()); + string md = OfficeTextExtractor.extract_markdown (path); + check (md.contains ("## Slide 1") && md.contains ("第一页要点"), "pptx: 首页提取"); + check (md.contains ("副要点"), "pptx: 段落边界换行"); + int pos2 = md.index_of ("第二页内容"); + int pos10 = md.index_of ("第十页内容"); + check (pos2 >= 0 && pos10 > pos2, "pptx: slide2 在 slide10 之前 (自然排序)"); +} + +static void test_odt () throws Error { + var xml = """ +一级标题段落一 +段落二续行 +"""; + var builder = new ZipBuilder (); + builder.add_stored ("content.xml", xml.data); + string path = write_fixture ("t.odt", builder.finish ()); + string md = OfficeTextExtractor.extract_markdown (path); + check (md.contains ("一级标题"), "odt: 标题提取"); + check (md.contains ("段落二\n续行"), "odt: line-break 处理"); +} + +static void test_eocd_trailing_comment () throws Error { + var builder = new ZipBuilder (); + builder.add_stored ("word/document.xml", + "注释容错".data); + uint8[] zip = builder.finish (); + // 模拟 ZIP 尾部注释: EOCD 之后附带任意字节 + uint8[] garbage = "this is a zip comment, not part of the archive".data; + uint8[] padded = new uint8[zip.length + garbage.length]; + for (int i = 0; i < zip.length; i++) padded[i] = zip[i]; + for (int i = 0; i < garbage.length; i++) padded[zip.length + i] = garbage[i]; + string path = write_fixture ("t3.docx", padded); + string md = OfficeTextExtractor.extract_markdown (path); + check (md.contains ("注释容错"), "docx: EOCD 尾部注释容错"); +} + +static void expect_not_supported (string name, uint8[] data) throws Error { + string path = write_fixture (name, data); + bool threw = false; + try { + OfficeTextExtractor.extract_markdown (path); + } catch (Error e) { + threw = (e is IOError.NOT_SUPPORTED); + } + check (threw, name + " → NOT_SUPPORTED"); +} + +static void test_error_paths () throws Error { + // 完全不是 ZIP 的垃圾数据 + expect_not_supported ("garbage.docx", "definitely not a zip archive........".data); + // ZIP64: EOCD 中 cd_offset = 0xFFFFFFFF + expect_not_supported ("zip64.xlsx", new uint8[] { + 0x50, 0x4b, 0x05, 0x06, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x01, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00 + }); + // 加密条目: central directory flags bit0 = 1 + var builder = new ZipBuilder (); + builder.add_stored ("word/document.xml", "".data, 0x0001); + expect_not_supported ("enc.docx", builder.finish ()); +} + +static void test_txt_rejected () throws Error { + string path = write_fixture ("t.txt", "plain text".data); + bool threw = false; + try { + OfficeTextExtractor.extract_markdown (path); + } catch (Error e) { + threw = (e is IOError.NOT_SUPPORTED); + } + check (threw, "txt → NOT_SUPPORTED (模式外格式由调用方门禁拦截)"); +} + +int main () { + run ("supports", () => { test_supports (); }); + run ("docx stored", () => { test_docx_stored (); }); + run ("docx deflate", () => { test_docx_deflate (); }); + run ("xlsx", () => { test_xlsx (); }); + run ("pptx", () => { test_pptx (); }); + run ("odt", () => { test_odt (); }); + run ("eocd comment", () => { test_eocd_trailing_comment (); }); + run ("error paths", () => { test_error_paths (); }); + run ("txt rejected", () => { test_txt_rejected (); }); + + print ("\n%s\n", failures == 0 ? "ALL TESTS PASSED" : "%d TEST(S) FAILED".printf (failures)); + return failures == 0 ? 0 : 1; +} diff --git a/tests/test_vlm_queue.vala b/tests/test_vlm_queue.vala index 13cd208..aff9e3e 100644 --- a/tests/test_vlm_queue.vala +++ b/tests/test_vlm_queue.vala @@ -7,7 +7,7 @@ // // 注意: 真实 executor 模式 (参考 vlm_task_runner.vala): // 1. 在 worker 线程上执行实际工作 -// 2. 通过 Idle.add 在主线程上发射 manager.task_completed (path, md, status, from_cache) +// 2. 通过 Idle.add 在主线程上发射 manager.task_completed (path, md, status, from_cache, error) // 3. 调用 manager.notify_finished (path) 释放并发槽 // 测试用 executor 必须遵循同样模式。 @@ -43,7 +43,7 @@ void pump_loop (uint ms = 200) { // 模拟立即完成的 executor: 在 Idle 上发 task_completed, 然后释放槽 void immediate_executor (string path, VLMQueueManager m) { Idle.add (() => { - m.task_completed (path, "md-for-" + path, PreprocessStatus.COMPLETED, false); + m.task_completed (path, "md-for-" + path, PreprocessStatus.COMPLETED, false, null); return Source.REMOVE; }); m.notify_finished (path); @@ -116,7 +116,7 @@ void test_max_concurrency () { lock.unlock (); Idle.add (() => { - m.task_completed (path, "md", PreprocessStatus.COMPLETED, false); + m.task_completed (path, "md", PreprocessStatus.COMPLETED, false, null); return Source.REMOVE; }); m.notify_finished (path); @@ -146,7 +146,7 @@ void test_wait_for_completion () { mgr.executor = (path, m) => { Thread.usleep (100 * 1000); // 100ms Idle.add (() => { - m.task_completed (path, "md", PreprocessStatus.COMPLETED, false); + m.task_completed (path, "md", PreprocessStatus.COMPLETED, false, null); return Source.REMOVE; }); m.notify_finished (path); @@ -190,7 +190,7 @@ void test_wait_for_completion_timeout () { Thread.usleep (2000 * 1000); // 2 秒 // 测试结束后才通知, 避免线程泄漏 Idle.add (() => { - m.task_completed (path, "md", PreprocessStatus.COMPLETED, false); + m.task_completed (path, "md", PreprocessStatus.COMPLETED, false, null); return Source.REMOVE; }); m.notify_finished (path); @@ -227,7 +227,7 @@ void test_cancel_clears_pending () { lock.unlock (); Thread.usleep (100 * 1000); Idle.add (() => { - m.task_completed (path, "md", PreprocessStatus.COMPLETED, false); + m.task_completed (path, "md", PreprocessStatus.COMPLETED, false, null); return Source.REMOVE; }); m.notify_finished (path); @@ -258,7 +258,7 @@ void test_pause_resume () { mgr.executor = (path, m) => { Thread.usleep (20 * 1000); Idle.add (() => { - m.task_completed (path, "md", PreprocessStatus.COMPLETED, false); + m.task_completed (path, "md", PreprocessStatus.COMPLETED, false, null); return Source.REMOVE; }); m.notify_finished (path); @@ -313,7 +313,7 @@ void test_bg_threads_cleanup () { mgr.executor = (path, m) => { Thread.usleep (30 * 1000); Idle.add (() => { - m.task_completed (path, "md", PreprocessStatus.COMPLETED, false); + m.task_completed (path, "md", PreprocessStatus.COMPLETED, false, null); return Source.REMOVE; }); m.notify_finished (path); diff --git a/tools/build-converter-package.sh b/tools/build-converter-package.sh new file mode 100644 index 0000000..f22f58e --- /dev/null +++ b/tools/build-converter-package.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Build the OPTIONAL Windows LibreOffice converter package. +# +# 主包不捆绑 LibreOffice (体积原因)。Office 文档视觉转换链路 +# (soffice → PDF → pdftoppm → VLM) 依赖 BinaryConverter.resolve_soffice () +# 按以下顺序探测: +# 1. <应用包根>/tools/LibreOffice/program/soffice.exe ← 本转换器包的布局 +# 2. C:\Program Files\LibreOffice\program\soffice.exe ← 官方安装器默认目录 +# 3. PATH 中的 soffice ← Linux/macOS 现状 +# +# 本脚本下载官方 LibreOffice MSI, 用 msiexec 管理员解包 (不写注册表、 +# 不进 Program Files), 把程序树按 tools/LibreOffice/... 布局打成独立 ZIP。 +# 用户解压到 FileCollector 便携包根目录 (与 bin/ 同级) 即生效。 +# +# 在 MSYS2 UCRT64 终端执行 (需要 curl/zip/unzip; msiexec 由 Windows 提供)。 +set -euo pipefail + +readonly DIST_DIR="dist" +readonly WORK_DIR="converter-work" +# 固定 LibreOffice "still" 分支版本; 升级时同步更新, 并确认 URL 有效: +# https://download.documentfoundation.org/libreoffice/stable/ +readonly LO_VERSION="25.8.4" +readonly LO_URL="https://download.documentfoundation.org/libreoffice/stable/${LO_VERSION}/win/x86_64/LibreOffice_${LO_VERSION}_Win_x86-64.msi" + +rm -rf "${WORK_DIR}" +mkdir -p "${DIST_DIR}" "${WORK_DIR}/unpack" "${WORK_DIR}/staging/tools" + +echo "Downloading LibreOffice ${LO_VERSION} MSI..." +curl --fail --location --retry 3 --output "${WORK_DIR}/LibreOffice.msi" "${LO_URL}" + +MSI_WIN="$(cygpath -w "${PWD}/${WORK_DIR}/LibreOffice.msi")" +TARGET_WIN="$(cygpath -w "${PWD}/${WORK_DIR}/unpack")" + +echo "Extracting MSI (msiexec /a, administrative extract, no install)..." +# msiexec 的 /a /qn 参数不能直接从 bash 传入: MSYS2 会把 "/a" 当 POSIX 路径 +# 改写成 MSYS2 安装根路径 (与 meson --prefix=/usr 同一类陷阱), 借 cmd 复合 +# 字符串绕过参数转换。 +cmd //c "msiexec /a \"${MSI_WIN}\" /qn TARGETDIR=\"${TARGET_WIN}\"" + +# msi 解包的嵌套层级因版本而异, 按 soffice.exe 实际位置反推 LibreOffice 根 +SOFFICE="$(find "${WORK_DIR}/unpack" -name soffice.exe -print -quit)" +test -n "${SOFFICE}" +LO_ROOT="$(dirname "$(dirname "${SOFFICE}")")" # .../LibreOffice/program/soffice.exe → .../LibreOffice +test -f "${LO_ROOT}/program/soffice.exe" +test -d "${LO_ROOT}/share" + +cp -r "${LO_ROOT}" "${WORK_DIR}/staging/tools/LibreOffice" +test -f "${WORK_DIR}/staging/tools/LibreOffice/program/soffice.exe" +test -f "${WORK_DIR}/staging/tools/LibreOffice/program/soffice.bin" + +PACKAGE_PATH="${DIST_DIR}/filecollector-converter-windows-x64-${LO_VERSION}.zip" +( + cd "${WORK_DIR}/staging" + zip -r "../../${PACKAGE_PATH}" tools +) +unzip -l "${PACKAGE_PATH}" | grep -Eq 'tools/LibreOffice/program/soffice\.exe' +unzip -l "${PACKAGE_PATH}" | grep -Eq 'tools/LibreOffice/program/soffice\.bin' +sha256sum "${PACKAGE_PATH}" > "${PACKAGE_PATH}.sha256" + +du -h "${PACKAGE_PATH}" +echo "Built ${PACKAGE_PATH}" +echo "Usage: extract into the FileCollector portable package root (alongside bin/), keeping the tools/ layout." diff --git a/tools/build-windows-package.sh b/tools/build-windows-package.sh index f0108a5..7dc53ac 100755 --- a/tools/build-windows-package.sh +++ b/tools/build-windows-package.sh @@ -21,6 +21,12 @@ fi MSYS_PREFIX="${MSYSTEM_PREFIX:-/ucrt64}" export PYTHONUTF8=1 export LANG=C.UTF-8 +# meson --prefix=/usr 会被 MSYS2 参数转换改写为 MSYS2 安装根下的真实路径 +# (实测 --prefix=/usr → --prefix=D://usr), locale 因此被装进 +# install-windows/a/_temp/msys64/usr/... 错误嵌套路径, 打包时静默丢掉全部 +# 翻译 (v4.9.1 及之前的 Windows 包均为纯英文界面)。排除这两类参数; +# cmark 的 -DCMAKE_INSTALL_PREFIX=/ucrt64 指向真实环境根, 不在排除之列。 +export MSYS2_ARG_CONV_EXCL="/usr;--prefix" export PKG_CONFIG_PATH="${MSYS_PREFIX}/lib/pkgconfig:${PKG_CONFIG_PATH:-}" export CPATH="${MSYS_PREFIX}/include:${CPATH:-}" export LIBRARY_PATH="${MSYS_PREFIX}/lib:${LIBRARY_PATH:-}" @@ -64,6 +70,9 @@ cp -r data "${STAGING_DIR}/share/data" if [[ -d "${INSTALL_ROOT}/usr/share/locale" ]]; then cp -r "${INSTALL_ROOT}/usr/share/locale" "${STAGING_DIR}/locale" fi +# 翻译缺失曾静默出包 (v4.9.1 及之前的 Windows 包均为纯英文界面, 见上方 +# MSYS2_ARG_CONV_EXCL 注释), 此处硬性卡关: .mo 不在 staging 直接构建失败。 +test -f "${STAGING_DIR}/locale/zh_CN/LC_MESSAGES/filecollector.mo" # Dynamically loaded GTK/GIO components are not visible to ldd and must be staged explicitly. cp ${MSYS_PREFIX}/lib/gdk-pixbuf-2.0/2.10.0/loaders/*.dll \ @@ -90,6 +99,44 @@ python3 tools/verify_gnome_icon_theme.py "${STAGING_DIR}/share/icons" cp ${MSYS_PREFIX}/lib/gio/modules/libgiognutls.dll "${STAGING_DIR}/lib/gio/modules/" python3 tools/collect_dlls.py "${STAGING_DIR}/lib/gio/modules/libgiognutls.dll" "${STAGING_DIR}/bin" +# zip_exporter 通过外部 zip 命令打包, 普通用户机器没有 Info-ZIP, 便携包 +# 必须自带。MSYS2 的 zip 位于 msys 层 (/usr/bin), 依赖 msys-2.0.dll; +# 两者都要进 bin/ (CreateProcess 搜索调用方 exe 所在目录优先于 PATH, +# zip_exporter 的裸 "zip" 命令无需改 PATH 即可命中)。 +test -f /usr/bin/zip.exe +test -f /usr/bin/msys-2.0.dll +cp /usr/bin/zip.exe "${STAGING_DIR}/bin/zip.exe" +cp /usr/bin/msys-2.0.dll "${STAGING_DIR}/bin/msys-2.0.dll" + +# 二进制预处理 (视觉转换模式) 需要 poppler 工具: pdftoppm 把 PDF 渲染成页图 +# 送 VLM, pdftotext 供 "文本抽取" 模式抽纯文本。其依赖 (cairo/freetype/ +# fontconfig/jpeg/png/tiff) GTK 已带, collect_dlls.py 递归收集剩余部分。 +test -f "${MSYS_PREFIX}/bin/pdftoppm.exe" +test -f "${MSYS_PREFIX}/bin/pdftotext.exe" +cp "${MSYS_PREFIX}/bin/pdftoppm.exe" "${STAGING_DIR}/bin/pdftoppm.exe" +cp "${MSYS_PREFIX}/bin/pdftotext.exe" "${STAGING_DIR}/bin/pdftotext.exe" +python3 tools/collect_dlls.py \ + "${STAGING_DIR}/bin/pdftoppm.exe" "${STAGING_DIR}/bin/pdftotext.exe" \ + "${STAGING_DIR}/bin" + +# poppler 数据文件 (CJK CID 字体 CMap 等): 中文 PDF 没有它, pdftoppm/pdftotext +# 会输出乱码或空文本。优先取 MSYS2 环境内的 share/poppler; 没有则下载上游 +# poppler-data 补齐。 +if [[ -d "${MSYS_PREFIX}/share/poppler" ]]; then + cp -r "${MSYS_PREFIX}/share/poppler" "${STAGING_DIR}/share/poppler" +else + readonly POPPLER_DATA_VERSION="0.4.12" + curl --fail --location --retry 3 --output /tmp/poppler-data.tar.gz \ + "https://poppler.freedesktop.org/releases/poppler-data-${POPPLER_DATA_VERSION}.tar.gz" + tar -xzf /tmp/poppler-data.tar.gz -C /tmp + mkdir -p "${STAGING_DIR}/share/poppler" + # 上游 tarball 顶层即 cidToUnicode/ cMap/ unicodeMap/ 三目录 (无 share/ 嵌套) + for d in cidToUnicode cMap unicodeMap; do + cp -r "/tmp/poppler-data-${POPPLER_DATA_VERSION}/${d}" "${STAGING_DIR}/share/poppler/${d}" + done +fi +test -d "${STAGING_DIR}/share/poppler/cidToUnicode" + cat > "${STAGING_DIR}/bin/filecollector-launch.bat" <<'BAT' @echo off set "GDK_PIXBUF_MODULEDIR=%~dp0..\lib\gdk-pixbuf-2.0\2.10.0\loaders" @@ -109,6 +156,10 @@ unzip -l "${PACKAGE_PATH}" | grep -Eq 'bin/filecollector\.exe' unzip -l "${PACKAGE_PATH}" | grep -Eq 'bin/filecollector-launch\.bat' unzip -l "${PACKAGE_PATH}" | grep -Eq 'lib/gio/modules/libgiognutls\.dll' unzip -l "${PACKAGE_PATH}" | grep -Eq 'share/glib-2\.0/schemas/gschemas\.compiled' +unzip -l "${PACKAGE_PATH}" | grep -Eq 'locale/zh_CN/LC_MESSAGES/filecollector\.mo' +unzip -l "${PACKAGE_PATH}" | grep -Eq 'bin/zip\.exe' +unzip -l "${PACKAGE_PATH}" | grep -Eq 'bin/pdftoppm\.exe' +unzip -l "${PACKAGE_PATH}" | grep -Eq 'bin/pdftotext\.exe' # 再校验 ZIP 本体, 防止打包环节漏拷文件。 python3 tools/verify_windows_package.py "${PACKAGE_PATH}" sha256sum "${PACKAGE_PATH}" > "${PACKAGE_PATH}.sha256" diff --git a/vala-lint.conf b/vala-lint.conf index 3ea6aca..47f2d4f 100644 --- a/vala-lint.conf +++ b/vala-lint.conf @@ -43,7 +43,7 @@ ignore-comments=true [naming-convention] # 全大写标识符 (Vala 常量约定) 不算违反 naming-convention -exceptions=MAX_FILE_CONTENT_SIZE,MAX_FILE_SIZE,PEEK_SIZE,HICOLOR_RESOURCE_PREFIX,DOCUMENT_EXTENSIONS,IMAGE_EXTENSIONS,SNAPSHOT_ICON_CHOICES +exceptions=MAX_FILE_CONTENT_SIZE,MAX_FILE_SIZE,MAX_TEXT_SIZE,PEEK_SIZE,HICOLOR_RESOURCE_PREFIX,DOCUMENT_EXTENSIONS,IMAGE_EXTENSIONS,SNAPSHOT_ICON_CHOICES,SUPPORTED_EXTS,BINARY_MODE_VLM,BINARY_MODE_TEXT [note] keywords=TODO,FIXME,XXX,HACK