From 4cd96edddf09b60ad0ca7d4d20067217eaa36ffe Mon Sep 17 00:00:00 2001 From: hy Date: Thu, 16 Jul 2026 17:41:54 +0800 Subject: [PATCH] feat(ai): add PDF reading skill --- doc/test-plan/ai-test.md | 80 +++- electron-builder.yml | 2 + electron.vite.config.ts | 1 + package.json | 1 + pnpm-lock.yaml | 4 +- scripts/verify-runtime-bundles.mjs | 6 + src/main/ipc/handlers.ts | 10 +- src/main/services/ai/human/constants.ts | 8 +- .../ai/human/node/learningNode/skillLoader.ts | 1 + .../ai/human/shared/historyRenderers.ts | 81 ++++ .../ai/human/shared/toolResultRenderers.ts | 105 +++++ src/main/services/ai/human/skills.ts | 11 + src/main/services/ai/human/tools/index.ts | 2 + src/main/services/ai/human/tools/pdf/index.ts | 26 ++ .../ai/human/tools/pdf/outputSchemas.ts | 86 ++++ .../services/ai/human/tools/pdf/runtime.ts | 214 ++++++++++ .../services/ai/human/tools/pdf/schemas.ts | 41 ++ .../services/ai/human/tools/pdf/toolSpecs.ts | 183 +++++++++ .../ai/human/tools/pdf/tools/readTools.ts | 279 +++++++++++++ src/main/services/ai/human/tools/pdf/types.ts | 18 + src/main/services/ai/human/tools/registry.ts | 2 + src/main/services/ai/human/types.ts | 2 + .../services/assets/WorkspaceAssetService.ts | 63 +++ .../services/attachments/AttachmentService.ts | 51 +-- src/main/services/pdf/textNormalizer.ts | 126 ++++++ src/main/services/pdf/types.ts | 58 +++ src/main/services/pdf/worker/pdfWorker.ts | 380 ++++++++++++++++++ .../services/pdf/worker/pdfWorkerClient.ts | 103 +++++ src/main/services/pdf/worker/types.ts | 92 +++++ .../workbench/WorkbenchComposer.tsx | 2 +- .../src/components/workbench/types.ts | 5 + .../src/components/workbench/utils.ts | 35 +- src/shared/types.ts | 1 + .../human-pdf-visual-reading-live.test.ts | 180 +++++++++ ...er-image-recognition-collaboration.test.ts | 207 ++++++++++ .../ai/tool/pdf/pdf-read-contract.test.ts | 282 +++++++++++++ .../unit/ai/human-memory-renderers.test.ts | 73 ++++ .../ai/human-tool-result-renderers.test.ts | 47 +++ tests/case/unit/ai/pdf-tool-contract.test.ts | 43 ++ .../unit/ai/runtime-skill-contract.test.ts | 1 + .../unit/workbench/attachment-service.test.ts | 15 + tests/harness/backend/aiLiveTestData.ts | 24 ++ tests/harness/backend/aiToolHarness.ts | 41 +- tests/harness/backend/humanHarness.ts | 13 +- tests/harness/backend/llmConfig.ts | 33 ++ tests/harness/backend/realAiDepsFactory.ts | 1 + tests/harness/backend/test-types.ts | 4 + 47 files changed, 2969 insertions(+), 74 deletions(-) create mode 100644 src/main/services/ai/human/tools/pdf/index.ts create mode 100644 src/main/services/ai/human/tools/pdf/outputSchemas.ts create mode 100644 src/main/services/ai/human/tools/pdf/runtime.ts create mode 100644 src/main/services/ai/human/tools/pdf/schemas.ts create mode 100644 src/main/services/ai/human/tools/pdf/toolSpecs.ts create mode 100644 src/main/services/ai/human/tools/pdf/tools/readTools.ts create mode 100644 src/main/services/ai/human/tools/pdf/types.ts create mode 100644 src/main/services/assets/WorkspaceAssetService.ts create mode 100644 src/main/services/pdf/textNormalizer.ts create mode 100644 src/main/services/pdf/types.ts create mode 100644 src/main/services/pdf/worker/pdfWorker.ts create mode 100644 src/main/services/pdf/worker/pdfWorkerClient.ts create mode 100644 src/main/services/pdf/worker/types.ts create mode 100644 tests/case/ai/human/live/human-pdf-visual-reading-live.test.ts create mode 100644 tests/case/ai/human/mock-llm/human-pdf-render-image-recognition-collaboration.test.ts create mode 100644 tests/case/ai/tool/pdf/pdf-read-contract.test.ts create mode 100644 tests/case/unit/ai/pdf-tool-contract.test.ts diff --git a/doc/test-plan/ai-test.md b/doc/test-plan/ai-test.md index bba6d9b0..155f550c 100644 --- a/doc/test-plan/ai-test.md +++ b/doc/test-plan/ai-test.md @@ -12,47 +12,52 @@ ### 2.1 按 Suite / 分类汇总 -- `tool` 完整回归：`67 / 67` - - 检测时间：`2026-07-14 13:59:28 +08:00` +- `tool` 完整回归：`77 / 77` + - 检测时间：`2026-07-16 19:07:56 +08:00` - 执行命令：`pnpm test:ai:tool` - `tool / docx` - 通过率：`40 / 40` - - 检测时间：`2026-07-14 13:59:28 +08:00` + - 检测时间：`2026-07-16 19:07:56 +08:00` - 工具面只保留 `docx_outline / docx_search / docx_read_blocks / docx_edit`；模型侧任务内 history/rollback 已移除，内部 replay/revision 与用户文件级回退继续保留。 - `tool / text-file` - 通过率：`2 / 2` - - 检测时间：`2026-07-15 +08:00` + - 检测时间：`2026-07-16 19:07:56 +08:00` - 覆盖补充：`file_read` 行范围与总行数；`file_write` 的 create、rewrite、唯一 edit、replaceAll、原子失败、BOM/CRLF 保持、结构化 diff 重放和修改记录。 +- `tool / pdf` + - 通过率：`4 / 4` + - 检测时间：`2026-07-16 19:07:56 +08:00` + - 新增 `pdf_inspect / pdf_search / pdf_read_pages / pdf_render_pages`，只负责 PDF 信息、文本层和页面渲染。 + - 页面 PNG 写入 `.ncurator/runtime/assets//page-N.png`，不作为文件修改或普通 artifact；图片理解由 `llm_recognize` 独立完成。 - `tool / kb` - 通过率：`2 / 2` - - 检测时间：`2026-07-13 13:04:39 +08:00` + - 检测时间：`2026-07-16 19:07:56 +08:00` - 覆盖补充：文件路径与网页来源 URL 返回契约已通过。 - `tool / terminal` - 通过率：`6 / 6` - - 检测时间：`2026-07-16 +08:00` + - 检测时间：`2026-07-16 19:07:56 +08:00` - 覆盖补充：terminal 回归测试按“安全指令集”语义维护；持久终端覆盖 cwd 保持、多终端、长命令让出、增量读取和停止；terminal 不根据命令文本猜测文件产物，文件变化统一由顶层 Run 扫描捕获。 - `tool / web-browsing` - - 通过率：`6 / 6` - - 检测时间：`2026-07-13 13:04:39 +08:00` + - 通过率：`7 / 7` + - 检测时间：`2026-07-16 19:07:56 +08:00` - 覆盖补充：`web-crawl-contract` 覆盖 `web_crawl` 的 `topic` 必填、`markdown` 字段来源于 `fit_markdown`、`requestedUrl/finalUrl`、分组 links 评分、HTTP 失败与结果过大；同文件覆盖 `web_search` 结构化候选网页结果和 `browser_use` 缺少内置浏览器服务时的真实 diagnostic；用例使用全局 15 分钟回归超时，不再单独写死 30 秒。 - `tool / utility` - - 通过率：`5 / 5` - - 检测时间：`2026-07-13 13:04:39 +08:00` + - 通过率：`7 / 7` + - 检测时间：`2026-07-16 19:07:56 +08:00` - 覆盖补充：`wait_time` 等待时长与返回 `waitedMs`；`llm_recognize` 已覆盖普通文本透传、无法确认类文本透传与底层失败分支。 - `tool / hand` - 通过率：`9 / 9` - - 检测时间：`2026-07-13 13:04:39 +08:00` + - 检测时间：`2026-07-16 19:07:56 +08:00` - `human / mock-llm` - - 通过率：`22 / 22` - - 检测时间：`2026-07-14 13:59:02 +08:00` + - 通过率：`23 / 23` + - 检测时间：`2026-07-16 19:08:37 +08:00` - `human / live` - - 登记 case：`10`；当前有效结果 `10 / 10` 通过，其中 `human-terminal-preview-service-live` 已于 `2026-07-16` 使用真实模型定向通过。 - - 检测时间：`2026-07-16 10:04:01 +08:00` - - 实际模型：`deepseek-v4-flash`。 + - 登记 case：`11`；当前有效结果 `11 / 11` 通过，其中 `human-terminal-preview-service-live` 已于 `2026-07-16` 使用真实模型定向通过。 + - 最近定向检测：`human-pdf-visual-reading-live` 于 `2026-07-16 19:11:47 +08:00` 通过；其他 live case 沿用各自最近有效结果。 + - 实际主模型：`deepseek-v4-flash`；PDF 图像识别模型：`qwen3-vl-flash`。 - memory、compaction、resume 均保留并通过；compaction 使用 `16K` 上下文，自然触发 1 次压缩，不限制压缩次数，压缩后继续完成周报文件交付。 - KB/docx case 使用 2 次 KB 检索定位源资料，并真实更新 2 份目标文档；不再依赖模型枚举目标文件。 - positioned DOCX case 正确命中复杂排版保护，没有危险写入，并交付了可直接手动粘贴的完整内容。 - - unit 全量：`51` files、`203 / 203` tests 通过。 + - AI unit：`45` files、`197 / 197` tests 通过；检测时间：`2026-07-16 19:05:58 +08:00`。 - 样例评分： - `human-conversation-memory-multi-run-live`：`10 / 10`；风险主线、待确认项、负责人和后续追问连续性完整。 - `human-risk-report-draft-live`：`10 / 10`；新建周报、原文件安全、未闭环风险、负责人动作和谨慎结论均满足。 @@ -64,6 +69,7 @@ - `human-web-crawl-url-inference-reading-live`：`9.0 / 10`；自行推断 pandas 官网地址并完成两份 CSV 文档对比。 - `human-text-file-read-write-live`：通过；真实调用 `replaceAll`、唯一 `edit` 和回读，最终文件与工具结果边界均满足。 - `human-terminal-preview-service-live`：`10 / 10`；真实进入子目录启动预览服务，完成页面访问、增量日志读取、正常停止、终端关闭和端口释放。 + - `human-pdf-visual-reading-live`：`10 / 10`；PDF 文本读取和页面渲染成功，第 1、9 页均完成真实图像识别，识别失败为 0，未调用 terminal、KB 或 docx，未产生普通 artifact，源 PDF 保持不变。 ## 3. Suite 设计 @@ -72,6 +78,7 @@ - `tool` 只验证真实生产工具调用，不允许 fake。 - `tool / docx` 包含 `docx-edit-serial-actions-stability`，用于守住同一次 `docx_edit(actions)` 内插入、替换、删除后的 block 定位稳定性。 - `tool / text-file` 覆盖普通文本读取、四种写入 action、修改记录、结构化 diff 与回退前置数据。 +- `tool / pdf` 使用真实 `深度学习.pdf`，覆盖元数据、页级正文、literal/regex 搜索、页面 PNG、工作区边界、损坏文件和结果大小限制。 - `tool / web-browsing` 包含 `web-crawl-contract`，用于守住 `web_crawl`、`web_search` 和 `browser_use` 工具契约。 - 目标：验证 tool 与 `hand` 工具执行守卫在真实生产代码下的输入输出是否正常、合理。 - 约束：必须真实调用生产工具、真实 store、真实 docx apply service、真实 artifact 流。 @@ -97,6 +104,7 @@ - AI backend 回归 case 必须使用生产 `createAiRunContext` 作为唯一 runtime 入口。 - 回归 harness 必须使用真实 `StorageService / DocumentsStore / ChatStore / DocxApplyStore / LlmConfigStore / DocxApplyService / EmbeddingService`。 +- 每个 AI run 的 DOCX、PDF 及未来其他文档 worker 资源统一通过 `AiRunResourceManager` 在 run 结束时释放；资源释放失败不能覆盖任务结果。 - `mock-llm` 统一通过 scripted LLM driver 固定 LLM 输出；除 LLM 输出外，其余全部真实。 - `live` 统一通过真实 `LlmConfigStore` 注入 live 配置，并走生产统一 LLM executor。 - 凡是会进入 `human` 的回归 run，必须在进入 `human` 前完成 embedding 基础服务 preflight；不允许把 embedding 初始化推迟到某个工具调用时才暴露。 @@ -558,6 +566,35 @@ AI unit / 辅助测试目录： - Oracle：失败时 bytes 不变；成功替换次数准确；新内容中的 oldText 不会被再次递归替换；BOM 与 CRLF 保持。 - Not：不支持正则 replaceAll。 +#### pdf 类别 + +##### `pdf-inspect-read-search-contract` + +- 测试类型：`backend` +- What：验证真实 PDF 的元数据、页级文本和搜索结果。 +- Why：守住 PDF 文本阅读的基础契约。 +- How：对 `深度学习.pdf` 执行 `pdf_inspect -> pdf_read_pages -> pdf_search`。 +- Oracle：页数为 11，标题为“深度学习”，正文没有 NUL，且“过度拟合”能命中第 8 或第 9 页。 +- Not：不识别页面图片。 + +##### `pdf-render-pages-runtime-assets-contract` + +- 测试类型：`backend` +- What：验证指定页面会被渲染为可供其他工具使用的 PNG。 +- Why：PDF 与图像识别需要通过普通图片路径解耦协作。 +- How：渲染第 1、9 页并检查文件路径、PNG 文件头、源文件哈希和事件。 +- Oracle：路径为 `.ncurator/runtime/assets//page-N.png`；图片真实存在；源 PDF 不变；不产生普通 artifact。 +- Not：不在 PDF 工具内部做图像识别。 + +##### `pdf-failure-boundary-contract` + +- 测试类型：`backend` +- What：验证 PDF 输入边界会明确失败。 +- Why：防止路径逃逸、伪 PDF、非法 regex 和越界页码进入不确定状态。 +- How：分别传入越界页码、非法 regex、错误文件头以及指向工作区外的符号链接。 +- Oracle：每种输入都返回明确失败，不读取工作区外文件。 +- Not：不做密码破解或损坏文件修复。 + #### kb 类别 ##### `kb-list-documents-contract` @@ -1051,6 +1088,15 @@ AI unit / 辅助测试目录： - Oracle：真实 `kb_search` 返回有效命中；两份目标文档分别发生成功写入并产生两组携带 `applyId` 的 file artifact；两份来源课件保持不变。按 10 分制评估 KB 证据、两份目标文件、grounded 内容、来源安全和最终交付，严格超过 8 分通过；只完成一个目标文件最多 8 分，不能通过。 - Not：不强制固定执行路径，也不要求 KB 缩圈后必须再打开所有候选资料。 +##### `human-pdf-visual-reading-live` + +- 测试类型：`backend / live` +- What：验证真实模型会自主组合 PDF 文本读取、页面渲染和通用图像识别。 +- Why：PDF 工具不内置图片理解，必须验证工具解耦后 AI 仍能完成图文结合阅读。 +- How：只选择 `深度学习.pdf`，询问第 1 页训练闭环和第 9 页 Proper fit / Overfit 图形差异，仅开放 PDF 四个工具与 `llm_recognize`。 +- Oracle：任务完成；调用 `pdf_read_pages`、包含第 9 页的 `pdf_render_pages` 和 `llm_recognize`；第 9 页图像识别必须成功且全部图像识别失败数为 0；识别路径来自 render 输出并位于 `.ncurator/runtime/assets//page-9.png`；不调用 terminal、KB、docx；最终答复同时说明训练闭环和两组曲线差异，不能用“无法识别图片”等内容绕过；源 PDF 不变。 +- Not：不固定工具调用顺序和最终措辞，不要求把 PDF 页面图片登记为 artifact。 + ##### `human-web-crawl-multi-hop-cross-domain-reading-live` - 测试类型：`backend` diff --git a/electron-builder.yml b/electron-builder.yml index 4729fe00..fbaad0b6 100644 --- a/electron-builder.yml +++ b/electron-builder.yml @@ -24,6 +24,8 @@ extraResources: asarUnpack: - '**/node_modules/onnxruntime-node/bin/napi-v3/**/*' - '**/node_modules/node-pty/build/Release/**/*' + - '**/node_modules/@napi-rs/canvas/**/*' + - '**/node_modules/@napi-rs/canvas-*/**/*' win: icon: build/icon.ico executableName: Ncurator diff --git a/electron.vite.config.ts b/electron.vite.config.ts index 87f92399..ddb4b06a 100644 --- a/electron.vite.config.ts +++ b/electron.vite.config.ts @@ -14,6 +14,7 @@ export default defineConfig({ workspacePrepareWorker: resolve( 'src/main/services/workspace/worker/workspacePrepareWorker.ts' ), + pdfWorker: resolve('src/main/services/pdf/worker/pdfWorker.ts'), embeddingRuntimeProcess: resolve( 'src/main/services/vector/runtime/embeddingRuntimeProcess.ts' ) diff --git a/package.json b/package.json index e0da2fdb..1cee41b9 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,7 @@ "@langchain/langgraph": "^1.1.5", "@langchain/openai": "^1.4.5", "@langchain/textsplitters": "^1.0.1", + "@napi-rs/canvas": "0.1.80", "@node-rs/jieba": "^2.0.1", "@react-pdf/renderer": "^4.1.0", "@xmldom/xmldom": "^0.8.11", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f2ac2c33..76cf1946 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -47,6 +47,9 @@ importers: '@langchain/textsplitters': specifier: ^1.0.1 version: 1.0.1(@langchain/core@1.1.44(@opentelemetry/api@1.9.0)(openai@6.36.0(ws@8.21.0(bufferutil@4.1.0))(zod@4.3.6))) + '@napi-rs/canvas': + specifier: 0.1.80 + version: 0.1.80 '@node-rs/jieba': specifier: ^2.0.1 version: 2.0.1 @@ -8240,7 +8243,6 @@ snapshots: '@napi-rs/canvas-linux-x64-gnu': 0.1.80 '@napi-rs/canvas-linux-x64-musl': 0.1.80 '@napi-rs/canvas-win32-x64-msvc': 0.1.80 - optional: true '@napi-rs/wasm-runtime@0.2.12': dependencies: diff --git a/scripts/verify-runtime-bundles.mjs b/scripts/verify-runtime-bundles.mjs index 67b2ad8e..daa4ebba 100644 --- a/scripts/verify-runtime-bundles.mjs +++ b/scripts/verify-runtime-bundles.mjs @@ -15,6 +15,7 @@ const assertNoImports = (label, graph, forbidden) => { const mainGraph = readBundleGraph(path.join(mainOutDir, 'index.js')) const prepareWorkerGraph = readBundleGraph(path.join(mainOutDir, 'workspacePrepareWorker.js')) +const pdfWorkerGraph = readBundleGraph(path.join(mainOutDir, 'pdfWorker.js')) const embeddingRuntimeGraph = readBundleGraph(path.join(mainOutDir, 'embeddingRuntimeProcess.js')) assertNoImports('Electron main process', mainGraph, [ @@ -26,6 +27,11 @@ assertNoImports('Workspace prepare worker', prepareWorkerGraph, [ '@huggingface/transformers', 'onnxruntime-node' ]) +assertNoImports('PDF worker', pdfWorkerGraph, [ + 'electron', + '@huggingface/transformers', + 'onnxruntime-node' +]) assertNoImports('Embedding utility process', embeddingRuntimeGraph, ['electron']) if (!embeddingRuntimeGraph.externalImports.has('@huggingface/transformers')) { diff --git a/src/main/ipc/handlers.ts b/src/main/ipc/handlers.ts index ce91811e..e42fa331 100644 --- a/src/main/ipc/handlers.ts +++ b/src/main/ipc/handlers.ts @@ -16,6 +16,7 @@ import { AiFileChangeService } from '../services/ai/fileChanges/AiFileChangeServ import { getAiContextUsage } from '../services/ai/contextUsage' import { DocxApplyService } from '../services/ai/human/tools/docx/review/DocxApplyService' import { releaseDocxRunSession } from '../services/ai/human/tools/docx/worker/docxWorkerClient' +import { releasePdfRunSession } from '../services/pdf/worker/pdfWorkerClient' import type { AiLlmIoRecord } from '../../shared/types' import { buildWebReferenceItem, @@ -167,7 +168,8 @@ export function registerHandlers(services: { const docxApplyService = new DocxApplyService(storageService.docxApply, aiFileChangeService) const attachmentService = new AttachmentService() const runResourceManager = new AiRunResourceManager([ - { id: 'docx', releaseRun: releaseDocxRunSession } + { id: 'docx', releaseRun: releaseDocxRunSession }, + { id: 'pdf', releaseRun: releasePdfRunSession } ]) const chatRunService = new ChatRunService({ chatStore, @@ -1537,7 +1539,8 @@ export function registerHandlers(services: { 'tiff', 'md', 'json', - 'docx' + 'docx', + 'pdf' ] }, { @@ -1553,7 +1556,8 @@ export function registerHandlers(services: { 'tiff', 'md', 'json', - 'docx' + 'docx', + 'pdf' ] } ] diff --git a/src/main/services/ai/human/constants.ts b/src/main/services/ai/human/constants.ts index 5279fe72..39d2d953 100644 --- a/src/main/services/ai/human/constants.ts +++ b/src/main/services/ai/human/constants.ts @@ -3,6 +3,7 @@ export const AI_TOOL_FAMILY_IDS = [ 'text-file', 'kb', 'docx', + 'pdf', 'web', 'utility', 'image-recognition' @@ -16,7 +17,12 @@ export const BUILTIN_RUNTIME_SKILL_IDS = [ 'utility' ] as const -export const EXTERNAL_RUNTIME_SKILL_IDS = ['docx-read', 'docx-write', 'image-recognition'] as const +export const EXTERNAL_RUNTIME_SKILL_IDS = [ + 'docx-read', + 'docx-write', + 'pdf-read', + 'image-recognition' +] as const export const LEARNABLE_RUNTIME_SKILL_IDS = EXTERNAL_RUNTIME_SKILL_IDS diff --git a/src/main/services/ai/human/node/learningNode/skillLoader.ts b/src/main/services/ai/human/node/learningNode/skillLoader.ts index 00d28ee5..adaf8f26 100644 --- a/src/main/services/ai/human/node/learningNode/skillLoader.ts +++ b/src/main/services/ai/human/node/learningNode/skillLoader.ts @@ -13,6 +13,7 @@ const TOOL_FAMILY_BUILDERS: Record< 'text-file': TOOL_FAMILY_DEFINITIONS['text-file'].buildTools, kb: TOOL_FAMILY_DEFINITIONS.kb.buildTools, docx: TOOL_FAMILY_DEFINITIONS.docx.buildTools, + pdf: TOOL_FAMILY_DEFINITIONS.pdf.buildTools, web: TOOL_FAMILY_DEFINITIONS.web.buildTools, utility: TOOL_FAMILY_DEFINITIONS.utility.buildTools, 'image-recognition': TOOL_FAMILY_DEFINITIONS['image-recognition'].buildTools diff --git a/src/main/services/ai/human/shared/historyRenderers.ts b/src/main/services/ai/human/shared/historyRenderers.ts index 0f8e5130..30278e6a 100644 --- a/src/main/services/ai/human/shared/historyRenderers.ts +++ b/src/main/services/ai/human/shared/historyRenderers.ts @@ -135,6 +135,87 @@ const TOOL_HISTORY_WHITELIST_BY_NAME: Record = { blocks: DOCX_BLOCK_WHITELIST }, docx_edit: DOCX_WRITE_RESULT_WHITELIST, + pdf_inspect: { + sourcePath: true, + pageCount: true, + metadata: { + title: true, + author: true, + subject: true, + keywords: true, + creator: true, + producer: true, + creationDate: true, + modificationDate: true + }, + outline: { + title: true, + pageNumber: true, + depth: true + }, + encrypted: true, + warnings: true + }, + pdf_search: { + sourcePath: true, + pageCount: true, + query: true, + mode: true, + pageRange: { + fromPage: true, + toPage: true + }, + totalMatchCount: true, + returnedMatchCount: true, + matches: { + pageNumber: true, + matchedText: true, + snippet: true, + start: true, + end: true + }, + pagesWithNoTextCount: true, + pagesWithSuspiciousTextCount: true, + warnings: true, + error: true, + reason: true, + decisionHint: true + }, + pdf_read_pages: { + sourcePath: true, + pageCount: true, + pages: { + pageNumber: true, + text: true, + charCount: true, + textStatus: true, + imageCount: true, + width: true, + height: true, + rotation: true, + warnings: true + }, + warnings: true, + error: true, + reason: true, + decisionHint: true + }, + pdf_render_pages: { + sourcePath: true, + pageCount: true, + assetBucketId: true, + images: { + pageNumber: true, + path: true, + mimeType: true, + width: true, + height: true + }, + warnings: true, + error: true, + reason: true, + decisionHint: true + }, web_search: { query: true, results: { diff --git a/src/main/services/ai/human/shared/toolResultRenderers.ts b/src/main/services/ai/human/shared/toolResultRenderers.ts index 2cd98511..4f59c959 100644 --- a/src/main/services/ai/human/shared/toolResultRenderers.ts +++ b/src/main/services/ai/human/shared/toolResultRenderers.ts @@ -248,6 +248,103 @@ const renderDocxOutlineResult = (obj: JsonObject): string => .filter(Boolean) .join('\n') +const renderPdfPageList = (value: JsonValue | undefined): string => { + if (!Array.isArray(value) || value.length === 0) return 'none' + return value + .map((item, index) => { + if (!isJsonObject(item)) return renderGenericArrayItem(item, index) + return [ + `page ${readNumberTextField(item, 'pageNumber') || String(index + 1)}：${renderKeyValuePairs( + [ + ['textStatus', readStringField(item, 'textStatus')], + ['charCount', readNumberTextField(item, 'charCount')], + ['imageCount', readNumberTextField(item, 'imageCount')] + ] + )}`, + readStringField(item, 'text') || '(no text)' + ].join('\n') + }) + .join('\n\n') +} + +const renderPdfSearchMatches = (value: JsonValue | undefined): string => { + if (!Array.isArray(value) || value.length === 0) return 'none' + return value + .map((item, index) => { + if (!isJsonObject(item)) return renderGenericArrayItem(item, index) + return `${index + 1}. ${renderKeyValuePairs([ + ['pageNumber', readNumberTextField(item, 'pageNumber')], + ['matchedText', readStringField(item, 'matchedText')], + ['snippet', readStringField(item, 'snippet')] + ])}` + }) + .join('\n') +} + +const renderPdfImages = (value: JsonValue | undefined): string => { + if (!Array.isArray(value) || value.length === 0) return 'none' + return value + .map((item, index) => { + if (!isJsonObject(item)) return renderGenericArrayItem(item, index) + return `${index + 1}. ${renderKeyValuePairs([ + ['pageNumber', readNumberTextField(item, 'pageNumber')], + ['path', readStringField(item, 'path')], + ['mimeType', readStringField(item, 'mimeType')], + ['width', readNumberTextField(item, 'width')], + ['height', readNumberTextField(item, 'height')] + ])}` + }) + .join('\n') +} + +const renderPdfInspectResult = (obj: JsonObject): string => + [ + renderKeyValuePairs([ + ['sourcePath', readStringField(obj, 'sourcePath')], + ['pageCount', readNumberTextField(obj, 'pageCount')], + ['encrypted', renderPrimitive(obj.encrypted)] + ]), + 'metadata:', + isJsonObject(obj.metadata) ? renderGenericObject(obj.metadata) : 'none', + 'outline:', + Array.isArray(obj.outline) ? renderGenericArray(obj.outline) : 'none' + ].join('\n') + +const renderPdfSearchResult = (obj: JsonObject): string => + [ + renderKeyValuePairs([ + ['sourcePath', readStringField(obj, 'sourcePath')], + ['pageCount', readNumberTextField(obj, 'pageCount')], + ['query', readStringField(obj, 'query')], + ['mode', readStringField(obj, 'mode')], + ['returnedMatchCount', readNumberTextField(obj, 'returnedMatchCount')], + ['totalMatchCount', readNumberTextField(obj, 'totalMatchCount')] + ]), + 'matches:', + renderPdfSearchMatches(obj.matches) + ].join('\n') + +const renderPdfReadPagesResult = (obj: JsonObject): string => + [ + renderKeyValuePairs([ + ['sourcePath', readStringField(obj, 'sourcePath')], + ['pageCount', readNumberTextField(obj, 'pageCount')] + ]), + 'pages:', + renderPdfPageList(obj.pages) + ].join('\n') + +const renderPdfRenderPagesResult = (obj: JsonObject): string => + [ + renderKeyValuePairs([ + ['sourcePath', readStringField(obj, 'sourcePath')], + ['pageCount', readNumberTextField(obj, 'pageCount')], + ['assetBucketId', readStringField(obj, 'assetBucketId')] + ]), + 'images:', + renderPdfImages(obj.images) + ].join('\n') + const renderWebCrawlLinkList = (value: JsonValue | undefined): string => { if (!Array.isArray(value) || value.length === 0) return 'none' @@ -386,6 +483,14 @@ export const renderToolResultDataText = ( return renderDocxSearchResult(value) case 'docx_outline': return renderDocxOutlineResult(value) + case 'pdf_inspect': + return renderPdfInspectResult(value) + case 'pdf_search': + return renderPdfSearchResult(value) + case 'pdf_read_pages': + return renderPdfReadPagesResult(value) + case 'pdf_render_pages': + return renderPdfRenderPagesResult(value) case 'web_search': return renderWebSearchResult(value) case 'web_crawl': diff --git a/src/main/services/ai/human/skills.ts b/src/main/services/ai/human/skills.ts index 87767846..a60938f9 100644 --- a/src/main/services/ai/human/skills.ts +++ b/src/main/services/ai/human/skills.ts @@ -51,6 +51,17 @@ export const SKILL_DEFINITIONS: Record = { whenToUse: '适用于用户要求生成、补充、改写、删除或格式化 docx 内容的任务。', toolNames: ['docx_edit'] }, + 'pdf-read': { + skillId: 'pdf-read', + familyId: 'pdf', + source: 'external', + countsTowardToolBudget: true, + title: 'pdf-read', + description: + '能力：读取工作区内 PDF 的文档信息、页级文本、检索结果，并将指定页面渲染为图片。\n结果：返回 PDF 元数据、页码、正文、匹配位置或页面图片路径。\n边界：只读取 PDF，不修改源文件；文本结果取决于 PDF 文本层，页面图片不包含识别结果。', + whenToUse: '适用于需要阅读 PDF 文本、搜索 PDF 内容或查看 PDF 页面图形的任务。', + toolNames: ['pdf_inspect', 'pdf_search', 'pdf_read_pages', 'pdf_render_pages'] + }, 'image-recognition': { skillId: 'image-recognition', familyId: 'image-recognition', diff --git a/src/main/services/ai/human/tools/index.ts b/src/main/services/ai/human/tools/index.ts index 9954bf90..15033025 100644 --- a/src/main/services/ai/human/tools/index.ts +++ b/src/main/services/ai/human/tools/index.ts @@ -1,6 +1,7 @@ import type { AiToolFamilyId, ToolFamilyDefinition } from './types' import { docxToolFamily } from './docx' import { imageRecognitionToolFamily } from './imageRecognition' +import { pdfToolFamily } from './pdf' import { buildKbCapabilityTools } from './kb/retrievalTools' import { terminalToolFamily } from './terminal' import { textFileToolFamily } from './textFile' @@ -17,6 +18,7 @@ export const TOOL_FAMILY_DEFINITIONS: Record[0]) => { + const runtime = createPdfToolRuntime(input) + input.cache.pdfRuntime = runtime + return { tools: buildPdfReadTools(runtime), runtime } +} + +export const pdfToolFamily = createWorkspaceBoundToolFamily({ + familyId: 'pdf', + descriptionForTool: (name) => `pdf tool: ${name}`, + reason: 'pdf capability requires workspace binding before execution', + error: 'workspace is required for pdf capability', + buildWorkspaceTools: (input) => { + return buildPdfCapabilityTools({ + ...input, + safePreviewChars: 1400, + modelContextWindowTokens: + input.ctx.humanRuntimeOverrides?.modelContextWindowTokens || + DEFAULT_MODEL_CONTEXT_WINDOW_TOKENS + }).tools + } +}) diff --git a/src/main/services/ai/human/tools/pdf/outputSchemas.ts b/src/main/services/ai/human/tools/pdf/outputSchemas.ts new file mode 100644 index 00000000..87ae7781 --- /dev/null +++ b/src/main/services/ai/human/tools/pdf/outputSchemas.ts @@ -0,0 +1,86 @@ +import { z } from 'zod/v4' + +const warningsSchema = z.array(z.string()) + +const pdfMetadataSchema = z.object({ + title: z.string().optional(), + author: z.string().optional(), + subject: z.string().optional(), + keywords: z.string().optional(), + creator: z.string().optional(), + producer: z.string().optional(), + creationDate: z.string().optional(), + modificationDate: z.string().optional() +}) + +const pdfOutlineItemSchema = z.object({ + title: z.string(), + pageNumber: z.number().int().optional(), + depth: z.number().int() +}) + +const pdfPageSchema = z.object({ + pageNumber: z.number().int(), + text: z.string(), + charCount: z.number().int(), + textStatus: z.enum(['available', 'empty', 'suspicious']), + imageCount: z.number().int(), + width: z.number(), + height: z.number(), + rotation: z.number(), + warnings: warningsSchema +}) + +export const pdfInspectOutputSchema = z.object({ + sourcePath: z.string(), + pageCount: z.number().int(), + metadata: pdfMetadataSchema, + outline: z.array(pdfOutlineItemSchema), + encrypted: z.boolean(), + warnings: warningsSchema +}) + +export const pdfSearchOutputSchema = z.object({ + sourcePath: z.string(), + pageCount: z.number().int(), + query: z.string(), + mode: z.enum(['literal', 'regex']), + pageRange: z.object({ fromPage: z.number().int(), toPage: z.number().int() }), + totalMatchCount: z.number().int(), + returnedMatchCount: z.number().int(), + matches: z.array( + z.object({ + pageNumber: z.number().int(), + matchedText: z.string(), + snippet: z.string(), + start: z.number().int(), + end: z.number().int() + }) + ), + pagesWithNoTextCount: z.number().int(), + pagesWithSuspiciousTextCount: z.number().int(), + warnings: warningsSchema +}) + +export const pdfReadPagesOutputSchema = z.object({ + sourcePath: z.string(), + pageCount: z.number().int(), + pages: z.array(pdfPageSchema), + warnings: warningsSchema +}) + +export const pdfRenderPagesOutputSchema = z.object({ + sourcePath: z.string(), + pageCount: z.number().int(), + assetBucketId: z.string(), + images: z.array( + z.object({ + pageNumber: z.number().int(), + path: z.string(), + mimeType: z.literal('image/png'), + width: z.number().int(), + height: z.number().int() + }) + ), + warnings: warningsSchema +}) diff --git a/src/main/services/ai/human/tools/pdf/runtime.ts b/src/main/services/ai/human/tools/pdf/runtime.ts new file mode 100644 index 00000000..d274ef1b --- /dev/null +++ b/src/main/services/ai/human/tools/pdf/runtime.ts @@ -0,0 +1,214 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import type { AiToolCallResult, JsonObject } from '../../../../../../shared/types' +import type { PdfOpenResult } from '../../../../pdf/types' +import { getPdfWorkerClient } from '../../../../pdf/worker/pdfWorkerClient' +import type { PdfWorkerRequest, PdfWorkerResponse } from '../../../../pdf/worker/types' +import { isPathInsideRoot, resolveWorkspacePath } from '../terminal/commandPolicy' +import { createToolEventEmitter } from '../utils/toolEvents' +import type { ToolEventEmitter } from '../utils/toolEvents' +import type { PdfCapabilityToolsInput } from './types' + +type FailedPdfWorkerResponse = Extract + +export class PdfToolRuntimeError extends Error { + public readonly reason: FailedPdfWorkerResponse['code'] + public readonly warnings: string[] + + public constructor(response: FailedPdfWorkerResponse) { + super(response.error) + this.name = 'PdfToolRuntimeError' + this.reason = response.code + this.warnings = response.warnings || [] + } +} + +export type PdfToolResourceContext = { + family: 'pdf' + sourcePath: string + effectKind: 'inspected' | 'searched' | 'read' | 'rendered' +} + +export type PdfToolFailure = { + ok: false + error: string + warnings: string[] + reason: string + decisionHint?: JsonObject + resourceContext?: PdfToolResourceContext +} + +export type PdfReadSession = PdfOpenResult + +export type PdfToolRuntime = { + input: PdfCapabilityToolsInput + workspaceRootPath: string + toolEvents: ToolEventEmitter + resolvePdfPath: (rawPath: string) => Promise + ensureDocument: (sourcePath: string) => Promise + callWorker: (request: Omit) => Promise + buildResourceContext: ( + sourcePath: string, + effectKind: PdfToolResourceContext['effectKind'] + ) => PdfToolResourceContext + emitFailure: (input: { + toolName: string + toolCallId: string + error: string + reason: string + warnings?: string[] + decisionHint?: JsonObject + resourceContext?: PdfToolResourceContext + }) => PdfToolFailure + emitWorkerFailure: (input: { + toolName: string + toolCallId: string + response: FailedPdfWorkerResponse + resourceContext?: PdfToolResourceContext + }) => PdfToolFailure + emitSuccess: (input: { + toolName: string + toolCallId: string + result: JsonObject + resourceContext: PdfToolResourceContext + }) => void +} + +const buildPdfToolCallResult = (input: { + toolName: string + ok: boolean + data: AiToolCallResult['data'] + resourceContext?: PdfToolResourceContext + error?: string +}): AiToolCallResult => ({ + toolName: input.toolName, + ok: input.ok, + data: input.data, + resourceContext: input.resourceContext, + diagnostic: input.ok || !input.error ? undefined : { message: input.error } +}) + +export const createPdfToolRuntime = (input: PdfCapabilityToolsInput): PdfToolRuntime => { + const workspaceRootPath = path.resolve(input.workspaceRootPath) + const sessionBySourcePath = new Map() + const toolEvents = createToolEventEmitter({ + ctx: input.ctx, + runId: input.runId, + taskId: input.taskId, + stage: 'human_hand' + }) + + const callWorker: PdfToolRuntime['callWorker'] = async (request) => + await getPdfWorkerClient().call({ ...request, runId: input.runId } as PdfWorkerRequest) + + const resolvePdfPath = async (rawPath: string): Promise => { + const lexicalPath = resolveWorkspacePath({ + rawPath, + cwd: workspaceRootPath, + rootPath: workspaceRootPath + }) + if (!lexicalPath) throw new Error('PDF 路径必须位于当前工作区内') + if (path.extname(lexicalPath).toLowerCase() !== '.pdf') + throw new Error('sourcePath 必须是 .pdf 文件') + + const [realWorkspaceRoot, realSourcePath] = await Promise.all([ + fs.realpath(workspaceRootPath), + fs.realpath(lexicalPath) + ]) + if (!isPathInsideRoot(realSourcePath, realWorkspaceRoot)) { + throw new Error('PDF 真实路径必须位于当前工作区内') + } + if (path.extname(realSourcePath).toLowerCase() !== '.pdf') + throw new Error('sourcePath 必须是 .pdf 文件') + const stat = await fs.stat(realSourcePath) + if (!stat.isFile()) throw new Error('sourcePath 不是文件') + const handle = await fs.open(realSourcePath, 'r') + try { + const header = Buffer.alloc(5) + const { bytesRead } = await handle.read(header, 0, header.length, 0) + if (bytesRead !== 5 || header.toString('ascii') !== '%PDF-') { + throw new Error('文件头无效，不是可读取的 PDF') + } + } finally { + await handle.close() + } + return realSourcePath + } + + const ensureDocument = async (rawSourcePath: string): Promise => { + const sourcePath = await resolvePdfPath(rawSourcePath) + const cached = sessionBySourcePath.get(sourcePath) + if (cached) return cached + const response = await callWorker({ action: 'open', payload: { sourcePath } }) + if (!response.ok) throw new PdfToolRuntimeError(response) + if (response.payload.action !== 'open') throw new Error('PDF worker 协议错误（open）') + const session: PdfReadSession = response.payload + sessionBySourcePath.set(sourcePath, session) + return session + } + + const buildResourceContext: PdfToolRuntime['buildResourceContext'] = ( + sourcePath, + effectKind + ) => ({ family: 'pdf', sourcePath, effectKind }) + + const emitFailure: PdfToolRuntime['emitFailure'] = (params) => { + const failure: PdfToolFailure = { + ok: false, + error: params.error, + warnings: params.warnings || [], + reason: params.reason, + decisionHint: params.decisionHint, + resourceContext: params.resourceContext + } + toolEvents.emitResult({ + toolName: params.toolName, + toolCallId: params.toolCallId, + error: params.error, + result: buildPdfToolCallResult({ + toolName: params.toolName, + ok: false, + data: failure, + resourceContext: params.resourceContext, + error: params.error + }) + }) + return failure + } + + const emitWorkerFailure: PdfToolRuntime['emitWorkerFailure'] = (params) => + emitFailure({ + toolName: params.toolName, + toolCallId: params.toolCallId, + error: params.response.error, + reason: params.response.code, + warnings: params.response.warnings, + resourceContext: params.resourceContext + }) + + const emitSuccess: PdfToolRuntime['emitSuccess'] = (params) => { + toolEvents.emitResult({ + toolName: params.toolName, + toolCallId: params.toolCallId, + result: buildPdfToolCallResult({ + toolName: params.toolName, + ok: true, + data: params.result, + resourceContext: params.resourceContext + }) + }) + } + + return { + input, + workspaceRootPath, + toolEvents, + resolvePdfPath, + ensureDocument, + callWorker, + buildResourceContext, + emitFailure, + emitWorkerFailure, + emitSuccess + } +} diff --git a/src/main/services/ai/human/tools/pdf/schemas.ts b/src/main/services/ai/human/tools/pdf/schemas.ts new file mode 100644 index 00000000..6a55ed0e --- /dev/null +++ b/src/main/services/ai/human/tools/pdf/schemas.ts @@ -0,0 +1,41 @@ +import { z } from 'zod/v4' + +const sourcePathSchema = z.string().trim().min(1) +const pageNumberSchema = z.number().int().min(1) + +export const pdfInspectSchema = z.object({ + sourcePath: sourcePathSchema +}) + +export const pdfPageRangeSchema = z.object({ + fromPage: pageNumberSchema, + toPage: pageNumberSchema +}) + +export const pdfSearchSchema = z.object({ + sourcePath: sourcePathSchema, + query: z.string().min(1), + mode: z.enum(['literal', 'regex']).optional(), + pageRange: pdfPageRangeSchema.optional(), + maxMatches: z.number().int().min(1).max(50).optional() +}) + +export const pdfReadPagesSchema = z.object({ + sourcePath: sourcePathSchema, + selection: z.discriminatedUnion('type', [ + z.object({ + type: z.literal('page_range'), + fromPage: pageNumberSchema, + toPage: pageNumberSchema + }), + z.object({ + type: z.literal('page_numbers'), + pageNumbers: z.array(pageNumberSchema).min(1).max(50) + }) + ]) +}) + +export const pdfRenderPagesSchema = z.object({ + sourcePath: sourcePathSchema, + pageNumbers: z.array(pageNumberSchema).min(1).max(20) +}) diff --git a/src/main/services/ai/human/tools/pdf/toolSpecs.ts b/src/main/services/ai/human/tools/pdf/toolSpecs.ts new file mode 100644 index 00000000..8dd94515 --- /dev/null +++ b/src/main/services/ai/human/tools/pdf/toolSpecs.ts @@ -0,0 +1,183 @@ +import type { z } from 'zod/v4' +import type { JsonObject } from '../../../../../../shared/types' +import type { RuntimeSkillId } from '../../types' +import type { HumanToolSpec } from '../types' +import { createStaticExecutionMetaResolver, readStringArg } from '../utils/toolSpecUtils' +import { + pdfInspectOutputSchema, + pdfReadPagesOutputSchema, + pdfRenderPagesOutputSchema, + pdfSearchOutputSchema +} from './outputSchemas' +import { + pdfInspectSchema, + pdfReadPagesSchema, + pdfRenderPagesSchema, + pdfSearchSchema +} from './schemas' + +const PDF_READ_EXECUTION_META = createStaticExecutionMetaResolver({ + mutability: 'read_only', + getResourceKey: (args) => readStringArg(args, 'sourcePath'), + defaultResourceKey: 'pdf' +}) + +type PdfToolSpec< + TArgs extends JsonObject = JsonObject, + TOutput extends JsonObject = JsonObject, + TName extends string = string, + TSkillId extends RuntimeSkillId = 'pdf-read' +> = HumanToolSpec & { + description: string +} + +const PDF_SOURCE_PATH_EXAMPLE = '学习笔记/深度学习.pdf' + +const createPdfPromptSpec = (input: { + name: string + description: string + whenToUse: string + examples: Array<{ goal: string; args: JsonObject }> +}) => ({ + name: input.name, + description: input.description, + whenToUse: input.whenToUse, + examples: input.examples +}) + +export const PDF_INSPECT_TOOL_SPEC: PdfToolSpec< + z.infer, + z.infer, + 'pdf_inspect', + 'pdf-read' +> = { + name: 'pdf_inspect', + familyId: 'pdf', + skillIds: ['pdf-read'], + requiresWorkspace: true, + description: + '能力：读取 sourcePath 对应 PDF 的基础信息。\n结果：返回页数、元数据、书签提纲和警告。\n边界：只检查 PDF，不返回各页正文，不修改源文件。', + promptSpec: createPdfPromptSpec({ + name: 'pdf_inspect', + description: + '能力：读取 sourcePath 对应 PDF 的基础信息。\n结果：返回页数、元数据、书签提纲和警告。\n边界：只检查 PDF，不返回各页正文，不修改源文件。', + whenToUse: '适用于需要先了解 PDF 页数、元数据或目录结构的任务。', + examples: [ + { + goal: '先查看 PDF 的页数、标题和目录结构', + args: { sourcePath: PDF_SOURCE_PATH_EXAMPLE } + } + ] + }), + inputSchema: pdfInspectSchema, + outputSchema: pdfInspectOutputSchema, + resolveExecutionMeta: PDF_READ_EXECUTION_META +} + +export const PDF_SEARCH_TOOL_SPEC: PdfToolSpec< + z.infer, + z.infer, + 'pdf_search', + 'pdf-read' +> = { + name: 'pdf_search', + familyId: 'pdf', + skillIds: ['pdf-read'], + requiresWorkspace: true, + description: + '能力：在 sourcePath 对应 PDF 的文本层中按 literal 或 regex 搜索，可限制页码范围和返回数量。\n结果：返回命中页码、匹配文本、上下文片段、位置和文本层状态统计。\n边界：只搜索 PDF 文本层；扫描图片中的文字不会自动识别。', + promptSpec: createPdfPromptSpec({ + name: 'pdf_search', + description: + '能力：在 sourcePath 对应 PDF 的文本层中按 literal 或 regex 搜索，可限制页码范围和返回数量。\n结果：返回命中页码、匹配文本、上下文片段、位置和文本层状态统计。\n边界：只搜索 PDF 文本层；扫描图片中的文字不会自动识别。', + whenToUse: '适用于需要在 PDF 正文中定位关键词或表达式的任务。', + examples: [ + { + goal: '按原词定位 PDF 中讨论过度拟合的页面', + args: { sourcePath: PDF_SOURCE_PATH_EXAMPLE, query: '过度拟合' } + }, + { + goal: '用正则匹配 PDF 文本层中空格可能变化的英文术语', + args: { sourcePath: PDF_SOURCE_PATH_EXAMPLE, query: 'Proper\\s+fit', mode: 'regex' } + } + ] + }), + inputSchema: pdfSearchSchema, + outputSchema: pdfSearchOutputSchema, + resolveExecutionMeta: PDF_READ_EXECUTION_META +} + +export const PDF_READ_PAGES_TOOL_SPEC: PdfToolSpec< + z.infer, + z.infer, + 'pdf_read_pages', + 'pdf-read' +> = { + name: 'pdf_read_pages', + familyId: 'pdf', + skillIds: ['pdf-read'], + requiresWorkspace: true, + description: + '能力：按连续页码范围或指定页码集合读取 sourcePath 对应 PDF 的文本层。\n结果：逐页返回正文、页码、文本状态、图片数量、页面尺寸和警告。\n边界：不理解页面图片；空文本或可疑文本会明确标记，不修改源文件。', + promptSpec: createPdfPromptSpec({ + name: 'pdf_read_pages', + description: + '能力：按连续页码范围或指定页码集合读取 sourcePath 对应 PDF 的文本层。\n结果：逐页返回正文、页码、文本状态、图片数量、页面尺寸和警告。\n边界：不理解页面图片；空文本或可疑文本会明确标记，不修改源文件。', + whenToUse: '适用于需要精读 PDF 某一页或连续页码范围正文的任务。', + examples: [ + { + goal: '连续读取 PDF 前三页正文', + args: { + sourcePath: PDF_SOURCE_PATH_EXAMPLE, + selection: { type: 'page_range', fromPage: 1, toPage: 3 } + } + }, + { + goal: '只读取任务明确关心的第 1 页和第 9 页', + args: { + sourcePath: PDF_SOURCE_PATH_EXAMPLE, + selection: { type: 'page_numbers', pageNumbers: [1, 9] } + } + } + ] + }), + inputSchema: pdfReadPagesSchema, + outputSchema: pdfReadPagesOutputSchema, + resolveExecutionMeta: PDF_READ_EXECUTION_META +} + +export const PDF_RENDER_PAGES_TOOL_SPEC: PdfToolSpec< + z.infer, + z.infer, + 'pdf_render_pages', + 'pdf-read' +> = { + name: 'pdf_render_pages', + familyId: 'pdf', + skillIds: ['pdf-read'], + requiresWorkspace: true, + description: + '能力：将 sourcePath 对应 PDF 的指定页面渲染为 PNG 图片。\n结果：返回每页的页码、图片绝对路径、格式和尺寸；图片位于当前工作区 .ncurator/runtime/assets// 下。\n边界：只生成页面图片，不识别或解释图片内容，不把图片记为文件修改或普通 artifact。', + promptSpec: createPdfPromptSpec({ + name: 'pdf_render_pages', + description: + '能力：将 sourcePath 对应 PDF 的指定页面渲染为 PNG 图片。\n结果：返回每页的页码、图片绝对路径、格式和尺寸；图片位于当前工作区 .ncurator/runtime/assets// 下。\n边界：只生成页面图片，不识别或解释图片内容，不把图片记为文件修改或普通 artifact。', + whenToUse: '适用于 PDF 文本层不足以回答问题，需要查看页面图形或把页面交给图像识别工具的任务。', + examples: [ + { + goal: '把包含对比曲线的第 9 页渲染成图片，供后续图像识别', + args: { sourcePath: PDF_SOURCE_PATH_EXAMPLE, pageNumbers: [9] } + } + ] + }), + inputSchema: pdfRenderPagesSchema, + outputSchema: pdfRenderPagesOutputSchema, + resolveExecutionMeta: PDF_READ_EXECUTION_META +} + +export const PDF_TOOL_SPECS = [ + PDF_INSPECT_TOOL_SPEC, + PDF_SEARCH_TOOL_SPEC, + PDF_READ_PAGES_TOOL_SPEC, + PDF_RENDER_PAGES_TOOL_SPEC +] as const diff --git a/src/main/services/ai/human/tools/pdf/tools/readTools.ts b/src/main/services/ai/human/tools/pdf/tools/readTools.ts new file mode 100644 index 00000000..ffcfe133 --- /dev/null +++ b/src/main/services/ai/human/tools/pdf/tools/readTools.ts @@ -0,0 +1,279 @@ +import { tool } from '@langchain/core/tools' +import type { StructuredToolInterface } from '@langchain/core/tools' +import type { JsonObject } from '../../../../../../../shared/types' +import { withToolLifecycle } from '../../utils/lifecycle' +import { checkToolResultSize } from '../../utils/resultSizeGuard' +import { + PdfToolRuntimeError, + type PdfToolFailure, + type PdfToolResourceContext, + type PdfToolRuntime +} from '../runtime' +import { + PDF_INSPECT_TOOL_SPEC, + PDF_READ_PAGES_TOOL_SPEC, + PDF_RENDER_PAGES_TOOL_SPEC, + PDF_SEARCH_TOOL_SPEC +} from '../toolSpecs' + +const uniquePageNumbers = (pageNumbers: number[]): number[] => [...new Set(pageNumbers)] + +const runTool = async (input: { + runtime: PdfToolRuntime + toolName: string + toolCallId: string + sourcePath: string + effectKind: PdfToolResourceContext['effectKind'] + execute: ( + session: Awaited> + ) => Promise +}): Promise => { + try { + input.runtime.input.ctx.checkCancelled() + const document = await input.runtime.ensureDocument(input.sourcePath) + const resourceContext = input.runtime.buildResourceContext( + document.sourcePath, + input.effectKind + ) + const result = await input.execute(document) + if ('ok' in result && result.ok === false) return result + const size = checkToolResultSize({ + toolName: input.toolName, + value: result, + modelContextWindowTokens: input.runtime.input.modelContextWindowTokens + }) + if (size.tooLarge) { + return input.runtime.emitFailure({ + toolName: input.toolName, + toolCallId: input.toolCallId, + reason: 'result_too_large', + error: `PDF 工具结果过大：预计 ${String(size.estimatedTokens)} tokens，限制 ${String(size.limit)} tokens`, + decisionHint: size.decisionHint, + resourceContext + }) + } + input.runtime.emitSuccess({ + toolName: input.toolName, + toolCallId: input.toolCallId, + result, + resourceContext + }) + return result + } catch (error) { + return input.runtime.emitFailure({ + toolName: input.toolName, + toolCallId: input.toolCallId, + reason: error instanceof PdfToolRuntimeError ? error.reason : 'invalid_input', + error: error instanceof Error ? error.message : String(error), + warnings: error instanceof PdfToolRuntimeError ? error.warnings : undefined + }) + } +} + +export const buildPdfReadTools = (runtime: PdfToolRuntime): StructuredToolInterface[] => { + const inspectTool = tool( + async (args) => + await withToolLifecycle({ + emitter: runtime.toolEvents, + toolName: PDF_INSPECT_TOOL_SPEC.name, + args: args as JsonObject, + execute: async ({ toolCallId }) => + await runTool({ + runtime, + toolName: PDF_INSPECT_TOOL_SPEC.name, + toolCallId, + sourcePath: args.sourcePath, + effectKind: 'inspected', + execute: async (document) => ({ + sourcePath: document.sourcePath, + pageCount: document.pageCount, + metadata: document.metadata, + outline: document.outline, + encrypted: document.encrypted, + warnings: document.warnings + }) + }) + }), + { + name: PDF_INSPECT_TOOL_SPEC.name, + description: PDF_INSPECT_TOOL_SPEC.description, + schema: PDF_INSPECT_TOOL_SPEC.inputSchema + } + ) + + const searchTool = tool( + async (args) => + await withToolLifecycle({ + emitter: runtime.toolEvents, + toolName: PDF_SEARCH_TOOL_SPEC.name, + args: args as JsonObject, + execute: async ({ toolCallId }) => + await runTool({ + runtime, + toolName: PDF_SEARCH_TOOL_SPEC.name, + toolCallId, + sourcePath: args.sourcePath, + effectKind: 'searched', + execute: async (document) => { + const fromPage = args.pageRange?.fromPage ?? 1 + const toPage = args.pageRange?.toPage ?? document.pageCount + const response = await runtime.callWorker({ + action: 'search', + payload: { + docId: document.docId, + query: args.query, + mode: args.mode || 'literal', + fromPage, + toPage, + maxMatches: args.maxMatches || 20 + } + }) + const resourceContext = runtime.buildResourceContext(document.sourcePath, 'searched') + if (!response.ok) { + return runtime.emitWorkerFailure({ + toolName: PDF_SEARCH_TOOL_SPEC.name, + toolCallId, + response, + resourceContext + }) + } + if (response.payload.action !== 'search') + throw new Error('PDF worker 协议错误（search）') + return { + sourcePath: document.sourcePath, + pageCount: document.pageCount, + query: args.query, + mode: args.mode || 'literal', + pageRange: { fromPage, toPage }, + totalMatchCount: response.payload.totalMatchCount, + returnedMatchCount: response.payload.matches.length, + matches: response.payload.matches, + pagesWithNoTextCount: response.payload.pagesWithNoTextCount, + pagesWithSuspiciousTextCount: response.payload.pagesWithSuspiciousTextCount, + warnings: response.payload.warnings + } + } + }) + }), + { + name: PDF_SEARCH_TOOL_SPEC.name, + description: PDF_SEARCH_TOOL_SPEC.description, + schema: PDF_SEARCH_TOOL_SPEC.inputSchema + } + ) + + const readPagesTool = tool( + async (args) => + await withToolLifecycle({ + emitter: runtime.toolEvents, + toolName: PDF_READ_PAGES_TOOL_SPEC.name, + args: args as JsonObject, + execute: async ({ toolCallId }) => + await runTool({ + runtime, + toolName: PDF_READ_PAGES_TOOL_SPEC.name, + toolCallId, + sourcePath: args.sourcePath, + effectKind: 'read', + execute: async (document) => { + const selection = args.selection + let pageNumbers: number[] + if (selection.type === 'page_numbers') { + pageNumbers = uniquePageNumbers(selection.pageNumbers) + } else { + if (selection.fromPage > selection.toPage) { + throw new Error('fromPage 不能大于 toPage') + } + const pageCount = selection.toPage - selection.fromPage + 1 + if (pageCount > 50) throw new Error('单次最多读取 50 页') + pageNumbers = Array.from( + { length: pageCount }, + (_, index) => selection.fromPage + index + ) + } + if (pageNumbers.length === 0 || pageNumbers.length > 50) { + throw new Error('单次最多读取 50 页，且页码范围不能为空') + } + const response = await runtime.callWorker({ + action: 'read_pages', + payload: { docId: document.docId, pageNumbers } + }) + const resourceContext = runtime.buildResourceContext(document.sourcePath, 'read') + if (!response.ok) { + return runtime.emitWorkerFailure({ + toolName: PDF_READ_PAGES_TOOL_SPEC.name, + toolCallId, + response, + resourceContext + }) + } + if (response.payload.action !== 'read_pages') + throw new Error('PDF worker 协议错误（read_pages）') + return { + sourcePath: document.sourcePath, + pageCount: document.pageCount, + pages: response.payload.pages, + warnings: response.payload.pages.flatMap((page) => page.warnings) + } + } + }) + }), + { + name: PDF_READ_PAGES_TOOL_SPEC.name, + description: PDF_READ_PAGES_TOOL_SPEC.description, + schema: PDF_READ_PAGES_TOOL_SPEC.inputSchema + } + ) + + const renderPagesTool = tool( + async (args) => + await withToolLifecycle({ + emitter: runtime.toolEvents, + toolName: PDF_RENDER_PAGES_TOOL_SPEC.name, + args: args as JsonObject, + execute: async ({ toolCallId }) => + await runTool({ + runtime, + toolName: PDF_RENDER_PAGES_TOOL_SPEC.name, + toolCallId, + sourcePath: args.sourcePath, + effectKind: 'rendered', + execute: async (document) => { + const response = await runtime.callWorker({ + action: 'render_pages', + payload: { + docId: document.docId, + workspaceRootPath: runtime.workspaceRootPath, + pageNumbers: uniquePageNumbers(args.pageNumbers) + } + }) + const resourceContext = runtime.buildResourceContext(document.sourcePath, 'rendered') + if (!response.ok) { + return runtime.emitWorkerFailure({ + toolName: PDF_RENDER_PAGES_TOOL_SPEC.name, + toolCallId, + response, + resourceContext + }) + } + if (response.payload.action !== 'render_pages') + throw new Error('PDF worker 协议错误（render_pages）') + return { + sourcePath: document.sourcePath, + pageCount: document.pageCount, + assetBucketId: response.payload.assetBucketId, + images: response.payload.images, + warnings: response.payload.warnings + } + } + }) + }), + { + name: PDF_RENDER_PAGES_TOOL_SPEC.name, + description: PDF_RENDER_PAGES_TOOL_SPEC.description, + schema: PDF_RENDER_PAGES_TOOL_SPEC.inputSchema + } + ) + + return [inspectTool, searchTool, readPagesTool, renderPagesTool] +} diff --git a/src/main/services/ai/human/tools/pdf/types.ts b/src/main/services/ai/human/tools/pdf/types.ts new file mode 100644 index 00000000..d2a3456f --- /dev/null +++ b/src/main/services/ai/human/tools/pdf/types.ts @@ -0,0 +1,18 @@ +import type { z } from 'zod/v4' +import type { WorkspaceCapabilityBuildInput } from '../types' +import type { + pdfInspectSchema, + pdfReadPagesSchema, + pdfRenderPagesSchema, + pdfSearchSchema +} from './schemas' + +export type PdfCapabilityToolsInput = WorkspaceCapabilityBuildInput & { + safePreviewChars: number + modelContextWindowTokens: number +} + +export type PdfInspectRequest = z.infer +export type PdfSearchRequest = z.infer +export type PdfReadPagesRequest = z.infer +export type PdfRenderPagesRequest = z.infer diff --git a/src/main/services/ai/human/tools/registry.ts b/src/main/services/ai/human/tools/registry.ts index 4ae040bf..c06809f7 100644 --- a/src/main/services/ai/human/tools/registry.ts +++ b/src/main/services/ai/human/tools/registry.ts @@ -1,6 +1,7 @@ import { DOCX_TOOL_SPECS } from './docx/toolSpecs' import { IMAGE_RECOGNITION_TOOL_SPECS } from './imageRecognition/toolSpecs' import { KB_TOOL_SPECS } from './kb/toolSpecs' +import { PDF_TOOL_SPECS } from './pdf/toolSpecs' import { UTILITY_TOOL_SPECS } from './utility/toolSpecs' import { WEB_TOOL_SPECS } from './web/toolSpecs' import { @@ -17,6 +18,7 @@ export const ALL_TOOL_SPECS = [ ...TEXT_FILE_TOOL_SPECS, ...KB_TOOL_SPECS, ...DOCX_TOOL_SPECS, + ...PDF_TOOL_SPECS, ...WEB_TOOL_SPECS, ...UTILITY_TOOL_SPECS, ...IMAGE_RECOGNITION_TOOL_SPECS diff --git a/src/main/services/ai/human/types.ts b/src/main/services/ai/human/types.ts index 6574ec6e..e5b6f775 100644 --- a/src/main/services/ai/human/types.ts +++ b/src/main/services/ai/human/types.ts @@ -3,6 +3,7 @@ import type { JsonObject } from '../../../../shared/types' import type { z } from 'zod/v4' import type { AiToolFamilyId } from './tools/types' import type { DocxToolRuntime } from './tools/docx/runtime' +import type { PdfToolRuntime } from './tools/pdf/runtime' import { HUMAN_ROUND_EXECUTION_KINDS, HUMAN_SKILL_SOURCES, @@ -105,6 +106,7 @@ export type HumanRuntimeCache = { familyToolsets: Partial> skillToolsets: Partial> docxRuntime?: DocxToolRuntime + pdfRuntime?: PdfToolRuntime approvedTerminalCommandCounts?: Map approvedBrowserUseTaskIds?: Set toolCallCount: number diff --git a/src/main/services/assets/WorkspaceAssetService.ts b/src/main/services/assets/WorkspaceAssetService.ts new file mode 100644 index 00000000..f8064ba3 --- /dev/null +++ b/src/main/services/assets/WorkspaceAssetService.ts @@ -0,0 +1,63 @@ +import { randomUUID } from 'node:crypto' +import fs from 'node:fs/promises' +import path from 'node:path' + +const WORKSPACE_NCURATOR_DIR = '.ncurator' +const WORKSPACE_RUNTIME_DIR = 'runtime' +const WORKSPACE_ASSETS_DIR = 'assets' +const WORKSPACE_SCRIPTS_DIR = 'scripts' + +const normalizeWorkspaceRoot = (workspaceRootPath: string): string => { + const value = String(workspaceRootPath || '').trim() + if (!value) throw new Error('缺少工作区根目录') + return path.resolve(value) +} + +export const getWorkspaceRuntimeDir = (workspaceRootPath: string): string => + path.join( + normalizeWorkspaceRoot(workspaceRootPath), + WORKSPACE_NCURATOR_DIR, + WORKSPACE_RUNTIME_DIR + ) + +export const getWorkspaceAssetsRootDir = (workspaceRootPath: string): string => + path.join(getWorkspaceRuntimeDir(workspaceRootPath), WORKSPACE_ASSETS_DIR) + +export const getWorkspaceAssetBucketDir = ( + workspaceRootPath: string, + assetBucketId: string +): string => { + const normalizedBucketId = String(assetBucketId || '').trim() + if (!normalizedBucketId || path.basename(normalizedBucketId) !== normalizedBucketId) { + throw new Error('assetBucketId 无效') + } + return path.join(getWorkspaceAssetsRootDir(workspaceRootPath), normalizedBucketId) +} + +export const ensureWorkspaceRuntimeDirs = async (workspaceRootPath: string): Promise => { + const runtimeDir = getWorkspaceRuntimeDir(workspaceRootPath) + await Promise.all([ + fs.mkdir(path.join(runtimeDir, WORKSPACE_ASSETS_DIR), { recursive: true }), + fs.mkdir(path.join(runtimeDir, WORKSPACE_SCRIPTS_DIR), { recursive: true }) + ]) +} + +export const ensureWorkspaceAssetBucket = async ( + workspaceRootPath: string, + assetBucketId: string +): Promise => { + await ensureWorkspaceRuntimeDirs(workspaceRootPath) + const directoryPath = getWorkspaceAssetBucketDir(workspaceRootPath, assetBucketId) + await fs.mkdir(directoryPath, { recursive: true }) + return directoryPath +} + +export const createWorkspaceAssetBucket = async ( + workspaceRootPath: string +): Promise<{ assetBucketId: string; directoryPath: string }> => { + const assetBucketId = randomUUID() + return { + assetBucketId, + directoryPath: await ensureWorkspaceAssetBucket(workspaceRootPath, assetBucketId) + } +} diff --git a/src/main/services/attachments/AttachmentService.ts b/src/main/services/attachments/AttachmentService.ts index 4c68470d..118975ed 100644 --- a/src/main/services/attachments/AttachmentService.ts +++ b/src/main/services/attachments/AttachmentService.ts @@ -2,11 +2,7 @@ import fs from 'fs' import path from 'path' import { v4 as uuidv4 } from 'uuid' import type { ChatAttachment } from '../../../shared/types' - -const WORKSPACE_NCURATOR_DIR = '.ncurator' -const WORKSPACE_RUNTIME_DIR = 'runtime' -const WORKSPACE_ASSETS_DIR = 'assets' -const WORKSPACE_SCRIPTS_DIR = 'scripts' +import { ensureWorkspaceAssetBucket } from '../assets/WorkspaceAssetService' const SUPPORTED_IMAGE_EXTENSIONS = new Set([ 'png', @@ -18,7 +14,7 @@ const SUPPORTED_IMAGE_EXTENSIONS = new Set([ 'tif', 'tiff' ]) -const SUPPORTED_FILE_EXTENSIONS = new Set(['docx', 'md', 'json']) +const SUPPORTED_FILE_EXTENSIONS = new Set(['docx', 'pdf', 'md', 'json']) const MIME_BY_EXTENSION: Record = { png: 'image/png', @@ -31,6 +27,7 @@ const MIME_BY_EXTENSION: Record = { tiff: 'image/tiff', md: 'text/markdown', json: 'application/json', + pdf: 'application/pdf', docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' } @@ -85,38 +82,6 @@ const isSupportedAttachment = (input: { return SUPPORTED_FILE_EXTENSIONS.has(ext) } -const getWorkspaceRuntimeDir = (workspaceRootPath: string): string => - path.join( - path.resolve(String(workspaceRootPath || '').trim()), - WORKSPACE_NCURATOR_DIR, - WORKSPACE_RUNTIME_DIR - ) - -const getWorkspaceAssetsDir = (workspaceRootPath: string, assetBucketId: string): string => - path.join( - getWorkspaceRuntimeDir(workspaceRootPath), - WORKSPACE_ASSETS_DIR, - String(assetBucketId || '').trim() - ) - -const ensureWorkspaceAttachmentDirs = async ( - workspaceRootPath: string, - assetBucketId: string -): Promise => { - const rootPath = path.resolve(String(workspaceRootPath || '').trim()) - if (!rootPath) { - throw new Error('缺少工作区根目录，无法保存附件') - } - const assetDir = getWorkspaceAssetsDir(rootPath, assetBucketId) - await Promise.all([ - fs.promises.mkdir(assetDir, { recursive: true }), - fs.promises.mkdir(path.join(getWorkspaceRuntimeDir(rootPath), WORKSPACE_SCRIPTS_DIR), { - recursive: true - }) - ]) - return assetDir -} - const splitFileName = (name: string): { stem: string; extWithDot: string } => { const safeName = path.basename(String(name || 'attachment').trim() || 'attachment') const parsed = path.parse(safeName) @@ -192,10 +157,7 @@ export class AttachmentService { throw new Error('暂不支持该文件格式') } - const dirPath = await ensureWorkspaceAttachmentDirs( - input.workspaceRootPath, - input.assetBucketId - ) + const dirPath = await ensureWorkspaceAssetBucket(input.workspaceRootPath, input.assetBucketId) const targetPath = await resolveUniqueTargetPath(dirPath, input.name) await fs.promises.writeFile(targetPath, Buffer.from(input.bytes)) return this.createAttachmentFromPath({ @@ -229,10 +191,7 @@ export class AttachmentService { throw new Error('暂不支持该文件格式') } - const dirPath = await ensureWorkspaceAttachmentDirs( - input.workspaceRootPath, - input.assetBucketId - ) + const dirPath = await ensureWorkspaceAssetBucket(input.workspaceRootPath, input.assetBucketId) const targetPath = await resolveUniqueTargetPath(dirPath, displayName) await fs.promises.copyFile(sourcePath, targetPath) return this.createAttachmentFromPath({ diff --git a/src/main/services/pdf/textNormalizer.ts b/src/main/services/pdf/textNormalizer.ts new file mode 100644 index 00000000..696e6e80 --- /dev/null +++ b/src/main/services/pdf/textNormalizer.ts @@ -0,0 +1,126 @@ +type PdfTextItemLike = { + str: string + transform: number[] + width: number + height: number + hasEOL?: boolean +} + +const CJK_CHAR_PATTERN = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u +// JavaScript 没有等价的 Unicode 属性可一次表达这组 PDF 控制字符。 +// eslint-disable-next-line no-control-regex +const CONTROL_CHAR_PATTERN = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/gu +const PRIVATE_USE_PATTERN = /[\uE000-\uF8FF]/gu + +const isCjkBoundary = (left: string, right: string): boolean => + CJK_CHAR_PATTERN.test(left.slice(-1)) && CJK_CHAR_PATTERN.test(right.slice(0, 1)) + +const shouldInsertSpace = (input: { + previousText: string + nextText: string + previousEndX?: number + nextX: number + fontSize: number +}): boolean => { + if (!input.previousText || !input.nextText) return false + if (/\s$/u.test(input.previousText) || /^\s/u.test(input.nextText)) return false + if (isCjkBoundary(input.previousText, input.nextText)) return false + if (/^[,.;:!?，。；：！？、)\]}]/u.test(input.nextText)) return false + if (/[([{（【《“‘]$/u.test(input.previousText)) return false + if (typeof input.previousEndX !== 'number') return true + return input.nextX - input.previousEndX > Math.max(1, input.fontSize * 0.18) +} + +export const normalizePdfText = (value: string): string => { + const normalized = value + .normalize('NFKC') + .replace(/\r/gu, '') + .replace(CONTROL_CHAR_PATTERN, '') + .replace(/\u00a0/gu, ' ') + + return normalized + .split('\n') + .map((line) => line.trim().replace(/[\t ]+/gu, ' ')) + .filter(Boolean) + .join('\n') + .replace( + /([\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]) +(?=[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}])/gu, + '$1' + ) +} + +export const buildPdfPageText = ( + rawItems: readonly unknown[] +): { text: string; suspicious: boolean; warnings: string[] } => { + const items = rawItems.filter((item): item is PdfTextItemLike => + Boolean( + item && + typeof item === 'object' && + 'str' in item && + typeof (item as { str?: unknown }).str === 'string' && + 'transform' in item && + Array.isArray((item as { transform?: unknown }).transform) + ) + ) + const rawJoined = items.map((item) => item.str).join('') + const replacementCount = (rawJoined.match(/\uFFFD/gu) || []).length + const privateUseCount = (rawJoined.match(PRIVATE_USE_PATTERN) || []).length + const controlCount = (rawJoined.match(CONTROL_CHAR_PATTERN) || []).length + const suspiciousCount = replacementCount + privateUseCount + controlCount + const suspicious = rawJoined.length > 0 && suspiciousCount / rawJoined.length > 0.01 + + const lines: string[] = [] + let line = '' + let currentY: number | undefined + let previousEndX: number | undefined + let previousFontSize = 10 + + const flush = () => { + const normalizedLine = normalizePdfText(line) + if (normalizedLine) lines.push(normalizedLine) + line = '' + currentY = undefined + previousEndX = undefined + } + + for (const item of items) { + const text = item.str + if (!text) { + if (item.hasEOL) flush() + continue + } + const x = Number(item.transform[4] || 0) + const y = Number(item.transform[5] || 0) + const fontSize = Math.max(1, Math.abs(Number(item.transform[3] || item.height || 10))) + const isNewLine = + typeof currentY === 'number' && Math.abs(y - currentY) > Math.max(2, previousFontSize * 0.55) + if (isNewLine) flush() + if ( + shouldInsertSpace({ + previousText: line, + nextText: text, + previousEndX, + nextX: x, + fontSize + }) + ) { + line += ' ' + } + line += text + currentY = y + previousFontSize = fontSize + previousEndX = x + Math.max(0, Number(item.width || 0)) + if (item.hasEOL) flush() + } + flush() + + const warnings: string[] = [] + if (replacementCount > 0) warnings.push(`检测到 ${String(replacementCount)} 个替换字符`) + if (privateUseCount > 0) warnings.push(`检测到 ${String(privateUseCount)} 个私用区字符`) + if (controlCount > 0) warnings.push(`已清理 ${String(controlCount)} 个控制字符`) + return { + text: normalizePdfText(lines.join('\n')), + suspicious, + warnings + } +} diff --git a/src/main/services/pdf/types.ts b/src/main/services/pdf/types.ts new file mode 100644 index 00000000..a9be62bc --- /dev/null +++ b/src/main/services/pdf/types.ts @@ -0,0 +1,58 @@ +export type PdfDocumentMetadata = { + title?: string + author?: string + subject?: string + keywords?: string + creator?: string + producer?: string + creationDate?: string + modificationDate?: string +} + +export type PdfOutlineItem = { + title: string + pageNumber?: number + depth: number +} + +export type PdfOpenResult = { + docId: string + sourcePath: string + pageCount: number + metadata: PdfDocumentMetadata + outline: PdfOutlineItem[] + encrypted: boolean + warnings: string[] +} + +export type PdfPageTextStatus = 'available' | 'empty' | 'suspicious' + +export type PdfPageReadResult = { + pageNumber: number + text: string + charCount: number + textStatus: PdfPageTextStatus + imageCount: number + width: number + height: number + rotation: number + warnings: string[] +} + +export type PdfSearchMode = 'literal' | 'regex' + +export type PdfSearchMatch = { + pageNumber: number + matchedText: string + snippet: string + start: number + end: number +} + +export type PdfRenderedImage = { + pageNumber: number + path: string + mimeType: 'image/png' + width: number + height: number +} diff --git a/src/main/services/pdf/worker/pdfWorker.ts b/src/main/services/pdf/worker/pdfWorker.ts new file mode 100644 index 00000000..ad672e25 --- /dev/null +++ b/src/main/services/pdf/worker/pdfWorker.ts @@ -0,0 +1,380 @@ +import { randomUUID } from 'node:crypto' +import fs from 'node:fs/promises' +import { parentPort } from 'node:worker_threads' +import { createCanvas } from '@napi-rs/canvas' +import * as pdfjs from 'pdfjs-dist/legacy/build/pdf.mjs' +import type { PDFDocumentProxy, PDFPageProxy } from 'pdfjs-dist/types/src/display/api' +import { createWorkspaceAssetBucket } from '../../assets/WorkspaceAssetService' +import { buildPdfPageText, normalizePdfText } from '../textNormalizer' +import type { + PdfDocumentMetadata, + PdfOutlineItem, + PdfPageReadResult, + PdfSearchMatch +} from '../types' +import type { PdfWorkerRequest, PdfWorkerResponse } from './types' + +type PdfSnapshot = { + docId: string + sourcePath: string + document: PDFDocumentProxy + pageCount: number + pageByNumber: Map + pageTextByNumber: Map +} + +const runSessions = new Map>() + +const getRunSession = (runId: string, create = false): Map | undefined => { + const existing = runSessions.get(runId) + if (existing || !create) return existing + const next = new Map() + runSessions.set(runId, next) + return next +} + +const getSnapshot = (runId: string, docId: string): PdfSnapshot => { + const snapshot = getRunSession(runId)?.get(docId) + if (!snapshot) throw new Error(`pdf session not found: ${docId}`) + return snapshot +} + +const readInfoString = (info: Record, key: string): string | undefined => { + const value = info[key] + return typeof value === 'string' && value.trim() ? value.trim() : undefined +} + +const buildMetadata = (infoValue: unknown): PdfDocumentMetadata => { + const info = + infoValue && typeof infoValue === 'object' ? (infoValue as Record) : {} + return { + title: readInfoString(info, 'Title'), + author: readInfoString(info, 'Author'), + subject: readInfoString(info, 'Subject'), + keywords: readInfoString(info, 'Keywords'), + creator: readInfoString(info, 'Creator'), + producer: readInfoString(info, 'Producer'), + creationDate: readInfoString(info, 'CreationDate'), + modificationDate: readInfoString(info, 'ModDate') + } +} + +const resolveOutlinePageNumber = async ( + document: PDFDocumentProxy, + destination: string | unknown[] | null | undefined +): Promise => { + if (!destination) return undefined + const resolvedDestination = + typeof destination === 'string' ? await document.getDestination(destination) : destination + if (!Array.isArray(resolvedDestination) || !resolvedDestination[0]) return undefined + const first = resolvedDestination[0] + if (typeof first === 'number') return first + 1 + if (typeof first !== 'object') return undefined + try { + return ( + (await document.getPageIndex(first as Parameters[0])) + 1 + ) + } catch { + return undefined + } +} + +const buildOutline = async (document: PDFDocumentProxy): Promise => { + const rootItems = (await document.getOutline()) || [] + const output: PdfOutlineItem[] = [] + const visit = async (items: typeof rootItems, depth: number): Promise => { + for (const item of items) { + if (output.length >= 500) return + output.push({ + title: normalizePdfText(String(item.title || '')) || '未命名书签', + pageNumber: await resolveOutlinePageNumber(document, item.dest), + depth + }) + if (Array.isArray(item.items) && item.items.length > 0) { + await visit(item.items, depth + 1) + } + } + } + await visit(rootItems, 1) + return output +} + +const getPage = async (snapshot: PdfSnapshot, pageNumber: number): Promise => { + const cached = snapshot.pageByNumber.get(pageNumber) + if (cached) return cached + const page = await snapshot.document.getPage(pageNumber) + snapshot.pageByNumber.set(pageNumber, page) + return page +} + +const countPageImages = async (page: PDFPageProxy): Promise => { + try { + const operators = await page.getOperatorList() + const imageOps = new Set([ + pdfjs.OPS.paintImageXObject, + pdfjs.OPS.paintInlineImageXObject, + pdfjs.OPS.paintImageMaskXObject + ]) + return operators.fnArray.filter((operator) => imageOps.has(operator)).length + } catch { + return 0 + } +} + +const readPage = async (snapshot: PdfSnapshot, pageNumber: number): Promise => { + const cached = snapshot.pageTextByNumber.get(pageNumber) + if (cached) return cached + const page = await getPage(snapshot, pageNumber) + const viewport = page.getViewport({ scale: 1 }) + const content = await page.getTextContent() + const normalized = buildPdfPageText(content.items) + const result: PdfPageReadResult = { + pageNumber, + text: normalized.text, + charCount: normalized.text.length, + textStatus: normalized.text ? (normalized.suspicious ? 'suspicious' : 'available') : 'empty', + imageCount: await countPageImages(page), + width: Math.round(viewport.width), + height: Math.round(viewport.height), + rotation: viewport.rotation, + warnings: normalized.warnings + } + snapshot.pageTextByNumber.set(pageNumber, result) + return result +} + +const validatePageNumbers = (snapshot: PdfSnapshot, pageNumbers: number[]): void => { + for (const pageNumber of pageNumbers) { + if (!Number.isInteger(pageNumber) || pageNumber < 1 || pageNumber > snapshot.pageCount) { + throw new Error( + `pageNumber 超出范围：${String(pageNumber)}，有效范围是 1-${String(snapshot.pageCount)}` + ) + } + } +} + +const buildSnippet = (text: string, start: number, end: number): string => { + const from = Math.max(0, start - 80) + const to = Math.min(text.length, end + 80) + return text.slice(from, to).replace(/\s+/gu, ' ').trim() +} + +const findLiteralMatches = (text: string, query: string, pageNumber: number): PdfSearchMatch[] => { + const matches: PdfSearchMatch[] = [] + const haystack = text.toLocaleLowerCase() + const needle = query.toLocaleLowerCase() + let fromIndex = 0 + while (needle && fromIndex <= haystack.length) { + const start = haystack.indexOf(needle, fromIndex) + if (start < 0) break + const end = start + needle.length + matches.push({ + pageNumber, + matchedText: text.slice(start, end), + snippet: buildSnippet(text, start, end), + start, + end + }) + fromIndex = Math.max(end, start + 1) + } + return matches +} + +const findRegexMatches = (text: string, query: string, pageNumber: number): PdfSearchMatch[] => { + const expression = new RegExp(query, 'giu') + const matches: PdfSearchMatch[] = [] + for (const match of text.matchAll(expression)) { + const start = match.index ?? 0 + const matchedText = match[0] || '' + if (!matchedText) continue + const end = start + matchedText.length + matches.push({ + pageNumber, + matchedText, + snippet: buildSnippet(text, start, end), + start, + end + }) + } + return matches +} + +const renderPage = async (input: { + snapshot: PdfSnapshot + pageNumber: number + directoryPath: string +}) => { + const page = await getPage(input.snapshot, input.pageNumber) + const baseViewport = page.getViewport({ scale: 1 }) + const scale = Math.min(2, 3000 / Math.max(baseViewport.width, baseViewport.height)) + const viewport = page.getViewport({ scale }) + const width = Math.max(1, Math.ceil(viewport.width)) + const height = Math.max(1, Math.ceil(viewport.height)) + const canvas = createCanvas(width, height) + const canvasContext = canvas.getContext('2d') + await page.render({ + canvas: canvas as unknown as HTMLCanvasElement, + canvasContext: canvasContext as unknown as CanvasRenderingContext2D, + viewport + }).promise + const targetPath = `${input.directoryPath}/page-${String(input.pageNumber)}.png` + await fs.writeFile(targetPath, canvas.toBuffer('image/png')) + return { + pageNumber: input.pageNumber, + path: targetPath, + mimeType: 'image/png' as const, + width, + height + } +} + +const toPdfFailure = (requestId: string, error: unknown): PdfWorkerResponse => { + const message = error instanceof Error ? error.message : String(error) + const name = error instanceof Error ? error.name : '' + if (name === 'PasswordException' || /password/iu.test(message)) { + return { requestId, ok: false, code: 'password_required', error: 'PDF 需要密码才能读取' } + } + if (name === 'InvalidPDFException' || /invalid pdf/iu.test(message)) { + return { requestId, ok: false, code: 'invalid_pdf', error: 'PDF 文件无效或已损坏' } + } + if (/pageNumber 超出范围|pageNumbers|query|regex|regular expression/iu.test(message)) { + return { requestId, ok: false, code: 'invalid_input', error: message } + } + return { requestId, ok: false, code: 'failed', error: message } +} + +export const handlePdfWorkerRequest = async ( + request: PdfWorkerRequest +): Promise => { + try { + if (request.action === 'open') { + const bytes = new Uint8Array(await fs.readFile(request.payload.sourcePath)) + const document = await pdfjs.getDocument({ data: bytes }).promise + const docId = randomUUID() + const snapshot: PdfSnapshot = { + docId, + sourcePath: request.payload.sourcePath, + document, + pageCount: document.numPages, + pageByNumber: new Map(), + pageTextByNumber: new Map() + } + getRunSession(request.runId, true)?.set(docId, snapshot) + const metadata = await document.getMetadata().catch(() => ({ info: {} })) + return { + requestId: request.requestId, + ok: true, + payload: { + action: 'open', + docId, + sourcePath: snapshot.sourcePath, + pageCount: snapshot.pageCount, + metadata: buildMetadata(metadata.info), + outline: await buildOutline(document), + encrypted: false, + warnings: [] + } + } + } + + if (request.action === 'read_pages') { + const snapshot = getSnapshot(request.runId, request.payload.docId) + validatePageNumbers(snapshot, request.payload.pageNumbers) + return { + requestId: request.requestId, + ok: true, + payload: { + action: 'read_pages', + pages: await Promise.all( + request.payload.pageNumbers.map((page) => readPage(snapshot, page)) + ) + } + } + } + + if (request.action === 'search') { + const snapshot = getSnapshot(request.runId, request.payload.docId) + validatePageNumbers(snapshot, [request.payload.fromPage, request.payload.toPage]) + if (request.payload.fromPage > request.payload.toPage) { + throw new Error('fromPage 不能大于 toPage') + } + if (request.payload.mode === 'regex') void new RegExp(request.payload.query, 'giu') + const allMatches: PdfSearchMatch[] = [] + let pagesWithNoTextCount = 0 + let pagesWithSuspiciousTextCount = 0 + for ( + let pageNumber = request.payload.fromPage; + pageNumber <= request.payload.toPage; + pageNumber += 1 + ) { + const page = await readPage(snapshot, pageNumber) + if (page.textStatus === 'empty') pagesWithNoTextCount += 1 + if (page.textStatus === 'suspicious') pagesWithSuspiciousTextCount += 1 + allMatches.push( + ...(request.payload.mode === 'regex' + ? findRegexMatches(page.text, request.payload.query, pageNumber) + : findLiteralMatches(page.text, request.payload.query, pageNumber)) + ) + } + return { + requestId: request.requestId, + ok: true, + payload: { + action: 'search', + totalMatchCount: allMatches.length, + matches: allMatches.slice(0, request.payload.maxMatches), + pagesWithNoTextCount, + pagesWithSuspiciousTextCount, + warnings: [] + } + } + } + + if (request.action === 'render_pages') { + const snapshot = getSnapshot(request.runId, request.payload.docId) + validatePageNumbers(snapshot, request.payload.pageNumbers) + const bucket = await createWorkspaceAssetBucket(request.payload.workspaceRootPath) + const images: Array>> = [] + for (const pageNumber of request.payload.pageNumbers) { + images.push( + await renderPage({ + snapshot, + pageNumber, + directoryPath: bucket.directoryPath + }) + ) + } + return { + requestId: request.requestId, + ok: true, + payload: { + action: 'render_pages', + assetBucketId: bucket.assetBucketId, + images, + warnings: [] + } + } + } + + const released = getRunSession(request.runId)?.size || 0 + const session = getRunSession(request.runId) + if (session) { + await Promise.all([...session.values()].map((snapshot) => snapshot.document.destroy())) + } + runSessions.delete(request.runId) + return { + requestId: request.requestId, + ok: true, + payload: { action: 'release_run', releasedDocCount: released } + } + } catch (error) { + return toPdfFailure(request.requestId, error) + } +} + +const workerParentPort = parentPort +if (workerParentPort) { + workerParentPort.on('message', (request: PdfWorkerRequest) => { + void handlePdfWorkerRequest(request).then((response) => workerParentPort.postMessage(response)) + }) +} diff --git a/src/main/services/pdf/worker/pdfWorkerClient.ts b/src/main/services/pdf/worker/pdfWorkerClient.ts new file mode 100644 index 00000000..99106a48 --- /dev/null +++ b/src/main/services/pdf/worker/pdfWorkerClient.ts @@ -0,0 +1,103 @@ +import { randomUUID } from 'node:crypto' +import fs from 'node:fs' +import { Worker } from 'node:worker_threads' +import { fileURLToPath } from 'node:url' +import { handlePdfWorkerRequest } from './pdfWorker' +import type { + PdfWorkerAction, + PdfWorkerRequest, + PdfWorkerResponse, + PendingPdfWorkerRequest +} from './types' + +const DEFAULT_TIMEOUT_MS = 45_000 + +class PdfWorkerClient { + private worker: Worker | null = null + private readonly pending = new Map() + private inProcessMode = false + private readonly workerUrl = new URL('./pdfWorker.js', import.meta.url) + + private ensureWorker(): Worker { + if (this.inProcessMode) throw new Error('pdf worker is running in-process') + if (this.worker) return this.worker + const workerPath = fileURLToPath(this.workerUrl) + if (!fs.existsSync(workerPath)) { + this.inProcessMode = true + throw new Error(`pdf worker entry not found: ${workerPath}`) + } + const worker = new Worker(this.workerUrl, { name: 'pdf-worker' }) + worker.on('message', (response: PdfWorkerResponse) => { + const pending = this.pending.get(response.requestId) + if (!pending) return + clearTimeout(pending.timeout) + this.pending.delete(response.requestId) + pending.resolve(response) + }) + worker.on('error', (error) => { + this.failPending(new Error(`pdf worker error: ${error.message}`)) + this.worker = null + }) + worker.on('exit', (code) => { + if (code !== 0) this.failPending(new Error(`pdf worker exited with code ${String(code)}`)) + this.worker = null + }) + this.worker = worker + return worker + } + + private failPending(error: Error): void { + for (const pending of this.pending.values()) { + clearTimeout(pending.timeout) + pending.reject(error) + } + this.pending.clear() + } + + public async call( + input: Omit, 'requestId'>, + timeoutMs = DEFAULT_TIMEOUT_MS + ): Promise { + const request = { ...input, requestId: randomUUID() } as Extract< + PdfWorkerRequest, + { action: TAction } + > + if (this.inProcessMode) return await handlePdfWorkerRequest(request) + + let worker: Worker + try { + worker = this.ensureWorker() + } catch { + this.inProcessMode = true + return await handlePdfWorkerRequest(request) + } + + return await new Promise((resolve, reject) => { + const timeout = setTimeout( + () => { + this.pending.delete(request.requestId) + reject(new Error(`pdf worker timeout (${input.action})`)) + }, + Math.max(1000, timeoutMs) + ) + this.pending.set(request.requestId, { resolve, reject, timeout }) + worker.postMessage(request) + }) + } +} + +let singleton: PdfWorkerClient | null = null + +export const getPdfWorkerClient = (): PdfWorkerClient => { + if (!singleton) singleton = new PdfWorkerClient() + return singleton +} + +export const releasePdfRunSession = async (runId: string): Promise => { + if (!singleton) return + try { + await singleton.call({ runId, action: 'release_run', payload: {} }) + } catch { + // ignore release failures + } +} diff --git a/src/main/services/pdf/worker/types.ts b/src/main/services/pdf/worker/types.ts new file mode 100644 index 00000000..b1bfd50d --- /dev/null +++ b/src/main/services/pdf/worker/types.ts @@ -0,0 +1,92 @@ +import type { + PdfOpenResult, + PdfPageReadResult, + PdfRenderedImage, + PdfSearchMatch, + PdfSearchMode +} from '../types' + +export type PdfWorkerAction = 'open' | 'read_pages' | 'search' | 'render_pages' | 'release_run' + +export type PdfWorkerRequest = + | { + requestId: string + runId: string + action: 'open' + payload: { sourcePath: string } + } + | { + requestId: string + runId: string + action: 'read_pages' + payload: { docId: string; pageNumbers: number[] } + } + | { + requestId: string + runId: string + action: 'search' + payload: { + docId: string + query: string + mode: PdfSearchMode + fromPage: number + toPage: number + maxMatches: number + } + } + | { + requestId: string + runId: string + action: 'render_pages' + payload: { + docId: string + workspaceRootPath: string + pageNumbers: number[] + } + } + | { + requestId: string + runId: string + action: 'release_run' + payload: Record + } + +export type PdfWorkerSuccessPayload = + | ({ action: 'open' } & PdfOpenResult) + | { action: 'read_pages'; pages: PdfPageReadResult[] } + | { + action: 'search' + totalMatchCount: number + matches: PdfSearchMatch[] + pagesWithNoTextCount: number + pagesWithSuspiciousTextCount: number + warnings: string[] + } + | { + action: 'render_pages' + assetBucketId: string + images: PdfRenderedImage[] + warnings: string[] + } + | { action: 'release_run'; releasedDocCount: number } + +export type PdfWorkerResponse = + | { requestId: string; ok: true; payload: PdfWorkerSuccessPayload } + | { + requestId: string + ok: false + code: + | 'invalid_input' + | 'invalid_pdf' + | 'password_required' + | 'encrypted_pdf_unsupported' + | 'failed' + error: string + warnings?: string[] + } + +export type PendingPdfWorkerRequest = { + resolve: (response: PdfWorkerResponse) => void + reject: (reason: Error) => void + timeout: NodeJS.Timeout +} diff --git a/src/renderer/src/components/workbench/WorkbenchComposer.tsx b/src/renderer/src/components/workbench/WorkbenchComposer.tsx index dfb6ef65..ca37968e 100644 --- a/src/renderer/src/components/workbench/WorkbenchComposer.tsx +++ b/src/renderer/src/components/workbench/WorkbenchComposer.tsx @@ -92,7 +92,7 @@ type WorkbenchComposerProps = { const isTextNode = (value: Node | null): value is Text => value instanceof Text const CARET_ANCHOR_ROLE = 'composer-caret-anchor' const LARGE_FILE_WARNING_BYTES = 10 * 1024 * 1024 -const SUPPORTED_ATTACHMENT_FILE_EXTENSIONS = new Set(['md', 'json', 'docx']) +const SUPPORTED_ATTACHMENT_FILE_EXTENSIONS = new Set(['md', 'json', 'docx', 'pdf']) const getAttachmentFileExt = (name: string): string => { const last = diff --git a/src/renderer/src/components/workbench/types.ts b/src/renderer/src/components/workbench/types.ts index c03285a1..6a2c6fad 100644 --- a/src/renderer/src/components/workbench/types.ts +++ b/src/renderer/src/components/workbench/types.ts @@ -100,6 +100,10 @@ export type AssistantToolName = | 'docx_search' | 'docx_read_blocks' | 'docx_edit' + | 'pdf_inspect' + | 'pdf_search' + | 'pdf_read_pages' + | 'pdf_render_pages' | 'web_search' | 'web_crawl' | 'browser_use' @@ -137,6 +141,7 @@ export type AssistantProcessSkillKey = | 'docx-edit' | 'file-read' | 'file-write' + | 'pdf-read' | 'kb' | 'terminal' | 'web-search' diff --git a/src/renderer/src/components/workbench/utils.ts b/src/renderer/src/components/workbench/utils.ts index 2242b9f8..286bd294 100644 --- a/src/renderer/src/components/workbench/utils.ts +++ b/src/renderer/src/components/workbench/utils.ts @@ -80,6 +80,10 @@ const isAssistantToolName = (toolName: string): toolName is AssistantToolName => toolName === 'docx_search' || toolName === 'docx_read_blocks' || toolName === 'docx_edit' || + toolName === 'pdf_inspect' || + toolName === 'pdf_search' || + toolName === 'pdf_read_pages' || + toolName === 'pdf_render_pages' || toolName === 'web_search' || toolName === 'web_crawl' || toolName === 'browser_use' || @@ -208,6 +212,7 @@ const getSkillDisplayName = (skillId?: RuntimeSkillId): string => { if (skillId === 'kb-retrieval') return '资料查阅器' if (skillId === 'docx-read') return 'docx 阅读器' if (skillId === 'docx-write') return 'docx 写作器' + if (skillId === 'pdf-read') return 'PDF 阅读器' if (skillId === 'image-recognition') return '图像识别' return '新能力' } @@ -223,6 +228,14 @@ const getToolUiKind = (toolName?: AssistantToolName): AssistantProcessSkillKey = if (toolName === 'docx_edit') return 'docx-edit' if (toolName === 'file_read') return 'file-read' if (toolName === 'file_write') return 'file-write' + if ( + toolName === 'pdf_inspect' || + toolName === 'pdf_search' || + toolName === 'pdf_read_pages' || + toolName === 'pdf_render_pages' + ) { + return 'pdf-read' + } if (toolName === 'kb_search' || toolName === 'kb_list_documents') return 'kb' if (toolName === 'terminal_run' || toolName === 'terminal_read' || toolName === 'terminal_stop') { return 'terminal' @@ -242,7 +255,7 @@ const getDisplaySkillKey = (display: AssistantProcessDisplay): AssistantProcessS const getToolIconKind = (skillKey: AssistantProcessSkillKey): AssistantProcessIconKind => { if (skillKey === 'docx-edit') return 'edit' - if (skillKey === 'docx-read') return 'document' + if (skillKey === 'docx-read' || skillKey === 'pdf-read') return 'document' if (skillKey === 'file-read') return 'document' if (skillKey === 'file-write') return 'edit' if (skillKey === 'kb') return 'knowledge' @@ -317,6 +330,13 @@ const getToolSummaryText = ( } return sentence(text(`编辑 ${normalizedCount} 个文件失败`)) } + if (skillKey === 'pdf-read') { + if (status === 'running') return sentence(text(`正在读取 ${normalizedCount} 个 PDF 文件`)) + if (status === 'completed') return sentence(text(`已读取 ${normalizedCount} 个 PDF 文件`)) + if (status === 'partial_failed') + return sentence(text(`部分读取 ${normalizedCount} 个 PDF 文件失败`)) + return sentence(text(`读取 ${normalizedCount} 个 PDF 文件失败`)) + } if (skillKey === 'kb') { if (status === 'running') return sentence(text('正在检索知识库')) if (status === 'completed') return sentence(text('已检索知识库')) @@ -394,6 +414,15 @@ const getToolItemText = ( if (toolName === 'file_write') { return sentence(text('编辑文件 '), text(fileLabel)) } + if (toolName === 'pdf_inspect') { + return sentence(text('查看 PDF 信息 '), text(fileLabel)) + } + if (toolName === 'pdf_read_pages') { + return sentence(text('读取 PDF 页面 '), text(fileLabel)) + } + if (toolName === 'pdf_render_pages') { + return sentence(text('渲染 PDF 页面 '), text(fileLabel)) + } if (toolName === 'terminal_run') { return sentence(text('执行命令 '), text(commandLabel)) } @@ -406,9 +435,9 @@ const getToolItemText = ( if (toolName === 'terminal_stop') { return sentence(text('停止终端 '), text(readStringValue(input || {}, 'terminalId') || '任务')) } - if (toolName === 'kb_search' || toolName === 'docx_search') { + if (toolName === 'kb_search' || toolName === 'docx_search' || toolName === 'pdf_search') { return sentence( - text(toolName === 'docx_search' ? '在文档中搜索 ' : '在知识库中搜索 '), + text(toolName === 'kb_search' ? '在知识库中搜索 ' : '在文档中搜索 '), text(queryLabel) ) } diff --git a/src/shared/types.ts b/src/shared/types.ts index ccb77fe8..6bdcd95b 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -821,6 +821,7 @@ export type RuntimeSkillId = | 'text-file' | 'docx-read' | 'docx-write' + | 'pdf-read' | 'image-recognition' | 'kb-retrieval' | 'web-browsing' diff --git a/tests/case/ai/human/live/human-pdf-visual-reading-live.test.ts b/tests/case/ai/human/live/human-pdf-visual-reading-live.test.ts new file mode 100644 index 00000000..e6081304 --- /dev/null +++ b/tests/case/ai/human/live/human-pdf-visual-reading-live.test.ts @@ -0,0 +1,180 @@ +import { createHash } from 'node:crypto' +import fs from 'node:fs/promises' +import path from 'node:path' +import { describe, it } from 'vitest' +import { + hasLiveLlmConfigInEnv, + loadLiveLlmConfigFromEnv, + loadLiveVisionLlmConfigFromEnv +} from '../../../../harness/backend/llmConfig' +import { createHumanTestRunner } from '../../../../harness/backend/humanHarness' +import { runBackendCase } from '../../../../harness/backend/run-backend-case' +import { + buildSelectedPdfFixtureReference, + HUMAN_PDF_VISUAL_READING_USER_INPUT, + PDF_READING_FIXTURE_PATH +} from '../../../../harness/backend/aiLiveTestData' +import { filterAiRunEventsByType, runWithStructuredOutputRetry } from '../testUtils' +import type { JsonObject } from '../../../../../src/shared/types' + +const fileHash = async (filePath: string): Promise => + createHash('sha256') + .update(await fs.readFile(filePath)) + .digest('hex') + +const readRenderedImagePaths = (data: unknown): string[] => { + if (!data || typeof data !== 'object' || Array.isArray(data)) return [] + const images = (data as JsonObject).images + if (!Array.isArray(images)) return [] + return images + .map((image) => + image && typeof image === 'object' && !Array.isArray(image) && typeof image.path === 'string' + ? image.path + : '' + ) + .filter(Boolean) +} + +describe('ai human live backend', () => { + it('human-pdf-visual-reading-live', async () => { + await runWithStructuredOutputRetry(async () => { + await runBackendCase({ + caseId: 'human-pdf-visual-reading-live', + harnessMode: 'live', + domain: 'ai', + suite: 'human', + groups: ['live'], + description: + '验证 live 模型能自主组合 PDF 文本读取、页面渲染和通用图像识别，回答图文结合问题。', + fixtures: [PDF_READING_FIXTURE_PATH], + runMode: 'single', + runtimeModelName: hasLiveLlmConfigInEnv() ? loadLiveLlmConfigFromEnv().model_name : null, + execute: async (runtime) => { + if (!hasLiveLlmConfigInEnv()) { + return { skipped: true, output: null, events: [], beforeHash: '', afterHash: '' } + } + + const primaryConfig = loadLiveLlmConfigFromEnv() + const visionConfig = loadLiveVisionLlmConfigFromEnv() + await runtime.seedLlmConfig(primaryConfig) + await runtime.seedLlmConfig(visionConfig) + const reference = buildSelectedPdfFixtureReference(runtime) + const beforeHash = await fileHash(reference.absolutePath || '') + const runner = createHumanTestRunner(runtime, { + selectedReferences: [reference], + visionModel: { + configId: visionConfig.id, + modelName: visionConfig.model_name + }, + allowedToolNames: [ + 'pdf_inspect', + 'pdf_search', + 'pdf_read_pages', + 'pdf_render_pages', + 'llm_recognize' + ] + }) + const output = await runner.run({ + taskId: runtime.taskId, + userInput: HUMAN_PDF_VISUAL_READING_USER_INPUT + }) + return { + skipped: false, + output, + events: runtime.eventCollector.snapshot(), + beforeHash, + afterHash: await fileHash(reference.absolutePath || '') + } + }, + assert: async (result, runtime) => { + if (result.skipped) { + runtime.assertThat(true, { + name: 'live-config-not-provided', + message: '未提供 live 模型配置，已跳过 human PDF 图文阅读 live case。' + }) + return + } + + const starts = filterAiRunEventsByType(result.events, 'tool_call_started') + const results = filterAiRunEventsByType(result.events, 'tool_call_result') + const toolNames = starts.map((event) => event.toolName) + const renderPaths = results + .filter((event) => event.toolName === 'pdf_render_pages' && event.result?.ok === true) + .flatMap((event) => readRenderedImagePaths(event.result?.data)) + const recognizePaths = starts + .filter((event) => event.toolName === 'llm_recognize') + .map((event) => event.input.path) + .filter((value): value is string => typeof value === 'string') + const successfulRecognizeCallIds = new Set( + results + .filter((event) => event.toolName === 'llm_recognize' && event.result?.ok === true) + .map((event) => event.toolCallId) + ) + const successfulRecognizePaths = starts + .filter( + (event) => + event.toolName === 'llm_recognize' && + successfulRecognizeCallIds.has(event.toolCallId) + ) + .map((event) => event.input.path) + .filter((value): value is string => typeof value === 'string') + const failedRecognizeCount = results.filter( + (event) => event.toolName === 'llm_recognize' && event.result?.ok === false + ).length + const delivery = String(result.output?.deliveryMarkdown || '') + const page9Path = renderPaths.find( + (imagePath) => path.basename(imagePath) === 'page-9.png' + ) + const expectedAssetRoot = path.join( + runtime.workspaceRoot, + '.ncurator', + 'runtime', + 'assets' + ) + + runtime.assertThat( + result.output?.status === 'completed' && + toolNames.includes('pdf_read_pages') && + toolNames.includes('pdf_render_pages') && + toolNames.includes('llm_recognize') && + Boolean(page9Path) && + recognizePaths.some((imagePath) => renderPaths.includes(imagePath)) && + successfulRecognizePaths.some( + (imagePath) => path.basename(imagePath) === 'page-9.png' + ) && + failedRecognizeCount === 0 && + Boolean( + page9Path && path.relative(expectedAssetRoot, page9Path).split(path.sep).length >= 2 + ) && + !toolNames.some( + (name) => + name === 'terminal_run' || name.startsWith('kb_') || name.startsWith('docx_') + ) && + /(inputs|输入)/i.test(delivery) && + /(parameters|参数)/i.test(delivery) && + /(loss|损失)/i.test(delivery) && + /(update|更新)/i.test(delivery) && + /Proper fit/i.test(delivery) && + /Overfit/i.test(delivery) && + /(训练.*验证|验证.*训练|泛化|间距|差距)/i.test(delivery) && + !/(无法|不能|不支持).{0,12}(看到|识别|读取).{0,8}(图片|图像|曲线)/i.test(delivery) && + result.beforeHash === result.afterHash, + { + name: 'human-live-composes-pdf-render-and-image-recognition', + message: + 'live human 应自主组合 PDF 阅读、页面渲染和图像识别，回答训练闭环及过度拟合图形差异。', + data: { + toolNames, + renderPaths, + recognizePaths, + successfulRecognizePaths, + failedRecognizeCount, + deliveryMarkdown: result.output?.deliveryMarkdown + } + } + ) + } + }) + }) + }) +}) diff --git a/tests/case/ai/human/mock-llm/human-pdf-render-image-recognition-collaboration.test.ts b/tests/case/ai/human/mock-llm/human-pdf-render-image-recognition-collaboration.test.ts new file mode 100644 index 00000000..f08eb2d8 --- /dev/null +++ b/tests/case/ai/human/mock-llm/human-pdf-render-image-recognition-collaboration.test.ts @@ -0,0 +1,207 @@ +import { createServer } from 'node:http' +import type { AddressInfo, Socket } from 'node:net' +import path from 'node:path' +import { describe, it } from 'vitest' +import { createHumanTestRunner } from '../../../../harness/backend/humanHarness' +import { runBackendCase } from '../../../../harness/backend/run-backend-case' +import { + buildSelectedPdfFixtureReference, + PDF_READING_FIXTURE_PATH +} from '../../../../harness/backend/aiLiveTestData' +import { + buildCompleteTaskDecision, + buildLearnSkillsDecision, + buildToolDecision, + filterAiRunEventsByType +} from '../testUtils' +import type { JsonObject } from '../../../../../src/shared/types' + +const startVisionServer = async (): Promise<{ baseUrl: string; close: () => Promise }> => { + const sockets = new Set() + const server = createServer((request, response) => { + request.resume() + response.writeHead(200, { 'content-type': 'application/json; charset=utf-8' }) + response.end( + JSON.stringify({ + id: 'chatcmpl-pdf-vision', + object: 'chat.completion', + created: Math.floor(Date.now() / 1000), + model: 'gpt-5', + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: JSON.stringify({ + supports_image_recognition: true, + result: '第9页右侧 Overfit 图中训练曲线继续改善，但验证曲线变差，二者差距明显。' + }) + }, + finish_reason: 'stop' + } + ] + }) + ) + }) + server.on('connection', (socket) => { + sockets.add(socket) + socket.on('close', () => sockets.delete(socket)) + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', () => resolve()) + }) + const address = server.address() as AddressInfo + return { + baseUrl: `http://127.0.0.1:${String(address.port)}/v1`, + close: async () => { + sockets.forEach((socket) => socket.destroy()) + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())) + ) + } + } +} + +const renderedPathsFromData = (data: unknown): string[] => { + if (!data || typeof data !== 'object' || Array.isArray(data)) return [] + const images = (data as JsonObject).images + if (!Array.isArray(images)) return [] + return images.flatMap((image) => { + if (!image || typeof image !== 'object' || Array.isArray(image)) return [] + return typeof image.path === 'string' ? [image.path] : [] + }) +} + +describe('ai human mock-llm backend', () => { + it('human-pdf-render-image-recognition-collaboration', async () => { + await runBackendCase({ + caseId: 'human-pdf-render-image-recognition-collaboration', + harnessMode: 'mock-llm', + domain: 'ai', + suite: 'human', + groups: ['mock-llm'], + description: '验证 PDF 页面图片路径会回流给 brain，并可继续交给通用图像识别工具。', + fixtures: [PDF_READING_FIXTURE_PATH], + runMode: 'single', + execute: async (runtime) => { + const visionServer = await startVisionServer() + try { + await runtime.seedLlmConfig({ + id: 'mock-llm-scripted', + name: 'PDF 图像识别测试配置', + providerKind: 'openai-compatible', + base_url: visionServer.baseUrl, + model_name: 'gpt-5', + api_key: 'test-key' + }) + const reference = buildSelectedPdfFixtureReference(runtime) + const runner = createHumanTestRunner(runtime, { + modelName: 'gpt-5', + selectedReferences: [reference], + allowedToolNames: ['pdf_read_pages', 'pdf_render_pages', 'llm_recognize'], + structuredOutputScript: { + 'human.brain': [ + { + output: buildLearnSkillsDecision({ + skillIds: ['pdf-read', 'image-recognition'], + summary: '补齐 PDF 阅读和图像识别能力。' + }) + }, + { + output: { + thought: { next: '读取文本并渲染需要观察的页面。' }, + nextAction: { + action: 'hand', + calls: [ + { + toolName: 'pdf_read_pages', + args: { + sourcePath: reference.absolutePath, + selection: { type: 'page_numbers', pageNumbers: [1, 9] } + } + }, + { + toolName: 'pdf_render_pages', + args: { + sourcePath: reference.absolutePath, + pageNumbers: [9] + } + } + ] + } + } + }, + ({ user }) => { + const imagePath = user.match( + /[^\s"<>]*\.ncurator\/runtime\/assets\/[^/\s"<>]+\/page-9\.png/u + )?.[0] + if (!imagePath) + throw new Error('rendered page-9 image path missing from brain prompt') + return buildToolDecision({ + toolName: 'llm_recognize', + args: { + path: imagePath, + instruction: '说明 Proper fit 和 Overfit 两组曲线的直观差异。' + }, + summary: '识别第9页对比图。' + }) + }, + { + output: buildCompleteTaskDecision({ + summary: 'PDF 文本和图片信息都已读取。' + }) + } + ], + 'human.mouth': [ + { + output: { + summary: '已完成 PDF 图文阅读。', + deliveryMarkdown: + '训练闭环由 inputs、parameters、architecture、predictions、labels、loss 和 update 组成；Overfit 时训练与验证表现差距明显。' + } + } + ] + } + }) + const output = await runner.run({ + taskId: runtime.taskId, + userInput: '结合第1页文字和第9页图形说明训练闭环与过度拟合。' + }) + return { output, events: runtime.eventCollector.snapshot() } + } finally { + await visionServer.close() + } + }, + assert: async (result, runtime) => { + const starts = filterAiRunEventsByType(result.events, 'tool_call_started') + const toolResults = filterAiRunEventsByType(result.events, 'tool_call_result') + const renderPaths = toolResults + .filter((event) => event.toolName === 'pdf_render_pages' && event.result?.ok === true) + .flatMap((event) => renderedPathsFromData(event.result?.data)) + const recognizePaths = starts + .filter((event) => event.toolName === 'llm_recognize') + .map((event) => event.input.path) + .filter((value): value is string => typeof value === 'string') + + runtime.assertThat( + result.output.status === 'completed' && + starts.some((event) => event.toolName === 'pdf_read_pages') && + renderPaths.some((imagePath) => path.basename(imagePath) === 'page-9.png') && + recognizePaths.some((imagePath) => renderPaths.includes(imagePath)) && + toolResults.some( + (event) => + event.toolName === 'llm_recognize' && + event.result?.ok === true && + JSON.stringify(event.result.data).includes('验证曲线') + ), + { + name: 'pdf-render-path-feeds-generic-image-recognition', + message: 'PDF 渲染结果中的真实图片路径应进入下一轮 brain，并交给 llm_recognize。', + data: { renderPaths, recognizePaths } + } + ) + } + }) + }) +}) diff --git a/tests/case/ai/tool/pdf/pdf-read-contract.test.ts b/tests/case/ai/tool/pdf/pdf-read-contract.test.ts new file mode 100644 index 00000000..6aadad01 --- /dev/null +++ b/tests/case/ai/tool/pdf/pdf-read-contract.test.ts @@ -0,0 +1,282 @@ +import { createHash } from 'node:crypto' +import fs from 'node:fs/promises' +import path from 'node:path' +import { describe, it } from 'vitest' +import { createPdfToolHarness } from '../../../../harness/backend/aiToolHarness' +import { runBackendCase } from '../../../../harness/backend/run-backend-case' + +const PDF_FIXTURE = path.resolve('tests/fixtures/workspace/学习笔记/深度学习.pdf') + +type PdfFailure = { ok: false; error: string; reason: string } +type PdfInspectResult = { + sourcePath: string + pageCount: number + metadata: { title?: string } + warnings: string[] +} +type PdfReadResult = { + sourcePath: string + pageCount: number + pages: Array<{ + pageNumber: number + text: string + textStatus: 'available' | 'empty' | 'suspicious' + imageCount: number + }> +} +type PdfSearchResult = { + totalMatchCount: number + returnedMatchCount: number + matches: Array<{ pageNumber: number; matchedText: string; snippet: string }> +} +type PdfRenderResult = { + sourcePath: string + pageCount: number + assetBucketId: string + images: Array<{ + pageNumber: number + path: string + mimeType: 'image/png' + width: number + height: number + }> +} + +const sha256 = async (filePath: string): Promise => + createHash('sha256') + .update(await fs.readFile(filePath)) + .digest('hex') + +describe('ai tool pdf backend', () => { + it('pdf-inspect-read-search-contract', async () => { + await runBackendCase({ + caseId: 'pdf-inspect-read-search-contract', + harnessMode: 'tool-real', + domain: 'ai', + suite: 'tool', + groups: ['pdf'], + description: '验证真实 PDF 的元数据、页级正文和搜索结果可稳定读取。', + fixtures: [PDF_FIXTURE], + runMode: 'single', + execute: async (runtime) => { + const sourcePath = runtime.getFixtureWorkspacePath(PDF_FIXTURE) + const harness = createPdfToolHarness(runtime) + try { + const inspect = await harness.invoke('pdf_inspect', { sourcePath }) + const read = await harness.invoke('pdf_read_pages', { + sourcePath, + selection: { type: 'page_numbers', pageNumbers: [1, 9] } + }) + const search = await harness.invoke('pdf_search', { + sourcePath, + query: '过度拟合', + mode: 'literal', + maxMatches: 20 + }) + return { inspect, read, search } + } finally { + await harness.release() + } + }, + assert: async (result, runtime) => { + runtime.assertThat( + result.inspect.pageCount === 11 && result.inspect.metadata.title === '深度学习', + { + name: 'pdf-inspect-returns-real-metadata', + message: 'pdf_inspect 应返回 fixture 的真实页数和标题。', + data: result.inspect + } + ) + runtime.assertThat( + result.read.pages.length === 2 && + result.read.pages[0]?.pageNumber === 1 && + /architecture/i.test(result.read.pages[0]?.text || '') && + !result.read.pages.some((page) => page.text.includes('\u0000')), + { + name: 'pdf-read-pages-returns-normalized-text', + message: 'pdf_read_pages 应按页返回无 NUL 的真实文本层。', + data: result.read + } + ) + runtime.assertThat( + result.search.totalMatchCount > 0 && + result.search.matches.some((match) => match.pageNumber === 8 || match.pageNumber === 9), + { + name: 'pdf-search-locates-overfitting-pages', + message: 'pdf_search 应在第 8 或第 9 页找到“过度拟合”。', + data: result.search + } + ) + } + }) + }) + + it('pdf-render-pages-runtime-assets-contract', async () => { + await runBackendCase({ + caseId: 'pdf-render-pages-runtime-assets-contract', + harnessMode: 'tool-real', + domain: 'ai', + suite: 'tool', + groups: ['pdf'], + description: '验证 PDF 页面图片写入 runtime assets，且不修改源 PDF、不产生普通 artifact。', + fixtures: [PDF_FIXTURE], + runMode: 'single', + execute: async (runtime) => { + const sourcePath = runtime.getFixtureWorkspacePath(PDF_FIXTURE) + const beforeHash = await sha256(sourcePath) + const harness = createPdfToolHarness(runtime) + try { + const render = await harness.invoke('pdf_render_pages', { + sourcePath, + pageNumbers: [1, 9] + }) + const headers = await Promise.all( + render.images.map(async (image) => + (await fs.readFile(image.path)).subarray(0, 8).toString('hex') + ) + ) + return { + render, + headers, + beforeHash, + afterHash: await sha256(sourcePath), + artifactCount: runtime.eventCollector + .snapshot() + .filter((event) => event.type === 'file_artifact').length + } + } finally { + await harness.release() + } + }, + assert: async (result, runtime) => { + const expectedParent = path.join( + runtime.workspaceRoot, + '.ncurator', + 'runtime', + 'assets', + result.render.assetBucketId + ) + runtime.assertThat( + result.render.images.length === 2 && + result.render.images.every((image) => path.dirname(image.path) === expectedParent) && + result.render.images.map((image) => path.basename(image.path)).join(',') === + 'page-1.png,page-9.png' && + result.headers.every((header) => header === '89504e470d0a1a0a'), + { + name: 'pdf-render-writes-valid-runtime-assets', + message: '页面图片应写入 .ncurator/runtime/assets/，并且是有效 PNG。', + data: result.render + } + ) + runtime.assertThat(result.beforeHash === result.afterHash && result.artifactCount === 0, { + name: 'pdf-render-does-not-mutate-source-or-emit-artifact', + message: '渲染页面不能修改源 PDF，也不能产生普通 file artifact。', + data: { artifactCount: result.artifactCount } + }) + } + }) + }) + + it('pdf-failure-boundary-contract', async () => { + await runBackendCase({ + caseId: 'pdf-failure-boundary-contract', + harnessMode: 'tool-real', + domain: 'ai', + suite: 'tool', + groups: ['pdf'], + description: '验证越界页码、非法 regex、伪 PDF 和逃逸工作区的符号链接会明确失败。', + fixtures: [PDF_FIXTURE], + runMode: 'single', + execute: async (runtime) => { + const sourcePath = runtime.getFixtureWorkspacePath(PDF_FIXTURE) + const corruptPath = path.join(runtime.workspaceRoot, 'corrupt.pdf') + const outsidePath = path.join(path.dirname(runtime.workspaceRoot), 'outside-pdf.pdf') + const symlinkPath = path.join(runtime.workspaceRoot, 'outside-link.pdf') + await fs.writeFile(corruptPath, 'not a pdf') + await fs.copyFile(sourcePath, outsidePath) + await fs.symlink(outsidePath, symlinkPath) + const harness = createPdfToolHarness(runtime) + try { + const outOfRange = await harness.invoke('pdf_read_pages', { + sourcePath, + selection: { type: 'page_numbers', pageNumbers: [99] } + }) + const badRegex = await harness.invoke('pdf_search', { + sourcePath, + query: '[', + mode: 'regex' + }) + const oversizedRange = await harness.invoke('pdf_read_pages', { + sourcePath, + selection: { type: 'page_range', fromPage: 1, toPage: 1000000 } + }) + const corrupt = await harness.invoke('pdf_inspect', { + sourcePath: corruptPath + }) + const outside = await harness.invoke('pdf_inspect', { + sourcePath: symlinkPath + }) + return { outOfRange, badRegex, oversizedRange, corrupt, outside } + } finally { + await harness.release() + await fs.rm(outsidePath, { force: true }) + } + }, + assert: async (result, runtime) => { + runtime.assertThat( + [ + result.outOfRange, + result.badRegex, + result.oversizedRange, + result.corrupt, + result.outside + ].every((failure) => failure.ok === false && failure.error.length > 0), + { + name: 'pdf-invalid-inputs-fail-explicitly', + message: '所有 PDF 边界错误都应返回明确失败信息。', + data: result + } + ) + } + }) + }) + + it('pdf-result-size-contract', async () => { + await runBackendCase({ + caseId: 'pdf-result-size-contract', + harnessMode: 'tool-real', + domain: 'ai', + suite: 'tool', + groups: ['pdf'], + description: '验证 PDF 正文过大时返回统一 hand_tool_result_too_large 提示。', + fixtures: [PDF_FIXTURE], + runMode: 'single', + execute: async (runtime) => { + const sourcePath = runtime.getFixtureWorkspacePath(PDF_FIXTURE) + const harness = createPdfToolHarness(runtime, { + humanRuntimeOverrides: { modelContextWindowTokens: 100 } + }) + try { + return await harness.invoke< + PdfFailure & { decisionHint?: { code?: string; detail?: unknown } } + >('pdf_read_pages', { + sourcePath, + selection: { type: 'page_numbers', pageNumbers: [1] } + }) + } finally { + await harness.release() + } + }, + assert: async (result, runtime) => { + runtime.assertThat( + result.ok === false && result.decisionHint?.code === 'hand_tool_result_too_large', + { + name: 'pdf-large-result-returns-shared-decision-hint', + message: 'PDF 工具结果过大时应复用 hand_tool_result_too_large。', + data: result + } + ) + } + }) + }) +}) diff --git a/tests/case/unit/ai/human-memory-renderers.test.ts b/tests/case/unit/ai/human-memory-renderers.test.ts index 2964721f..703a18ef 100644 --- a/tests/case/unit/ai/human-memory-renderers.test.ts +++ b/tests/case/unit/ai/human-memory-renderers.test.ts @@ -284,6 +284,79 @@ describe('ai unit human memory renderers', () => { expect(rendered).toContain('"totalLines":1') }) + it('keeps PDF page text and rendered image paths in brain history', () => { + const initialState = createInitialHumanState({ + taskId: 'task-history-pdf', + userInput: '阅读 PDF 图文' + }) + const imagePath = '/workspace/.ncurator/runtime/assets/uuid/page-9.png' + const calls = [ + { + toolName: 'pdf_read_pages', + args: { + sourcePath: '/workspace/深度学习.pdf', + selection: { type: 'page_numbers', pageNumbers: [9] } + } + }, + { + toolName: 'pdf_render_pages', + args: { sourcePath: '/workspace/深度学习.pdf', pageNumbers: [9] } + } + ] + const taskRun = appendRoundRecord({ + state: initialState, + thought: { + thought: { next: '读取 PDF 页面并渲染图片' }, + nextAction: { action: 'hand', calls } + }, + execution: buildHandExecution({ + calls, + toolResults: [ + { + toolName: 'pdf_read_pages', + ok: true, + data: { + sourcePath: '/workspace/深度学习.pdf', + pageCount: 11, + pages: [ + { + pageNumber: 9, + text: '过度拟合会让训练与验证表现差距变大', + charCount: 19, + textStatus: 'available', + imageCount: 1 + } + ] + } + }, + { + toolName: 'pdf_render_pages', + ok: true, + data: { + sourcePath: '/workspace/深度学习.pdf', + pageCount: 11, + assetBucketId: 'uuid', + images: [ + { + pageNumber: 9, + path: imagePath, + mimeType: 'image/png', + width: 1224, + height: 1584 + } + ] + } + } + ] + }) + }) + + const rendered = renderBrainRoundSection(taskRun.rounds) + expect(rendered).toContain('过度拟合会让训练与验证表现差距变大') + expect(rendered).toContain(imagePath) + expect(rendered).toContain('"pageNumber":9') + }) + it('renders kb_list_documents result ids in round history', () => { const initialState = createInitialHumanState({ taskId: 'task-history-kb-list-documents', diff --git a/tests/case/unit/ai/human-tool-result-renderers.test.ts b/tests/case/unit/ai/human-tool-result-renderers.test.ts index c5faa6de..4a2decf2 100644 --- a/tests/case/unit/ai/human-tool-result-renderers.test.ts +++ b/tests/case/unit/ai/human-tool-result-renderers.test.ts @@ -129,4 +129,51 @@ describe('ai unit human tool result renderers', () => { expect(rendered).not.toContain('currentRevisionRef') expect(rendered).not.toContain('createdBlocks') }) + + it('renders PDF page text, search snippets and rendered image paths', () => { + const readRendered = renderToolResultDataText({ + toolName: 'pdf_read_pages', + data: { + sourcePath: '/workspace/深度学习.pdf', + pageCount: 11, + pages: [ + { + pageNumber: 1, + textStatus: 'available', + charCount: 42, + imageCount: 1, + text: 'inputs parameters architecture predictions labels loss update' + } + ] + } + }) + const searchRendered = renderToolResultDataText({ + toolName: 'pdf_search', + data: { + sourcePath: '/workspace/深度学习.pdf', + pageCount: 11, + query: '过度拟合', + mode: 'literal', + totalMatchCount: 1, + returnedMatchCount: 1, + matches: [{ pageNumber: 9, matchedText: '过度拟合', snippet: '右图属于过度拟合' }] + } + }) + const imagePath = '/workspace/.ncurator/runtime/assets/uuid/page-9.png' + const renderRendered = renderToolResultDataText({ + toolName: 'pdf_render_pages', + data: { + sourcePath: '/workspace/深度学习.pdf', + pageCount: 11, + assetBucketId: 'uuid', + images: [{ pageNumber: 9, path: imagePath, mimeType: 'image/png', width: 100, height: 200 }] + } + }) + + expect(readRendered).toContain('page 1') + expect(readRendered).toContain('architecture') + expect(searchRendered).toContain('pageNumber=9') + expect(searchRendered).toContain('右图属于过度拟合') + expect(renderRendered).toContain(imagePath) + }) }) diff --git a/tests/case/unit/ai/pdf-tool-contract.test.ts b/tests/case/unit/ai/pdf-tool-contract.test.ts new file mode 100644 index 00000000..2fb551ce --- /dev/null +++ b/tests/case/unit/ai/pdf-tool-contract.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' +import { SKILL_DEFINITIONS } from '../../../../src/main/services/ai/human/skills' +import { PDF_TOOL_SPECS } from '../../../../src/main/services/ai/human/tools/pdf/toolSpecs' +import { + pdfReadPagesSchema, + pdfRenderPagesSchema, + pdfSearchSchema +} from '../../../../src/main/services/ai/human/tools/pdf/schemas' + +describe('ai unit PDF tool contract', () => { + it('exposes only inspect, search, page reading and page rendering', () => { + expect(PDF_TOOL_SPECS.map((spec) => spec.name)).toEqual([ + 'pdf_inspect', + 'pdf_search', + 'pdf_read_pages', + 'pdf_render_pages' + ]) + expect(SKILL_DEFINITIONS['pdf-read'].toolNames).toEqual(PDF_TOOL_SPECS.map((spec) => spec.name)) + expect(PDF_TOOL_SPECS.map((spec) => spec.name)).not.toContain('pdf_recognize_pages') + }) + + it('keeps selection, search and render limits in schemas', () => { + expect( + pdfReadPagesSchema.safeParse({ + sourcePath: '学习笔记/深度学习.pdf', + selection: { type: 'page_range', fromPage: 1, toPage: 9 } + }).success + ).toBe(true) + expect( + pdfSearchSchema.safeParse({ + sourcePath: '学习笔记/深度学习.pdf', + query: '过度拟合', + maxMatches: 51 + }).success + ).toBe(false) + expect( + pdfRenderPagesSchema.safeParse({ + sourcePath: '学习笔记/深度学习.pdf', + pageNumbers: Array.from({ length: 21 }, (_, index) => index + 1) + }).success + ).toBe(false) + }) +}) diff --git a/tests/case/unit/ai/runtime-skill-contract.test.ts b/tests/case/unit/ai/runtime-skill-contract.test.ts index 34702b1d..03087d1a 100644 --- a/tests/case/unit/ai/runtime-skill-contract.test.ts +++ b/tests/case/unit/ai/runtime-skill-contract.test.ts @@ -17,6 +17,7 @@ describe('ai unit runtime skill contract', () => { expect(listExternalSkillDefinitions().map((definition) => definition.skillId)).toEqual([ 'docx-read', 'docx-write', + 'pdf-read', 'image-recognition' ]) }) diff --git a/tests/case/unit/workbench/attachment-service.test.ts b/tests/case/unit/workbench/attachment-service.test.ts index 5fceea44..28081dd8 100644 --- a/tests/case/unit/workbench/attachment-service.test.ts +++ b/tests/case/unit/workbench/attachment-service.test.ts @@ -68,4 +68,19 @@ describe('attachment service runtime directories', () => { expect(second.path).toBe(path.join(bucketPath, 'reference-1.json')) await expect(fs.readFile(second.path, 'utf8')).resolves.toBe('{"ready":true}') }) + + it('accepts PDF attachments and assigns application/pdf', async () => { + const attachment = await service.saveAttachment({ + workspaceRootPath, + assetBucketId: 'bucket-pdf', + name: 'lecture.pdf', + bytes: Buffer.from('%PDF-1.7\n') + }) + + expect(attachment.kind).toBe('file') + expect(attachment.mimeType).toBe('application/pdf') + expect(attachment.path).toBe( + path.join(workspaceRootPath, '.ncurator', 'runtime', 'assets', 'bucket-pdf', 'lecture.pdf') + ) + }) }) diff --git a/tests/harness/backend/aiLiveTestData.ts b/tests/harness/backend/aiLiveTestData.ts index 75280ad5..6bf456ac 100644 --- a/tests/harness/backend/aiLiveTestData.ts +++ b/tests/harness/backend/aiLiveTestData.ts @@ -15,6 +15,11 @@ export const DOCX_POSITIONED_FIXTURE_PATHS = [ `${WORKSPACE_LEARNING_FIXTURE_DIR}/图形学.docx` ] as const +export const PDF_READING_FIXTURE_PATH = `${WORKSPACE_LEARNING_FIXTURE_DIR}/深度学习.pdf` as const + +export const HUMAN_PDF_VISUAL_READING_USER_INPUT = + '请只根据我选中的《深度学习.pdf》回答两个问题：1. 用通俗的话说明第1页流程图里 inputs、parameters、architecture、predictions、labels、loss 和 update 是怎么形成训练闭环的；2. 结合第9页 Proper fit 和 Overfit 对比图，说明两条曲线有什么直观区别，以及为什么右边属于过度拟合。' + export const DOCX_MULTI_EDIT_FIXTURE_PATHS = [ `${SHARED_WORKSPACE_FIXTURE_PATH}/人工智能探究课堂PRD.docx`, `${SHARED_WORKSPACE_FIXTURE_PATH}/交互式探究学习案例：产业转移.docx` @@ -102,6 +107,25 @@ export const buildSelectedDocxFixtureReferences = ( ): SelectedReference[] => buildSelectedDocxReferences(resolveSelectedDocxFixtures(runtime, fixturePaths), indexedFiles) +export const buildSelectedPdfFixtureReference = ( + runtime: BackendTestRuntime, + fixturePath: string = PDF_READING_FIXTURE_PATH +): SelectedReference => { + const workspacePath = runtime.getFixtureWorkspacePath(fixturePath) + const relativePath = path.relative(runtime.workspaceRoot, workspacePath).split(path.sep).join('/') + return { + kind: 'file', + workspaceId: 'test-workspace', + nodeId: 'fixture-pdf-1', + name: path.basename(workspacePath), + relativePath, + absolutePath: workspacePath, + fileExt: '.pdf', + kbDocumentIds: [], + kbAvailability: 'not_indexed' + } +} + export const captureDocxFixtureBytes = async ( runtime: BackendTestRuntime, fixturePaths: readonly string[] diff --git a/tests/harness/backend/aiToolHarness.ts b/tests/harness/backend/aiToolHarness.ts index c98ef7f9..60fb94f8 100644 --- a/tests/harness/backend/aiToolHarness.ts +++ b/tests/harness/backend/aiToolHarness.ts @@ -14,12 +14,15 @@ import { } from 'docx' import { buildDocxCapabilityTools } from '../../../src/main/services/ai/human/tools/docx' import { releaseDocxRunSession } from '../../../src/main/services/ai/human/tools/docx/worker/docxWorkerClient' +import { AiRunResourceManager } from '../../../src/main/services/ai/run/AiRunResourceManager' import { DEFAULT_MODEL_CONTEXT_WINDOW_TOKENS } from '../../../src/main/services/ai/human/constants' import { buildKbCapabilityTools } from '../../../src/main/services/ai/human/tools/kb/retrievalTools' import { buildTerminalCapabilityTools } from '../../../src/main/services/ai/human/tools/terminal/terminalTools' import { TerminalSessionManager } from '../../../src/main/services/ai/human/tools/terminal/TerminalSessionManager' import { buildTextFileCapabilityTools } from '../../../src/main/services/ai/human/tools/textFile' import { imageRecognitionToolFamily } from '../../../src/main/services/ai/human/tools/imageRecognition' +import { buildPdfCapabilityTools } from '../../../src/main/services/ai/human/tools/pdf' +import { releasePdfRunSession } from '../../../src/main/services/pdf/worker/pdfWorkerClient' import { utilityToolFamily } from '../../../src/main/services/ai/human/tools/utility' import { webToolFamily } from '../../../src/main/services/ai/human/tools/web' import { createHumanRuntimeCache } from '../../../src/main/services/ai/human/node/utils/runtimeCache' @@ -79,6 +82,9 @@ export const createDocxToolHarness = ( const ctx = runtime.createAiRunContext(contextInput) const cache = createHumanRuntimeCache() const activities: CapturedToolActivity[] = [] + const resourceManager = new AiRunResourceManager([ + { id: 'docx', releaseRun: releaseDocxRunSession } + ]) const built = buildDocxCapabilityTools({ ctx, cache, @@ -99,7 +105,7 @@ export const createDocxToolHarness = ( state: built.state, activities, async release(): Promise { - await releaseDocxRunSession(runtime.runId) + await resourceManager.releaseRun(runtime.runId) } } } @@ -169,6 +175,39 @@ export const createTextFileToolHarness = ( } } +export const createPdfToolHarness = ( + runtime: BackendTestRuntime, + contextInput: CreateAiRunContextInput = {} +) => { + const ctx = runtime.createAiRunContext(contextInput) + const cache = createHumanRuntimeCache() + const activities: CapturedToolActivity[] = [] + const resourceManager = new AiRunResourceManager([ + { id: 'pdf', releaseRun: releasePdfRunSession } + ]) + const built = buildPdfCapabilityTools({ + ctx, + cache, + runId: runtime.runId, + sessionId: contextInput.sessionId || 'test-session', + taskId: runtime.taskId, + workspaceRootPath: runtime.workspaceRoot, + safePreviewChars: 1400, + modelContextWindowTokens: + ctx.humanRuntimeOverrides?.modelContextWindowTokens || DEFAULT_MODEL_CONTEXT_WINDOW_TOKENS, + emitActivity: (activity) => activities.push(activity) + }) + + return { + ...createToolInvoker(built.tools as readonly ToolLike[]), + ctx, + activities, + async release(): Promise { + await resourceManager.releaseRun(runtime.runId) + } + } +} + export const createRetrievalToolHarness = ( runtime: BackendTestRuntime, contextInput: CreateAiRunContextInput = {} diff --git a/tests/harness/backend/humanHarness.ts b/tests/harness/backend/humanHarness.ts index bc32fc8e..db5bf7f8 100644 --- a/tests/harness/backend/humanHarness.ts +++ b/tests/harness/backend/humanHarness.ts @@ -1,5 +1,8 @@ import { createHumanGraphRunner } from '../../../src/main/services/ai/human/graph/humanGraph' import { runAiRuntimePreflight } from '../../../src/main/services/ai/runtimePreflight' +import { AiRunResourceManager } from '../../../src/main/services/ai/run/AiRunResourceManager' +import { releaseDocxRunSession } from '../../../src/main/services/ai/human/tools/docx/worker/docxWorkerClient' +import { releasePdfRunSession } from '../../../src/main/services/pdf/worker/pdfWorkerClient' import type { HumanRunDispatchOptions, HumanTaskInput, @@ -15,6 +18,10 @@ export const createHumanTestRunner = ( ...contextInput }) const run = createHumanGraphRunner({ ctx }) + const resourceManager = new AiRunResourceManager([ + { id: 'docx', releaseRun: releaseDocxRunSession }, + { id: 'pdf', releaseRun: releasePdfRunSession } + ]) return { ctx, @@ -23,7 +30,11 @@ export const createHumanTestRunner = ( await runAiRuntimePreflight({ ensureEmbeddingReady: runtime.ensureEmbeddingReady }) - return await run(taskInput, dispatchOptions) as HumanTaskOutput + try { + return (await run(taskInput, dispatchOptions)) as HumanTaskOutput + } finally { + await resourceManager.releaseRun(runtime.runId) + } } } } diff --git a/tests/harness/backend/llmConfig.ts b/tests/harness/backend/llmConfig.ts index faf866b9..6e3bdf04 100644 --- a/tests/harness/backend/llmConfig.ts +++ b/tests/harness/backend/llmConfig.ts @@ -31,6 +31,19 @@ const resolveLiveBaseUrl = (modelName: string): string => { return normalizeOpenAIBaseUrl(rawBaseUrl) } +const resolveLiveVisionModelName = (primaryConfig: LiveLlmConfig): string => { + const explicit = readEnv('AI_LIVE_VISION_MODEL_NAME') + if (explicit) return explicit + try { + if (new URL(primaryConfig.base_url).hostname.endsWith('dashscope.aliyuncs.com')) { + return 'qwen3-vl-flash' + } + } catch { + // 交给主配置校验报告无效 URL。 + } + return primaryConfig.model_name +} + export const hasLiveLlmConfigInEnv = (): boolean => { const modelName = resolveLiveModelName() const baseUrl = resolveLiveBaseUrl(modelName) @@ -67,3 +80,23 @@ export const loadLiveLlmConfigFromEnv = (): LiveLlmConfig => { api_key: apiKey } } + +export const loadLiveVisionLlmConfigFromEnv = (): LiveLlmConfig => { + const primaryConfig = loadLiveLlmConfigFromEnv() + const baseUrl = normalizeOpenAIBaseUrl( + readEnv('AI_LIVE_VISION_BASE_URL') || primaryConfig.base_url + ) + const apiKey = readEnv('AI_LIVE_VISION_API_KEY') || primaryConfig.api_key + const modelName = resolveLiveVisionModelName(primaryConfig) + return { + id: 'ai-live-vision-config', + name: 'AI Live Vision', + providerKind: resolveLlmProviderKind({ + providerKind: readEnv('AI_LIVE_VISION_PROVIDER_KIND') || inferLlmProviderKind(baseUrl), + baseUrl + }), + base_url: baseUrl, + model_name: modelName, + api_key: apiKey + } +} diff --git a/tests/harness/backend/realAiDepsFactory.ts b/tests/harness/backend/realAiDepsFactory.ts index 1cf7729e..9a59bcaf 100644 --- a/tests/harness/backend/realAiDepsFactory.ts +++ b/tests/harness/backend/realAiDepsFactory.ts @@ -173,6 +173,7 @@ export const createRealAiDepsFactory = async (input: { defaultModelName: input.runtimeModelName }) || '' }, + visionModel: contextInput.visionModel, workspace: bindWorkspace ? { workspaceId: 'test-workspace', diff --git a/tests/harness/backend/test-types.ts b/tests/harness/backend/test-types.ts index f15de6bd..19dfe617 100644 --- a/tests/harness/backend/test-types.ts +++ b/tests/harness/backend/test-types.ts @@ -166,6 +166,10 @@ export type CreateAiRunContextInput = { allowedToolNames?: string[] runtimeSettingsOverride?: Partial modelName?: string + visionModel?: { + configId: string + modelName: string + } approvalPolicy?: ApprovalPolicy | ApprovalResolver checkCancelledOverride?: () => void humanRuntimeOverrides?: {