feat(ai): add PDF reading skill
This commit is contained in:
1 parent
5e0ab0fa36
commit
4cd96edddf
47 files changed
+2969
-74
No files matched your search
+63
-17
@@ -12,47 +12,52 @@
|
||||
|
||||
### 2.1 按 Suite / 分类汇总
|
||||
|
||||
- `tool` 完整回归:`67 / 67`
|
||||
- 检测时间:`2026-07-14 13:59:28 +08:00`
|
||||
- `tool` 完整回归:`77 / 77`
|
||||
- 检测时间:`2026-07-16 19:07:56 +08:00`
|
||||
- 执行命令:`pnpm test:ai:tool`
|
||||
- `tool / docx`
|
||||
- 通过率:`40 / 40`
|
||||
- 检测时间:`2026-07-14 13:59:28 +08:00`
|
||||
- 检测时间:`2026-07-16 19:07:56 +08:00`
|
||||
- 工具面只保留 `docx_outline / docx_search / docx_read_blocks / docx_edit`;模型侧任务内 history/rollback 已移除,内部 replay/revision 与用户文件级回退继续保留。
|
||||
- `tool / text-file`
|
||||
- 通过率:`2 / 2`
|
||||
- 检测时间:`2026-07-15 +08:00`
|
||||
- 检测时间:`2026-07-16 19:07:56 +08:00`
|
||||
- 覆盖补充:`file_read` 行范围与总行数;`file_write` 的 create、rewrite、唯一 edit、replaceAll、原子失败、BOM/CRLF 保持、结构化 diff 重放和修改记录。
|
||||
- `tool / pdf`
|
||||
- 通过率:`4 / 4`
|
||||
- 检测时间:`2026-07-16 19:07:56 +08:00`
|
||||
- 新增 `pdf_inspect / pdf_search / pdf_read_pages / pdf_render_pages`,只负责 PDF 信息、文本层和页面渲染。
|
||||
- 页面 PNG 写入 `.ncurator/runtime/assets/<uuid>/page-N.png`,不作为文件修改或普通 artifact;图片理解由 `llm_recognize` 独立完成。
|
||||
- `tool / kb`
|
||||
- 通过率:`2 / 2`
|
||||
- 检测时间:`2026-07-13 13:04:39 +08:00`
|
||||
- 检测时间:`2026-07-16 19:07:56 +08:00`
|
||||
- 覆盖补充:文件路径与网页来源 URL 返回契约已通过。
|
||||
- `tool / terminal`
|
||||
- 通过率:`6 / 6`
|
||||
- 检测时间:`2026-07-16 +08:00`
|
||||
- 检测时间:`2026-07-16 19:07:56 +08:00`
|
||||
- 覆盖补充:terminal 回归测试按“安全指令集”语义维护;持久终端覆盖 cwd 保持、多终端、长命令让出、增量读取和停止;terminal 不根据命令文本猜测文件产物,文件变化统一由顶层 Run 扫描捕获。
|
||||
- `tool / web-browsing`
|
||||
- 通过率:`6 / 6`
|
||||
- 检测时间:`2026-07-13 13:04:39 +08:00`
|
||||
- 通过率:`7 / 7`
|
||||
- 检测时间:`2026-07-16 19:07:56 +08:00`
|
||||
- 覆盖补充:`web-crawl-contract` 覆盖 `web_crawl` 的 `topic` 必填、`markdown` 字段来源于 `fit_markdown`、`requestedUrl/finalUrl`、分组 links 评分、HTTP 失败与结果过大;同文件覆盖 `web_search` 结构化候选网页结果和 `browser_use` 缺少内置浏览器服务时的真实 diagnostic;用例使用全局 15 分钟回归超时,不再单独写死 30 秒。
|
||||
- `tool / utility`
|
||||
- 通过率:`5 / 5`
|
||||
- 检测时间:`2026-07-13 13:04:39 +08:00`
|
||||
- 通过率:`7 / 7`
|
||||
- 检测时间:`2026-07-16 19:07:56 +08:00`
|
||||
- 覆盖补充:`wait_time` 等待时长与返回 `waitedMs`;`llm_recognize` 已覆盖普通文本透传、无法确认类文本透传与底层失败分支。
|
||||
- `tool / hand`
|
||||
- 通过率:`9 / 9`
|
||||
- 检测时间:`2026-07-13 13:04:39 +08:00`
|
||||
- 检测时间:`2026-07-16 19:07:56 +08:00`
|
||||
- `human / mock-llm`
|
||||
- 通过率:`22 / 22`
|
||||
- 检测时间:`2026-07-14 13:59:02 +08:00`
|
||||
- 通过率:`23 / 23`
|
||||
- 检测时间:`2026-07-16 19:08:37 +08:00`
|
||||
- `human / live`
|
||||
- 登记 case:`10`;当前有效结果 `10 / 10` 通过,其中 `human-terminal-preview-service-live` 已于 `2026-07-16` 使用真实模型定向通过。
|
||||
- 检测时间:`2026-07-16 10:04:01 +08:00`
|
||||
- 实际模型:`deepseek-v4-flash`。
|
||||
- 登记 case:`11`;当前有效结果 `11 / 11` 通过,其中 `human-terminal-preview-service-live` 已于 `2026-07-16` 使用真实模型定向通过。
|
||||
- 最近定向检测:`human-pdf-visual-reading-live` 于 `2026-07-16 19:11:47 +08:00` 通过;其他 live case 沿用各自最近有效结果。
|
||||
- 实际主模型:`deepseek-v4-flash`;PDF 图像识别模型:`qwen3-vl-flash`。
|
||||
- memory、compaction、resume 均保留并通过;compaction 使用 `16K` 上下文,自然触发 1 次压缩,不限制压缩次数,压缩后继续完成周报文件交付。
|
||||
- KB/docx case 使用 2 次 KB 检索定位源资料,并真实更新 2 份目标文档;不再依赖模型枚举目标文件。
|
||||
- positioned DOCX case 正确命中复杂排版保护,没有危险写入,并交付了可直接手动粘贴的完整内容。
|
||||
- unit 全量:`51` files、`203 / 203` tests 通过。
|
||||
- AI unit:`45` files、`197 / 197` tests 通过;检测时间:`2026-07-16 19:05:58 +08:00`。
|
||||
- 样例评分:
|
||||
- `human-conversation-memory-multi-run-live`:`10 / 10`;风险主线、待确认项、负责人和后续追问连续性完整。
|
||||
- `human-risk-report-draft-live`:`10 / 10`;新建周报、原文件安全、未闭环风险、负责人动作和谨慎结论均满足。
|
||||
@@ -64,6 +69,7 @@
|
||||
- `human-web-crawl-url-inference-reading-live`:`9.0 / 10`;自行推断 pandas 官网地址并完成两份 CSV 文档对比。
|
||||
- `human-text-file-read-write-live`:通过;真实调用 `replaceAll`、唯一 `edit` 和回读,最终文件与工具结果边界均满足。
|
||||
- `human-terminal-preview-service-live`:`10 / 10`;真实进入子目录启动预览服务,完成页面访问、增量日志读取、正常停止、终端关闭和端口释放。
|
||||
- `human-pdf-visual-reading-live`:`10 / 10`;PDF 文本读取和页面渲染成功,第 1、9 页均完成真实图像识别,识别失败为 0,未调用 terminal、KB 或 docx,未产生普通 artifact,源 PDF 保持不变。
|
||||
|
||||
## 3. Suite 设计
|
||||
|
||||
@@ -72,6 +78,7 @@
|
||||
- `tool` 只验证真实生产工具调用,不允许 fake。
|
||||
- `tool / docx` 包含 `docx-edit-serial-actions-stability`,用于守住同一次 `docx_edit(actions)` 内插入、替换、删除后的 block 定位稳定性。
|
||||
- `tool / text-file` 覆盖普通文本读取、四种写入 action、修改记录、结构化 diff 与回退前置数据。
|
||||
- `tool / pdf` 使用真实 `深度学习.pdf`,覆盖元数据、页级正文、literal/regex 搜索、页面 PNG、工作区边界、损坏文件和结果大小限制。
|
||||
- `tool / web-browsing` 包含 `web-crawl-contract`,用于守住 `web_crawl`、`web_search` 和 `browser_use` 工具契约。
|
||||
- 目标:验证 tool 与 `hand` 工具执行守卫在真实生产代码下的输入输出是否正常、合理。
|
||||
- 约束:必须真实调用生产工具、真实 store、真实 docx apply service、真实 artifact 流。
|
||||
@@ -97,6 +104,7 @@
|
||||
|
||||
- AI backend 回归 case 必须使用生产 `createAiRunContext` 作为唯一 runtime 入口。
|
||||
- 回归 harness 必须使用真实 `StorageService / DocumentsStore / ChatStore / DocxApplyStore / LlmConfigStore / DocxApplyService / EmbeddingService`。
|
||||
- 每个 AI run 的 DOCX、PDF 及未来其他文档 worker 资源统一通过 `AiRunResourceManager` 在 run 结束时释放;资源释放失败不能覆盖任务结果。
|
||||
- `mock-llm` 统一通过 scripted LLM driver 固定 LLM 输出;除 LLM 输出外,其余全部真实。
|
||||
- `live` 统一通过真实 `LlmConfigStore` 注入 live 配置,并走生产统一 LLM executor。
|
||||
- 凡是会进入 `human` 的回归 run,必须在进入 `human` 前完成 embedding 基础服务 preflight;不允许把 embedding 初始化推迟到某个工具调用时才暴露。
|
||||
@@ -558,6 +566,35 @@ AI unit / 辅助测试目录:
|
||||
- Oracle:失败时 bytes 不变;成功替换次数准确;新内容中的 oldText 不会被再次递归替换;BOM 与 CRLF 保持。
|
||||
- Not:不支持正则 replaceAll。
|
||||
|
||||
#### pdf 类别
|
||||
|
||||
##### `pdf-inspect-read-search-contract`
|
||||
|
||||
- 测试类型:`backend`
|
||||
- What:验证真实 PDF 的元数据、页级文本和搜索结果。
|
||||
- Why:守住 PDF 文本阅读的基础契约。
|
||||
- How:对 `深度学习.pdf` 执行 `pdf_inspect -> pdf_read_pages -> pdf_search`。
|
||||
- Oracle:页数为 11,标题为“深度学习”,正文没有 NUL,且“过度拟合”能命中第 8 或第 9 页。
|
||||
- Not:不识别页面图片。
|
||||
|
||||
##### `pdf-render-pages-runtime-assets-contract`
|
||||
|
||||
- 测试类型:`backend`
|
||||
- What:验证指定页面会被渲染为可供其他工具使用的 PNG。
|
||||
- Why:PDF 与图像识别需要通过普通图片路径解耦协作。
|
||||
- How:渲染第 1、9 页并检查文件路径、PNG 文件头、源文件哈希和事件。
|
||||
- Oracle:路径为 `.ncurator/runtime/assets/<uuid>/page-N.png`;图片真实存在;源 PDF 不变;不产生普通 artifact。
|
||||
- Not:不在 PDF 工具内部做图像识别。
|
||||
|
||||
##### `pdf-failure-boundary-contract`
|
||||
|
||||
- 测试类型:`backend`
|
||||
- What:验证 PDF 输入边界会明确失败。
|
||||
- Why:防止路径逃逸、伪 PDF、非法 regex 和越界页码进入不确定状态。
|
||||
- How:分别传入越界页码、非法 regex、错误文件头以及指向工作区外的符号链接。
|
||||
- Oracle:每种输入都返回明确失败,不读取工作区外文件。
|
||||
- Not:不做密码破解或损坏文件修复。
|
||||
|
||||
#### kb 类别
|
||||
|
||||
##### `kb-list-documents-contract`
|
||||
@@ -1051,6 +1088,15 @@ AI unit / 辅助测试目录:
|
||||
- Oracle:真实 `kb_search` 返回有效命中;两份目标文档分别发生成功写入并产生两组携带 `applyId` 的 file artifact;两份来源课件保持不变。按 10 分制评估 KB 证据、两份目标文件、grounded 内容、来源安全和最终交付,严格超过 8 分通过;只完成一个目标文件最多 8 分,不能通过。
|
||||
- Not:不强制固定执行路径,也不要求 KB 缩圈后必须再打开所有候选资料。
|
||||
|
||||
##### `human-pdf-visual-reading-live`
|
||||
|
||||
- 测试类型:`backend / live`
|
||||
- What:验证真实模型会自主组合 PDF 文本读取、页面渲染和通用图像识别。
|
||||
- Why:PDF 工具不内置图片理解,必须验证工具解耦后 AI 仍能完成图文结合阅读。
|
||||
- How:只选择 `深度学习.pdf`,询问第 1 页训练闭环和第 9 页 Proper fit / Overfit 图形差异,仅开放 PDF 四个工具与 `llm_recognize`。
|
||||
- Oracle:任务完成;调用 `pdf_read_pages`、包含第 9 页的 `pdf_render_pages` 和 `llm_recognize`;第 9 页图像识别必须成功且全部图像识别失败数为 0;识别路径来自 render 输出并位于 `.ncurator/runtime/assets/<uuid>/page-9.png`;不调用 terminal、KB、docx;最终答复同时说明训练闭环和两组曲线差异,不能用“无法识别图片”等内容绕过;源 PDF 不变。
|
||||
- Not:不固定工具调用顺序和最终措辞,不要求把 PDF 页面图片登记为 artifact。
|
||||
|
||||
##### `human-web-crawl-multi-hop-cross-domain-reading-live`
|
||||
|
||||
- 测试类型:`backend`
|
||||
|
||||
@@ -24,6 +24,8 @@ extraResources:
|
||||
asarUnpack:
|
||||
- '**/node_modules/onnxruntime-node/bin/napi-v3/**/*'
|
||||
- '**/node_modules/node-pty/build/Release/**/*'
|
||||
- '**/node_modules/@napi-rs/canvas/**/*'
|
||||
- '**/node_modules/@napi-rs/canvas-*/**/*'
|
||||
win:
|
||||
icon: build/icon.ico
|
||||
executableName: Ncurator
|
||||
|
||||
@@ -14,6 +14,7 @@ export default defineConfig({
|
||||
workspacePrepareWorker: resolve(
|
||||
'src/main/services/workspace/worker/workspacePrepareWorker.ts'
|
||||
),
|
||||
pdfWorker: resolve('src/main/services/pdf/worker/pdfWorker.ts'),
|
||||
embeddingRuntimeProcess: resolve(
|
||||
'src/main/services/vector/runtime/embeddingRuntimeProcess.ts'
|
||||
)
|
||||
|
||||
@@ -76,6 +76,7 @@
|
||||
"@langchain/langgraph": "^1.1.5",
|
||||
"@langchain/openai": "^1.4.5",
|
||||
"@langchain/textsplitters": "^1.0.1",
|
||||
"@napi-rs/canvas": "0.1.80",
|
||||
"@node-rs/jieba": "^2.0.1",
|
||||
"@react-pdf/renderer": "^4.1.0",
|
||||
"@xmldom/xmldom": "^0.8.11",
|
||||
|
||||
Generated
+3
-1
@@ -47,6 +47,9 @@ importers:
|
||||
'@langchain/textsplitters':
|
||||
specifier: ^1.0.1
|
||||
version: 1.0.1(@langchain/core@1.1.44(@opentelemetry/api@1.9.0)(openai@6.36.0(ws@8.21.0(bufferutil@4.1.0))(zod@4.3.6)))
|
||||
'@napi-rs/canvas':
|
||||
specifier: 0.1.80
|
||||
version: 0.1.80
|
||||
'@node-rs/jieba':
|
||||
specifier: ^2.0.1
|
||||
version: 2.0.1
|
||||
@@ -8240,7 +8243,6 @@ snapshots:
|
||||
'@napi-rs/canvas-linux-x64-gnu': 0.1.80
|
||||
'@napi-rs/canvas-linux-x64-musl': 0.1.80
|
||||
'@napi-rs/canvas-win32-x64-msvc': 0.1.80
|
||||
optional: true
|
||||
|
||||
'@napi-rs/wasm-runtime@0.2.12':
|
||||
dependencies:
|
||||
|
||||
@@ -15,6 +15,7 @@ const assertNoImports = (label, graph, forbidden) => {
|
||||
|
||||
const mainGraph = readBundleGraph(path.join(mainOutDir, 'index.js'))
|
||||
const prepareWorkerGraph = readBundleGraph(path.join(mainOutDir, 'workspacePrepareWorker.js'))
|
||||
const pdfWorkerGraph = readBundleGraph(path.join(mainOutDir, 'pdfWorker.js'))
|
||||
const embeddingRuntimeGraph = readBundleGraph(path.join(mainOutDir, 'embeddingRuntimeProcess.js'))
|
||||
|
||||
assertNoImports('Electron main process', mainGraph, [
|
||||
@@ -26,6 +27,11 @@ assertNoImports('Workspace prepare worker', prepareWorkerGraph, [
|
||||
'@huggingface/transformers',
|
||||
'onnxruntime-node'
|
||||
])
|
||||
assertNoImports('PDF worker', pdfWorkerGraph, [
|
||||
'electron',
|
||||
'@huggingface/transformers',
|
||||
'onnxruntime-node'
|
||||
])
|
||||
assertNoImports('Embedding utility process', embeddingRuntimeGraph, ['electron'])
|
||||
|
||||
if (!embeddingRuntimeGraph.externalImports.has('@huggingface/transformers')) {
|
||||
|
||||
@@ -16,6 +16,7 @@ import { AiFileChangeService } from '../services/ai/fileChanges/AiFileChangeServ
|
||||
import { getAiContextUsage } from '../services/ai/contextUsage'
|
||||
import { DocxApplyService } from '../services/ai/human/tools/docx/review/DocxApplyService'
|
||||
import { releaseDocxRunSession } from '../services/ai/human/tools/docx/worker/docxWorkerClient'
|
||||
import { releasePdfRunSession } from '../services/pdf/worker/pdfWorkerClient'
|
||||
import type { AiLlmIoRecord } from '../../shared/types'
|
||||
import {
|
||||
buildWebReferenceItem,
|
||||
@@ -167,7 +168,8 @@ export function registerHandlers(services: {
|
||||
const docxApplyService = new DocxApplyService(storageService.docxApply, aiFileChangeService)
|
||||
const attachmentService = new AttachmentService()
|
||||
const runResourceManager = new AiRunResourceManager([
|
||||
{ id: 'docx', releaseRun: releaseDocxRunSession }
|
||||
{ id: 'docx', releaseRun: releaseDocxRunSession },
|
||||
{ id: 'pdf', releaseRun: releasePdfRunSession }
|
||||
])
|
||||
const chatRunService = new ChatRunService({
|
||||
chatStore,
|
||||
@@ -1537,7 +1539,8 @@ export function registerHandlers(services: {
|
||||
'tiff',
|
||||
'md',
|
||||
'json',
|
||||
'docx'
|
||||
'docx',
|
||||
'pdf'
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1553,7 +1556,8 @@ export function registerHandlers(services: {
|
||||
'tiff',
|
||||
'md',
|
||||
'json',
|
||||
'docx'
|
||||
'docx',
|
||||
'pdf'
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
@@ -3,6 +3,7 @@ export const AI_TOOL_FAMILY_IDS = [
|
||||
'text-file',
|
||||
'kb',
|
||||
'docx',
|
||||
'pdf',
|
||||
'web',
|
||||
'utility',
|
||||
'image-recognition'
|
||||
@@ -16,7 +17,12 @@ export const BUILTIN_RUNTIME_SKILL_IDS = [
|
||||
'utility'
|
||||
] as const
|
||||
|
||||
export const EXTERNAL_RUNTIME_SKILL_IDS = ['docx-read', 'docx-write', 'image-recognition'] as const
|
||||
export const EXTERNAL_RUNTIME_SKILL_IDS = [
|
||||
'docx-read',
|
||||
'docx-write',
|
||||
'pdf-read',
|
||||
'image-recognition'
|
||||
] as const
|
||||
|
||||
export const LEARNABLE_RUNTIME_SKILL_IDS = EXTERNAL_RUNTIME_SKILL_IDS
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ const TOOL_FAMILY_BUILDERS: Record<
|
||||
'text-file': TOOL_FAMILY_DEFINITIONS['text-file'].buildTools,
|
||||
kb: TOOL_FAMILY_DEFINITIONS.kb.buildTools,
|
||||
docx: TOOL_FAMILY_DEFINITIONS.docx.buildTools,
|
||||
pdf: TOOL_FAMILY_DEFINITIONS.pdf.buildTools,
|
||||
web: TOOL_FAMILY_DEFINITIONS.web.buildTools,
|
||||
utility: TOOL_FAMILY_DEFINITIONS.utility.buildTools,
|
||||
'image-recognition': TOOL_FAMILY_DEFINITIONS['image-recognition'].buildTools
|
||||
|
||||
@@ -135,6 +135,87 @@ const TOOL_HISTORY_WHITELIST_BY_NAME: Record<string, NestedWhitelistConfig> = {
|
||||
blocks: DOCX_BLOCK_WHITELIST
|
||||
},
|
||||
docx_edit: DOCX_WRITE_RESULT_WHITELIST,
|
||||
pdf_inspect: {
|
||||
sourcePath: true,
|
||||
pageCount: true,
|
||||
metadata: {
|
||||
title: true,
|
||||
author: true,
|
||||
subject: true,
|
||||
keywords: true,
|
||||
creator: true,
|
||||
producer: true,
|
||||
creationDate: true,
|
||||
modificationDate: true
|
||||
},
|
||||
outline: {
|
||||
title: true,
|
||||
pageNumber: true,
|
||||
depth: true
|
||||
},
|
||||
encrypted: true,
|
||||
warnings: true
|
||||
},
|
||||
pdf_search: {
|
||||
sourcePath: true,
|
||||
pageCount: true,
|
||||
query: true,
|
||||
mode: true,
|
||||
pageRange: {
|
||||
fromPage: true,
|
||||
toPage: true
|
||||
},
|
||||
totalMatchCount: true,
|
||||
returnedMatchCount: true,
|
||||
matches: {
|
||||
pageNumber: true,
|
||||
matchedText: true,
|
||||
snippet: true,
|
||||
start: true,
|
||||
end: true
|
||||
},
|
||||
pagesWithNoTextCount: true,
|
||||
pagesWithSuspiciousTextCount: true,
|
||||
warnings: true,
|
||||
error: true,
|
||||
reason: true,
|
||||
decisionHint: true
|
||||
},
|
||||
pdf_read_pages: {
|
||||
sourcePath: true,
|
||||
pageCount: true,
|
||||
pages: {
|
||||
pageNumber: true,
|
||||
text: true,
|
||||
charCount: true,
|
||||
textStatus: true,
|
||||
imageCount: true,
|
||||
width: true,
|
||||
height: true,
|
||||
rotation: true,
|
||||
warnings: true
|
||||
},
|
||||
warnings: true,
|
||||
error: true,
|
||||
reason: true,
|
||||
decisionHint: true
|
||||
},
|
||||
pdf_render_pages: {
|
||||
sourcePath: true,
|
||||
pageCount: true,
|
||||
assetBucketId: true,
|
||||
images: {
|
||||
pageNumber: true,
|
||||
path: true,
|
||||
mimeType: true,
|
||||
width: true,
|
||||
height: true
|
||||
},
|
||||
warnings: true,
|
||||
error: true,
|
||||
reason: true,
|
||||
decisionHint: true
|
||||
},
|
||||
web_search: {
|
||||
query: true,
|
||||
results: {
|
||||
|
||||
@@ -248,6 +248,103 @@ const renderDocxOutlineResult = (obj: JsonObject): string =>
|
||||
.filter(Boolean)
|
||||
.join('\n')
|
||||
|
||||
const renderPdfPageList = (value: JsonValue | undefined): string => {
|
||||
if (!Array.isArray(value) || value.length === 0) return 'none'
|
||||
return value
|
||||
.map((item, index) => {
|
||||
if (!isJsonObject(item)) return renderGenericArrayItem(item, index)
|
||||
return [
|
||||
`page ${readNumberTextField(item, 'pageNumber') || String(index + 1)}:${renderKeyValuePairs(
|
||||
[
|
||||
['textStatus', readStringField(item, 'textStatus')],
|
||||
['charCount', readNumberTextField(item, 'charCount')],
|
||||
['imageCount', readNumberTextField(item, 'imageCount')]
|
||||
]
|
||||
)}`,
|
||||
readStringField(item, 'text') || '(no text)'
|
||||
].join('\n')
|
||||
})
|
||||
.join('\n\n')
|
||||
}
|
||||
|
||||
const renderPdfSearchMatches = (value: JsonValue | undefined): string => {
|
||||
if (!Array.isArray(value) || value.length === 0) return 'none'
|
||||
return value
|
||||
.map((item, index) => {
|
||||
if (!isJsonObject(item)) return renderGenericArrayItem(item, index)
|
||||
return `${index + 1}. ${renderKeyValuePairs([
|
||||
['pageNumber', readNumberTextField(item, 'pageNumber')],
|
||||
['matchedText', readStringField(item, 'matchedText')],
|
||||
['snippet', readStringField(item, 'snippet')]
|
||||
])}`
|
||||
})
|
||||
.join('\n')
|
||||
}
|
||||
|
||||
const renderPdfImages = (value: JsonValue | undefined): string => {
|
||||
if (!Array.isArray(value) || value.length === 0) return 'none'
|
||||
return value
|
||||
.map((item, index) => {
|
||||
if (!isJsonObject(item)) return renderGenericArrayItem(item, index)
|
||||
return `${index + 1}. ${renderKeyValuePairs([
|
||||
['pageNumber', readNumberTextField(item, 'pageNumber')],
|
||||
['path', readStringField(item, 'path')],
|
||||
['mimeType', readStringField(item, 'mimeType')],
|
||||
['width', readNumberTextField(item, 'width')],
|
||||
['height', readNumberTextField(item, 'height')]
|
||||
])}`
|
||||
})
|
||||
.join('\n')
|
||||
}
|
||||
|
||||
const renderPdfInspectResult = (obj: JsonObject): string =>
|
||||
[
|
||||
renderKeyValuePairs([
|
||||
['sourcePath', readStringField(obj, 'sourcePath')],
|
||||
['pageCount', readNumberTextField(obj, 'pageCount')],
|
||||
['encrypted', renderPrimitive(obj.encrypted)]
|
||||
]),
|
||||
'metadata:',
|
||||
isJsonObject(obj.metadata) ? renderGenericObject(obj.metadata) : 'none',
|
||||
'outline:',
|
||||
Array.isArray(obj.outline) ? renderGenericArray(obj.outline) : 'none'
|
||||
].join('\n')
|
||||
|
||||
const renderPdfSearchResult = (obj: JsonObject): string =>
|
||||
[
|
||||
renderKeyValuePairs([
|
||||
['sourcePath', readStringField(obj, 'sourcePath')],
|
||||
['pageCount', readNumberTextField(obj, 'pageCount')],
|
||||
['query', readStringField(obj, 'query')],
|
||||
['mode', readStringField(obj, 'mode')],
|
||||
['returnedMatchCount', readNumberTextField(obj, 'returnedMatchCount')],
|
||||
['totalMatchCount', readNumberTextField(obj, 'totalMatchCount')]
|
||||
]),
|
||||
'matches:',
|
||||
renderPdfSearchMatches(obj.matches)
|
||||
].join('\n')
|
||||
|
||||
const renderPdfReadPagesResult = (obj: JsonObject): string =>
|
||||
[
|
||||
renderKeyValuePairs([
|
||||
['sourcePath', readStringField(obj, 'sourcePath')],
|
||||
['pageCount', readNumberTextField(obj, 'pageCount')]
|
||||
]),
|
||||
'pages:',
|
||||
renderPdfPageList(obj.pages)
|
||||
].join('\n')
|
||||
|
||||
const renderPdfRenderPagesResult = (obj: JsonObject): string =>
|
||||
[
|
||||
renderKeyValuePairs([
|
||||
['sourcePath', readStringField(obj, 'sourcePath')],
|
||||
['pageCount', readNumberTextField(obj, 'pageCount')],
|
||||
['assetBucketId', readStringField(obj, 'assetBucketId')]
|
||||
]),
|
||||
'images:',
|
||||
renderPdfImages(obj.images)
|
||||
].join('\n')
|
||||
|
||||
const renderWebCrawlLinkList = (value: JsonValue | undefined): string => {
|
||||
if (!Array.isArray(value) || value.length === 0) return 'none'
|
||||
|
||||
@@ -386,6 +483,14 @@ export const renderToolResultDataText = (
|
||||
return renderDocxSearchResult(value)
|
||||
case 'docx_outline':
|
||||
return renderDocxOutlineResult(value)
|
||||
case 'pdf_inspect':
|
||||
return renderPdfInspectResult(value)
|
||||
case 'pdf_search':
|
||||
return renderPdfSearchResult(value)
|
||||
case 'pdf_read_pages':
|
||||
return renderPdfReadPagesResult(value)
|
||||
case 'pdf_render_pages':
|
||||
return renderPdfRenderPagesResult(value)
|
||||
case 'web_search':
|
||||
return renderWebSearchResult(value)
|
||||
case 'web_crawl':
|
||||
|
||||
@@ -51,6 +51,17 @@ export const SKILL_DEFINITIONS: Record<RuntimeSkillId, SkillDefinition> = {
|
||||
whenToUse: '适用于用户要求生成、补充、改写、删除或格式化 docx 内容的任务。',
|
||||
toolNames: ['docx_edit']
|
||||
},
|
||||
'pdf-read': {
|
||||
skillId: 'pdf-read',
|
||||
familyId: 'pdf',
|
||||
source: 'external',
|
||||
countsTowardToolBudget: true,
|
||||
title: 'pdf-read',
|
||||
description:
|
||||
'能力:读取工作区内 PDF 的文档信息、页级文本、检索结果,并将指定页面渲染为图片。\n结果:返回 PDF 元数据、页码、正文、匹配位置或页面图片路径。\n边界:只读取 PDF,不修改源文件;文本结果取决于 PDF 文本层,页面图片不包含识别结果。',
|
||||
whenToUse: '适用于需要阅读 PDF 文本、搜索 PDF 内容或查看 PDF 页面图形的任务。',
|
||||
toolNames: ['pdf_inspect', 'pdf_search', 'pdf_read_pages', 'pdf_render_pages']
|
||||
},
|
||||
'image-recognition': {
|
||||
skillId: 'image-recognition',
|
||||
familyId: 'image-recognition',
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { AiToolFamilyId, ToolFamilyDefinition } from './types'
|
||||
import { docxToolFamily } from './docx'
|
||||
import { imageRecognitionToolFamily } from './imageRecognition'
|
||||
import { pdfToolFamily } from './pdf'
|
||||
import { buildKbCapabilityTools } from './kb/retrievalTools'
|
||||
import { terminalToolFamily } from './terminal'
|
||||
import { textFileToolFamily } from './textFile'
|
||||
@@ -17,6 +18,7 @@ export const TOOL_FAMILY_DEFINITIONS: Record<AiToolFamilyId, ToolFamilyDefinitio
|
||||
'text-file': textFileToolFamily,
|
||||
kb: kbToolFamily,
|
||||
docx: docxToolFamily,
|
||||
pdf: pdfToolFamily,
|
||||
web: webToolFamily,
|
||||
utility: utilityToolFamily,
|
||||
'image-recognition': imageRecognitionToolFamily
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import { DEFAULT_MODEL_CONTEXT_WINDOW_TOKENS } from '../../constants'
|
||||
import { createWorkspaceBoundToolFamily } from '../utils/workspaceGuards'
|
||||
import { createPdfToolRuntime } from './runtime'
|
||||
import { buildPdfReadTools } from './tools/readTools'
|
||||
|
||||
export const buildPdfCapabilityTools = (input: Parameters<typeof createPdfToolRuntime>[0]) => {
|
||||
const runtime = createPdfToolRuntime(input)
|
||||
input.cache.pdfRuntime = runtime
|
||||
return { tools: buildPdfReadTools(runtime), runtime }
|
||||
}
|
||||
|
||||
export const pdfToolFamily = createWorkspaceBoundToolFamily({
|
||||
familyId: 'pdf',
|
||||
descriptionForTool: (name) => `pdf tool: ${name}`,
|
||||
reason: 'pdf capability requires workspace binding before execution',
|
||||
error: 'workspace is required for pdf capability',
|
||||
buildWorkspaceTools: (input) => {
|
||||
return buildPdfCapabilityTools({
|
||||
...input,
|
||||
safePreviewChars: 1400,
|
||||
modelContextWindowTokens:
|
||||
input.ctx.humanRuntimeOverrides?.modelContextWindowTokens ||
|
||||
DEFAULT_MODEL_CONTEXT_WINDOW_TOKENS
|
||||
}).tools
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,86 @@
|
||||
import { z } from 'zod/v4'
|
||||
|
||||
const warningsSchema = z.array(z.string())
|
||||
|
||||
const pdfMetadataSchema = z.object({
|
||||
title: z.string().optional(),
|
||||
author: z.string().optional(),
|
||||
subject: z.string().optional(),
|
||||
keywords: z.string().optional(),
|
||||
creator: z.string().optional(),
|
||||
producer: z.string().optional(),
|
||||
creationDate: z.string().optional(),
|
||||
modificationDate: z.string().optional()
|
||||
})
|
||||
|
||||
const pdfOutlineItemSchema = z.object({
|
||||
title: z.string(),
|
||||
pageNumber: z.number().int().optional(),
|
||||
depth: z.number().int()
|
||||
})
|
||||
|
||||
const pdfPageSchema = z.object({
|
||||
pageNumber: z.number().int(),
|
||||
text: z.string(),
|
||||
charCount: z.number().int(),
|
||||
textStatus: z.enum(['available', 'empty', 'suspicious']),
|
||||
imageCount: z.number().int(),
|
||||
width: z.number(),
|
||||
height: z.number(),
|
||||
rotation: z.number(),
|
||||
warnings: warningsSchema
|
||||
})
|
||||
|
||||
export const pdfInspectOutputSchema = z.object({
|
||||
sourcePath: z.string(),
|
||||
pageCount: z.number().int(),
|
||||
metadata: pdfMetadataSchema,
|
||||
outline: z.array(pdfOutlineItemSchema),
|
||||
encrypted: z.boolean(),
|
||||
warnings: warningsSchema
|
||||
})
|
||||
|
||||
export const pdfSearchOutputSchema = z.object({
|
||||
sourcePath: z.string(),
|
||||
pageCount: z.number().int(),
|
||||
query: z.string(),
|
||||
mode: z.enum(['literal', 'regex']),
|
||||
pageRange: z.object({ fromPage: z.number().int(), toPage: z.number().int() }),
|
||||
totalMatchCount: z.number().int(),
|
||||
returnedMatchCount: z.number().int(),
|
||||
matches: z.array(
|
||||
z.object({
|
||||
pageNumber: z.number().int(),
|
||||
matchedText: z.string(),
|
||||
snippet: z.string(),
|
||||
start: z.number().int(),
|
||||
end: z.number().int()
|
||||
})
|
||||
),
|
||||
pagesWithNoTextCount: z.number().int(),
|
||||
pagesWithSuspiciousTextCount: z.number().int(),
|
||||
warnings: warningsSchema
|
||||
})
|
||||
|
||||
export const pdfReadPagesOutputSchema = z.object({
|
||||
sourcePath: z.string(),
|
||||
pageCount: z.number().int(),
|
||||
pages: z.array(pdfPageSchema),
|
||||
warnings: warningsSchema
|
||||
})
|
||||
|
||||
export const pdfRenderPagesOutputSchema = z.object({
|
||||
sourcePath: z.string(),
|
||||
pageCount: z.number().int(),
|
||||
assetBucketId: z.string(),
|
||||
images: z.array(
|
||||
z.object({
|
||||
pageNumber: z.number().int(),
|
||||
path: z.string(),
|
||||
mimeType: z.literal('image/png'),
|
||||
width: z.number().int(),
|
||||
height: z.number().int()
|
||||
})
|
||||
),
|
||||
warnings: warningsSchema
|
||||
})
|
||||
@@ -0,0 +1,214 @@
|
||||
import fs from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import type { AiToolCallResult, JsonObject } from '../../../../../../shared/types'
|
||||
import type { PdfOpenResult } from '../../../../pdf/types'
|
||||
import { getPdfWorkerClient } from '../../../../pdf/worker/pdfWorkerClient'
|
||||
import type { PdfWorkerRequest, PdfWorkerResponse } from '../../../../pdf/worker/types'
|
||||
import { isPathInsideRoot, resolveWorkspacePath } from '../terminal/commandPolicy'
|
||||
import { createToolEventEmitter } from '../utils/toolEvents'
|
||||
import type { ToolEventEmitter } from '../utils/toolEvents'
|
||||
import type { PdfCapabilityToolsInput } from './types'
|
||||
|
||||
type FailedPdfWorkerResponse = Extract<PdfWorkerResponse, { ok: false }>
|
||||
|
||||
export class PdfToolRuntimeError extends Error {
|
||||
public readonly reason: FailedPdfWorkerResponse['code']
|
||||
public readonly warnings: string[]
|
||||
|
||||
public constructor(response: FailedPdfWorkerResponse) {
|
||||
super(response.error)
|
||||
this.name = 'PdfToolRuntimeError'
|
||||
this.reason = response.code
|
||||
this.warnings = response.warnings || []
|
||||
}
|
||||
}
|
||||
|
||||
export type PdfToolResourceContext = {
|
||||
family: 'pdf'
|
||||
sourcePath: string
|
||||
effectKind: 'inspected' | 'searched' | 'read' | 'rendered'
|
||||
}
|
||||
|
||||
export type PdfToolFailure = {
|
||||
ok: false
|
||||
error: string
|
||||
warnings: string[]
|
||||
reason: string
|
||||
decisionHint?: JsonObject
|
||||
resourceContext?: PdfToolResourceContext
|
||||
}
|
||||
|
||||
export type PdfReadSession = PdfOpenResult
|
||||
|
||||
export type PdfToolRuntime = {
|
||||
input: PdfCapabilityToolsInput
|
||||
workspaceRootPath: string
|
||||
toolEvents: ToolEventEmitter
|
||||
resolvePdfPath: (rawPath: string) => Promise<string>
|
||||
ensureDocument: (sourcePath: string) => Promise<PdfReadSession>
|
||||
callWorker: (request: Omit<PdfWorkerRequest, 'requestId' | 'runId'>) => Promise<PdfWorkerResponse>
|
||||
buildResourceContext: (
|
||||
sourcePath: string,
|
||||
effectKind: PdfToolResourceContext['effectKind']
|
||||
) => PdfToolResourceContext
|
||||
emitFailure: (input: {
|
||||
toolName: string
|
||||
toolCallId: string
|
||||
error: string
|
||||
reason: string
|
||||
warnings?: string[]
|
||||
decisionHint?: JsonObject
|
||||
resourceContext?: PdfToolResourceContext
|
||||
}) => PdfToolFailure
|
||||
emitWorkerFailure: (input: {
|
||||
toolName: string
|
||||
toolCallId: string
|
||||
response: FailedPdfWorkerResponse
|
||||
resourceContext?: PdfToolResourceContext
|
||||
}) => PdfToolFailure
|
||||
emitSuccess: (input: {
|
||||
toolName: string
|
||||
toolCallId: string
|
||||
result: JsonObject
|
||||
resourceContext: PdfToolResourceContext
|
||||
}) => void
|
||||
}
|
||||
|
||||
const buildPdfToolCallResult = (input: {
|
||||
toolName: string
|
||||
ok: boolean
|
||||
data: AiToolCallResult['data']
|
||||
resourceContext?: PdfToolResourceContext
|
||||
error?: string
|
||||
}): AiToolCallResult => ({
|
||||
toolName: input.toolName,
|
||||
ok: input.ok,
|
||||
data: input.data,
|
||||
resourceContext: input.resourceContext,
|
||||
diagnostic: input.ok || !input.error ? undefined : { message: input.error }
|
||||
})
|
||||
|
||||
export const createPdfToolRuntime = (input: PdfCapabilityToolsInput): PdfToolRuntime => {
|
||||
const workspaceRootPath = path.resolve(input.workspaceRootPath)
|
||||
const sessionBySourcePath = new Map<string, PdfReadSession>()
|
||||
const toolEvents = createToolEventEmitter({
|
||||
ctx: input.ctx,
|
||||
runId: input.runId,
|
||||
taskId: input.taskId,
|
||||
stage: 'human_hand'
|
||||
})
|
||||
|
||||
const callWorker: PdfToolRuntime['callWorker'] = async (request) =>
|
||||
await getPdfWorkerClient().call({ ...request, runId: input.runId } as PdfWorkerRequest)
|
||||
|
||||
const resolvePdfPath = async (rawPath: string): Promise<string> => {
|
||||
const lexicalPath = resolveWorkspacePath({
|
||||
rawPath,
|
||||
cwd: workspaceRootPath,
|
||||
rootPath: workspaceRootPath
|
||||
})
|
||||
if (!lexicalPath) throw new Error('PDF 路径必须位于当前工作区内')
|
||||
if (path.extname(lexicalPath).toLowerCase() !== '.pdf')
|
||||
throw new Error('sourcePath 必须是 .pdf 文件')
|
||||
|
||||
const [realWorkspaceRoot, realSourcePath] = await Promise.all([
|
||||
fs.realpath(workspaceRootPath),
|
||||
fs.realpath(lexicalPath)
|
||||
])
|
||||
if (!isPathInsideRoot(realSourcePath, realWorkspaceRoot)) {
|
||||
throw new Error('PDF 真实路径必须位于当前工作区内')
|
||||
}
|
||||
if (path.extname(realSourcePath).toLowerCase() !== '.pdf')
|
||||
throw new Error('sourcePath 必须是 .pdf 文件')
|
||||
const stat = await fs.stat(realSourcePath)
|
||||
if (!stat.isFile()) throw new Error('sourcePath 不是文件')
|
||||
const handle = await fs.open(realSourcePath, 'r')
|
||||
try {
|
||||
const header = Buffer.alloc(5)
|
||||
const { bytesRead } = await handle.read(header, 0, header.length, 0)
|
||||
if (bytesRead !== 5 || header.toString('ascii') !== '%PDF-') {
|
||||
throw new Error('文件头无效,不是可读取的 PDF')
|
||||
}
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
return realSourcePath
|
||||
}
|
||||
|
||||
const ensureDocument = async (rawSourcePath: string): Promise<PdfReadSession> => {
|
||||
const sourcePath = await resolvePdfPath(rawSourcePath)
|
||||
const cached = sessionBySourcePath.get(sourcePath)
|
||||
if (cached) return cached
|
||||
const response = await callWorker({ action: 'open', payload: { sourcePath } })
|
||||
if (!response.ok) throw new PdfToolRuntimeError(response)
|
||||
if (response.payload.action !== 'open') throw new Error('PDF worker 协议错误(open)')
|
||||
const session: PdfReadSession = response.payload
|
||||
sessionBySourcePath.set(sourcePath, session)
|
||||
return session
|
||||
}
|
||||
|
||||
const buildResourceContext: PdfToolRuntime['buildResourceContext'] = (
|
||||
sourcePath,
|
||||
effectKind
|
||||
) => ({ family: 'pdf', sourcePath, effectKind })
|
||||
|
||||
const emitFailure: PdfToolRuntime['emitFailure'] = (params) => {
|
||||
const failure: PdfToolFailure = {
|
||||
ok: false,
|
||||
error: params.error,
|
||||
warnings: params.warnings || [],
|
||||
reason: params.reason,
|
||||
decisionHint: params.decisionHint,
|
||||
resourceContext: params.resourceContext
|
||||
}
|
||||
toolEvents.emitResult({
|
||||
toolName: params.toolName,
|
||||
toolCallId: params.toolCallId,
|
||||
error: params.error,
|
||||
result: buildPdfToolCallResult({
|
||||
toolName: params.toolName,
|
||||
ok: false,
|
||||
data: failure,
|
||||
resourceContext: params.resourceContext,
|
||||
error: params.error
|
||||
})
|
||||
})
|
||||
return failure
|
||||
}
|
||||
|
||||
const emitWorkerFailure: PdfToolRuntime['emitWorkerFailure'] = (params) =>
|
||||
emitFailure({
|
||||
toolName: params.toolName,
|
||||
toolCallId: params.toolCallId,
|
||||
error: params.response.error,
|
||||
reason: params.response.code,
|
||||
warnings: params.response.warnings,
|
||||
resourceContext: params.resourceContext
|
||||
})
|
||||
|
||||
const emitSuccess: PdfToolRuntime['emitSuccess'] = (params) => {
|
||||
toolEvents.emitResult({
|
||||
toolName: params.toolName,
|
||||
toolCallId: params.toolCallId,
|
||||
result: buildPdfToolCallResult({
|
||||
toolName: params.toolName,
|
||||
ok: true,
|
||||
data: params.result,
|
||||
resourceContext: params.resourceContext
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
input,
|
||||
workspaceRootPath,
|
||||
toolEvents,
|
||||
resolvePdfPath,
|
||||
ensureDocument,
|
||||
callWorker,
|
||||
buildResourceContext,
|
||||
emitFailure,
|
||||
emitWorkerFailure,
|
||||
emitSuccess
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { z } from 'zod/v4'
|
||||
|
||||
const sourcePathSchema = z.string().trim().min(1)
|
||||
const pageNumberSchema = z.number().int().min(1)
|
||||
|
||||
export const pdfInspectSchema = z.object({
|
||||
sourcePath: sourcePathSchema
|
||||
})
|
||||
|
||||
export const pdfPageRangeSchema = z.object({
|
||||
fromPage: pageNumberSchema,
|
||||
toPage: pageNumberSchema
|
||||
})
|
||||
|
||||
export const pdfSearchSchema = z.object({
|
||||
sourcePath: sourcePathSchema,
|
||||
query: z.string().min(1),
|
||||
mode: z.enum(['literal', 'regex']).optional(),
|
||||
pageRange: pdfPageRangeSchema.optional(),
|
||||
maxMatches: z.number().int().min(1).max(50).optional()
|
||||
})
|
||||
|
||||
export const pdfReadPagesSchema = z.object({
|
||||
sourcePath: sourcePathSchema,
|
||||
selection: z.discriminatedUnion('type', [
|
||||
z.object({
|
||||
type: z.literal('page_range'),
|
||||
fromPage: pageNumberSchema,
|
||||
toPage: pageNumberSchema
|
||||
}),
|
||||
z.object({
|
||||
type: z.literal('page_numbers'),
|
||||
pageNumbers: z.array(pageNumberSchema).min(1).max(50)
|
||||
})
|
||||
])
|
||||
})
|
||||
|
||||
export const pdfRenderPagesSchema = z.object({
|
||||
sourcePath: sourcePathSchema,
|
||||
pageNumbers: z.array(pageNumberSchema).min(1).max(20)
|
||||
})
|
||||
@@ -0,0 +1,183 @@
|
||||
import type { z } from 'zod/v4'
|
||||
import type { JsonObject } from '../../../../../../shared/types'
|
||||
import type { RuntimeSkillId } from '../../types'
|
||||
import type { HumanToolSpec } from '../types'
|
||||
import { createStaticExecutionMetaResolver, readStringArg } from '../utils/toolSpecUtils'
|
||||
import {
|
||||
pdfInspectOutputSchema,
|
||||
pdfReadPagesOutputSchema,
|
||||
pdfRenderPagesOutputSchema,
|
||||
pdfSearchOutputSchema
|
||||
} from './outputSchemas'
|
||||
import {
|
||||
pdfInspectSchema,
|
||||
pdfReadPagesSchema,
|
||||
pdfRenderPagesSchema,
|
||||
pdfSearchSchema
|
||||
} from './schemas'
|
||||
|
||||
const PDF_READ_EXECUTION_META = createStaticExecutionMetaResolver({
|
||||
mutability: 'read_only',
|
||||
getResourceKey: (args) => readStringArg(args, 'sourcePath'),
|
||||
defaultResourceKey: 'pdf'
|
||||
})
|
||||
|
||||
type PdfToolSpec<
|
||||
TArgs extends JsonObject = JsonObject,
|
||||
TOutput extends JsonObject = JsonObject,
|
||||
TName extends string = string,
|
||||
TSkillId extends RuntimeSkillId = 'pdf-read'
|
||||
> = HumanToolSpec<TArgs, TOutput, TName, TSkillId> & {
|
||||
description: string
|
||||
}
|
||||
|
||||
const PDF_SOURCE_PATH_EXAMPLE = '学习笔记/深度学习.pdf'
|
||||
|
||||
const createPdfPromptSpec = (input: {
|
||||
name: string
|
||||
description: string
|
||||
whenToUse: string
|
||||
examples: Array<{ goal: string; args: JsonObject }>
|
||||
}) => ({
|
||||
name: input.name,
|
||||
description: input.description,
|
||||
whenToUse: input.whenToUse,
|
||||
examples: input.examples
|
||||
})
|
||||
|
||||
export const PDF_INSPECT_TOOL_SPEC: PdfToolSpec<
|
||||
z.infer<typeof pdfInspectSchema>,
|
||||
z.infer<typeof pdfInspectOutputSchema>,
|
||||
'pdf_inspect',
|
||||
'pdf-read'
|
||||
> = {
|
||||
name: 'pdf_inspect',
|
||||
familyId: 'pdf',
|
||||
skillIds: ['pdf-read'],
|
||||
requiresWorkspace: true,
|
||||
description:
|
||||
'能力:读取 sourcePath 对应 PDF 的基础信息。\n结果:返回页数、元数据、书签提纲和警告。\n边界:只检查 PDF,不返回各页正文,不修改源文件。',
|
||||
promptSpec: createPdfPromptSpec({
|
||||
name: 'pdf_inspect',
|
||||
description:
|
||||
'能力:读取 sourcePath 对应 PDF 的基础信息。\n结果:返回页数、元数据、书签提纲和警告。\n边界:只检查 PDF,不返回各页正文,不修改源文件。',
|
||||
whenToUse: '适用于需要先了解 PDF 页数、元数据或目录结构的任务。',
|
||||
examples: [
|
||||
{
|
||||
goal: '先查看 PDF 的页数、标题和目录结构',
|
||||
args: { sourcePath: PDF_SOURCE_PATH_EXAMPLE }
|
||||
}
|
||||
]
|
||||
}),
|
||||
inputSchema: pdfInspectSchema,
|
||||
outputSchema: pdfInspectOutputSchema,
|
||||
resolveExecutionMeta: PDF_READ_EXECUTION_META
|
||||
}
|
||||
|
||||
export const PDF_SEARCH_TOOL_SPEC: PdfToolSpec<
|
||||
z.infer<typeof pdfSearchSchema>,
|
||||
z.infer<typeof pdfSearchOutputSchema>,
|
||||
'pdf_search',
|
||||
'pdf-read'
|
||||
> = {
|
||||
name: 'pdf_search',
|
||||
familyId: 'pdf',
|
||||
skillIds: ['pdf-read'],
|
||||
requiresWorkspace: true,
|
||||
description:
|
||||
'能力:在 sourcePath 对应 PDF 的文本层中按 literal 或 regex 搜索,可限制页码范围和返回数量。\n结果:返回命中页码、匹配文本、上下文片段、位置和文本层状态统计。\n边界:只搜索 PDF 文本层;扫描图片中的文字不会自动识别。',
|
||||
promptSpec: createPdfPromptSpec({
|
||||
name: 'pdf_search',
|
||||
description:
|
||||
'能力:在 sourcePath 对应 PDF 的文本层中按 literal 或 regex 搜索,可限制页码范围和返回数量。\n结果:返回命中页码、匹配文本、上下文片段、位置和文本层状态统计。\n边界:只搜索 PDF 文本层;扫描图片中的文字不会自动识别。',
|
||||
whenToUse: '适用于需要在 PDF 正文中定位关键词或表达式的任务。',
|
||||
examples: [
|
||||
{
|
||||
goal: '按原词定位 PDF 中讨论过度拟合的页面',
|
||||
args: { sourcePath: PDF_SOURCE_PATH_EXAMPLE, query: '过度拟合' }
|
||||
},
|
||||
{
|
||||
goal: '用正则匹配 PDF 文本层中空格可能变化的英文术语',
|
||||
args: { sourcePath: PDF_SOURCE_PATH_EXAMPLE, query: 'Proper\\s+fit', mode: 'regex' }
|
||||
}
|
||||
]
|
||||
}),
|
||||
inputSchema: pdfSearchSchema,
|
||||
outputSchema: pdfSearchOutputSchema,
|
||||
resolveExecutionMeta: PDF_READ_EXECUTION_META
|
||||
}
|
||||
|
||||
export const PDF_READ_PAGES_TOOL_SPEC: PdfToolSpec<
|
||||
z.infer<typeof pdfReadPagesSchema>,
|
||||
z.infer<typeof pdfReadPagesOutputSchema>,
|
||||
'pdf_read_pages',
|
||||
'pdf-read'
|
||||
> = {
|
||||
name: 'pdf_read_pages',
|
||||
familyId: 'pdf',
|
||||
skillIds: ['pdf-read'],
|
||||
requiresWorkspace: true,
|
||||
description:
|
||||
'能力:按连续页码范围或指定页码集合读取 sourcePath 对应 PDF 的文本层。\n结果:逐页返回正文、页码、文本状态、图片数量、页面尺寸和警告。\n边界:不理解页面图片;空文本或可疑文本会明确标记,不修改源文件。',
|
||||
promptSpec: createPdfPromptSpec({
|
||||
name: 'pdf_read_pages',
|
||||
description:
|
||||
'能力:按连续页码范围或指定页码集合读取 sourcePath 对应 PDF 的文本层。\n结果:逐页返回正文、页码、文本状态、图片数量、页面尺寸和警告。\n边界:不理解页面图片;空文本或可疑文本会明确标记,不修改源文件。',
|
||||
whenToUse: '适用于需要精读 PDF 某一页或连续页码范围正文的任务。',
|
||||
examples: [
|
||||
{
|
||||
goal: '连续读取 PDF 前三页正文',
|
||||
args: {
|
||||
sourcePath: PDF_SOURCE_PATH_EXAMPLE,
|
||||
selection: { type: 'page_range', fromPage: 1, toPage: 3 }
|
||||
}
|
||||
},
|
||||
{
|
||||
goal: '只读取任务明确关心的第 1 页和第 9 页',
|
||||
args: {
|
||||
sourcePath: PDF_SOURCE_PATH_EXAMPLE,
|
||||
selection: { type: 'page_numbers', pageNumbers: [1, 9] }
|
||||
}
|
||||
}
|
||||
]
|
||||
}),
|
||||
inputSchema: pdfReadPagesSchema,
|
||||
outputSchema: pdfReadPagesOutputSchema,
|
||||
resolveExecutionMeta: PDF_READ_EXECUTION_META
|
||||
}
|
||||
|
||||
export const PDF_RENDER_PAGES_TOOL_SPEC: PdfToolSpec<
|
||||
z.infer<typeof pdfRenderPagesSchema>,
|
||||
z.infer<typeof pdfRenderPagesOutputSchema>,
|
||||
'pdf_render_pages',
|
||||
'pdf-read'
|
||||
> = {
|
||||
name: 'pdf_render_pages',
|
||||
familyId: 'pdf',
|
||||
skillIds: ['pdf-read'],
|
||||
requiresWorkspace: true,
|
||||
description:
|
||||
'能力:将 sourcePath 对应 PDF 的指定页面渲染为 PNG 图片。\n结果:返回每页的页码、图片绝对路径、格式和尺寸;图片位于当前工作区 .ncurator/runtime/assets/<uuid>/ 下。\n边界:只生成页面图片,不识别或解释图片内容,不把图片记为文件修改或普通 artifact。',
|
||||
promptSpec: createPdfPromptSpec({
|
||||
name: 'pdf_render_pages',
|
||||
description:
|
||||
'能力:将 sourcePath 对应 PDF 的指定页面渲染为 PNG 图片。\n结果:返回每页的页码、图片绝对路径、格式和尺寸;图片位于当前工作区 .ncurator/runtime/assets/<uuid>/ 下。\n边界:只生成页面图片,不识别或解释图片内容,不把图片记为文件修改或普通 artifact。',
|
||||
whenToUse: '适用于 PDF 文本层不足以回答问题,需要查看页面图形或把页面交给图像识别工具的任务。',
|
||||
examples: [
|
||||
{
|
||||
goal: '把包含对比曲线的第 9 页渲染成图片,供后续图像识别',
|
||||
args: { sourcePath: PDF_SOURCE_PATH_EXAMPLE, pageNumbers: [9] }
|
||||
}
|
||||
]
|
||||
}),
|
||||
inputSchema: pdfRenderPagesSchema,
|
||||
outputSchema: pdfRenderPagesOutputSchema,
|
||||
resolveExecutionMeta: PDF_READ_EXECUTION_META
|
||||
}
|
||||
|
||||
export const PDF_TOOL_SPECS = [
|
||||
PDF_INSPECT_TOOL_SPEC,
|
||||
PDF_SEARCH_TOOL_SPEC,
|
||||
PDF_READ_PAGES_TOOL_SPEC,
|
||||
PDF_RENDER_PAGES_TOOL_SPEC
|
||||
] as const
|
||||
@@ -0,0 +1,279 @@
|
||||
import { tool } from '@langchain/core/tools'
|
||||
import type { StructuredToolInterface } from '@langchain/core/tools'
|
||||
import type { JsonObject } from '../../../../../../../shared/types'
|
||||
import { withToolLifecycle } from '../../utils/lifecycle'
|
||||
import { checkToolResultSize } from '../../utils/resultSizeGuard'
|
||||
import {
|
||||
PdfToolRuntimeError,
|
||||
type PdfToolFailure,
|
||||
type PdfToolResourceContext,
|
||||
type PdfToolRuntime
|
||||
} from '../runtime'
|
||||
import {
|
||||
PDF_INSPECT_TOOL_SPEC,
|
||||
PDF_READ_PAGES_TOOL_SPEC,
|
||||
PDF_RENDER_PAGES_TOOL_SPEC,
|
||||
PDF_SEARCH_TOOL_SPEC
|
||||
} from '../toolSpecs'
|
||||
|
||||
const uniquePageNumbers = (pageNumbers: number[]): number[] => [...new Set(pageNumbers)]
|
||||
|
||||
const runTool = async (input: {
|
||||
runtime: PdfToolRuntime
|
||||
toolName: string
|
||||
toolCallId: string
|
||||
sourcePath: string
|
||||
effectKind: PdfToolResourceContext['effectKind']
|
||||
execute: (
|
||||
session: Awaited<ReturnType<PdfToolRuntime['ensureDocument']>>
|
||||
) => Promise<JsonObject | PdfToolFailure>
|
||||
}): Promise<JsonObject | PdfToolFailure> => {
|
||||
try {
|
||||
input.runtime.input.ctx.checkCancelled()
|
||||
const document = await input.runtime.ensureDocument(input.sourcePath)
|
||||
const resourceContext = input.runtime.buildResourceContext(
|
||||
document.sourcePath,
|
||||
input.effectKind
|
||||
)
|
||||
const result = await input.execute(document)
|
||||
if ('ok' in result && result.ok === false) return result
|
||||
const size = checkToolResultSize({
|
||||
toolName: input.toolName,
|
||||
value: result,
|
||||
modelContextWindowTokens: input.runtime.input.modelContextWindowTokens
|
||||
})
|
||||
if (size.tooLarge) {
|
||||
return input.runtime.emitFailure({
|
||||
toolName: input.toolName,
|
||||
toolCallId: input.toolCallId,
|
||||
reason: 'result_too_large',
|
||||
error: `PDF 工具结果过大:预计 ${String(size.estimatedTokens)} tokens,限制 ${String(size.limit)} tokens`,
|
||||
decisionHint: size.decisionHint,
|
||||
resourceContext
|
||||
})
|
||||
}
|
||||
input.runtime.emitSuccess({
|
||||
toolName: input.toolName,
|
||||
toolCallId: input.toolCallId,
|
||||
result,
|
||||
resourceContext
|
||||
})
|
||||
return result
|
||||
} catch (error) {
|
||||
return input.runtime.emitFailure({
|
||||
toolName: input.toolName,
|
||||
toolCallId: input.toolCallId,
|
||||
reason: error instanceof PdfToolRuntimeError ? error.reason : 'invalid_input',
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
warnings: error instanceof PdfToolRuntimeError ? error.warnings : undefined
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const buildPdfReadTools = (runtime: PdfToolRuntime): StructuredToolInterface[] => {
|
||||
const inspectTool = tool(
|
||||
async (args) =>
|
||||
await withToolLifecycle({
|
||||
emitter: runtime.toolEvents,
|
||||
toolName: PDF_INSPECT_TOOL_SPEC.name,
|
||||
args: args as JsonObject,
|
||||
execute: async ({ toolCallId }) =>
|
||||
await runTool({
|
||||
runtime,
|
||||
toolName: PDF_INSPECT_TOOL_SPEC.name,
|
||||
toolCallId,
|
||||
sourcePath: args.sourcePath,
|
||||
effectKind: 'inspected',
|
||||
execute: async (document) => ({
|
||||
sourcePath: document.sourcePath,
|
||||
pageCount: document.pageCount,
|
||||
metadata: document.metadata,
|
||||
outline: document.outline,
|
||||
encrypted: document.encrypted,
|
||||
warnings: document.warnings
|
||||
})
|
||||
})
|
||||
}),
|
||||
{
|
||||
name: PDF_INSPECT_TOOL_SPEC.name,
|
||||
description: PDF_INSPECT_TOOL_SPEC.description,
|
||||
schema: PDF_INSPECT_TOOL_SPEC.inputSchema
|
||||
}
|
||||
)
|
||||
|
||||
const searchTool = tool(
|
||||
async (args) =>
|
||||
await withToolLifecycle({
|
||||
emitter: runtime.toolEvents,
|
||||
toolName: PDF_SEARCH_TOOL_SPEC.name,
|
||||
args: args as JsonObject,
|
||||
execute: async ({ toolCallId }) =>
|
||||
await runTool({
|
||||
runtime,
|
||||
toolName: PDF_SEARCH_TOOL_SPEC.name,
|
||||
toolCallId,
|
||||
sourcePath: args.sourcePath,
|
||||
effectKind: 'searched',
|
||||
execute: async (document) => {
|
||||
const fromPage = args.pageRange?.fromPage ?? 1
|
||||
const toPage = args.pageRange?.toPage ?? document.pageCount
|
||||
const response = await runtime.callWorker({
|
||||
action: 'search',
|
||||
payload: {
|
||||
docId: document.docId,
|
||||
query: args.query,
|
||||
mode: args.mode || 'literal',
|
||||
fromPage,
|
||||
toPage,
|
||||
maxMatches: args.maxMatches || 20
|
||||
}
|
||||
})
|
||||
const resourceContext = runtime.buildResourceContext(document.sourcePath, 'searched')
|
||||
if (!response.ok) {
|
||||
return runtime.emitWorkerFailure({
|
||||
toolName: PDF_SEARCH_TOOL_SPEC.name,
|
||||
toolCallId,
|
||||
response,
|
||||
resourceContext
|
||||
})
|
||||
}
|
||||
if (response.payload.action !== 'search')
|
||||
throw new Error('PDF worker 协议错误(search)')
|
||||
return {
|
||||
sourcePath: document.sourcePath,
|
||||
pageCount: document.pageCount,
|
||||
query: args.query,
|
||||
mode: args.mode || 'literal',
|
||||
pageRange: { fromPage, toPage },
|
||||
totalMatchCount: response.payload.totalMatchCount,
|
||||
returnedMatchCount: response.payload.matches.length,
|
||||
matches: response.payload.matches,
|
||||
pagesWithNoTextCount: response.payload.pagesWithNoTextCount,
|
||||
pagesWithSuspiciousTextCount: response.payload.pagesWithSuspiciousTextCount,
|
||||
warnings: response.payload.warnings
|
||||
}
|
||||
}
|
||||
})
|
||||
}),
|
||||
{
|
||||
name: PDF_SEARCH_TOOL_SPEC.name,
|
||||
description: PDF_SEARCH_TOOL_SPEC.description,
|
||||
schema: PDF_SEARCH_TOOL_SPEC.inputSchema
|
||||
}
|
||||
)
|
||||
|
||||
const readPagesTool = tool(
|
||||
async (args) =>
|
||||
await withToolLifecycle({
|
||||
emitter: runtime.toolEvents,
|
||||
toolName: PDF_READ_PAGES_TOOL_SPEC.name,
|
||||
args: args as JsonObject,
|
||||
execute: async ({ toolCallId }) =>
|
||||
await runTool({
|
||||
runtime,
|
||||
toolName: PDF_READ_PAGES_TOOL_SPEC.name,
|
||||
toolCallId,
|
||||
sourcePath: args.sourcePath,
|
||||
effectKind: 'read',
|
||||
execute: async (document) => {
|
||||
const selection = args.selection
|
||||
let pageNumbers: number[]
|
||||
if (selection.type === 'page_numbers') {
|
||||
pageNumbers = uniquePageNumbers(selection.pageNumbers)
|
||||
} else {
|
||||
if (selection.fromPage > selection.toPage) {
|
||||
throw new Error('fromPage 不能大于 toPage')
|
||||
}
|
||||
const pageCount = selection.toPage - selection.fromPage + 1
|
||||
if (pageCount > 50) throw new Error('单次最多读取 50 页')
|
||||
pageNumbers = Array.from(
|
||||
{ length: pageCount },
|
||||
(_, index) => selection.fromPage + index
|
||||
)
|
||||
}
|
||||
if (pageNumbers.length === 0 || pageNumbers.length > 50) {
|
||||
throw new Error('单次最多读取 50 页,且页码范围不能为空')
|
||||
}
|
||||
const response = await runtime.callWorker({
|
||||
action: 'read_pages',
|
||||
payload: { docId: document.docId, pageNumbers }
|
||||
})
|
||||
const resourceContext = runtime.buildResourceContext(document.sourcePath, 'read')
|
||||
if (!response.ok) {
|
||||
return runtime.emitWorkerFailure({
|
||||
toolName: PDF_READ_PAGES_TOOL_SPEC.name,
|
||||
toolCallId,
|
||||
response,
|
||||
resourceContext
|
||||
})
|
||||
}
|
||||
if (response.payload.action !== 'read_pages')
|
||||
throw new Error('PDF worker 协议错误(read_pages)')
|
||||
return {
|
||||
sourcePath: document.sourcePath,
|
||||
pageCount: document.pageCount,
|
||||
pages: response.payload.pages,
|
||||
warnings: response.payload.pages.flatMap((page) => page.warnings)
|
||||
}
|
||||
}
|
||||
})
|
||||
}),
|
||||
{
|
||||
name: PDF_READ_PAGES_TOOL_SPEC.name,
|
||||
description: PDF_READ_PAGES_TOOL_SPEC.description,
|
||||
schema: PDF_READ_PAGES_TOOL_SPEC.inputSchema
|
||||
}
|
||||
)
|
||||
|
||||
const renderPagesTool = tool(
|
||||
async (args) =>
|
||||
await withToolLifecycle({
|
||||
emitter: runtime.toolEvents,
|
||||
toolName: PDF_RENDER_PAGES_TOOL_SPEC.name,
|
||||
args: args as JsonObject,
|
||||
execute: async ({ toolCallId }) =>
|
||||
await runTool({
|
||||
runtime,
|
||||
toolName: PDF_RENDER_PAGES_TOOL_SPEC.name,
|
||||
toolCallId,
|
||||
sourcePath: args.sourcePath,
|
||||
effectKind: 'rendered',
|
||||
execute: async (document) => {
|
||||
const response = await runtime.callWorker({
|
||||
action: 'render_pages',
|
||||
payload: {
|
||||
docId: document.docId,
|
||||
workspaceRootPath: runtime.workspaceRootPath,
|
||||
pageNumbers: uniquePageNumbers(args.pageNumbers)
|
||||
}
|
||||
})
|
||||
const resourceContext = runtime.buildResourceContext(document.sourcePath, 'rendered')
|
||||
if (!response.ok) {
|
||||
return runtime.emitWorkerFailure({
|
||||
toolName: PDF_RENDER_PAGES_TOOL_SPEC.name,
|
||||
toolCallId,
|
||||
response,
|
||||
resourceContext
|
||||
})
|
||||
}
|
||||
if (response.payload.action !== 'render_pages')
|
||||
throw new Error('PDF worker 协议错误(render_pages)')
|
||||
return {
|
||||
sourcePath: document.sourcePath,
|
||||
pageCount: document.pageCount,
|
||||
assetBucketId: response.payload.assetBucketId,
|
||||
images: response.payload.images,
|
||||
warnings: response.payload.warnings
|
||||
}
|
||||
}
|
||||
})
|
||||
}),
|
||||
{
|
||||
name: PDF_RENDER_PAGES_TOOL_SPEC.name,
|
||||
description: PDF_RENDER_PAGES_TOOL_SPEC.description,
|
||||
schema: PDF_RENDER_PAGES_TOOL_SPEC.inputSchema
|
||||
}
|
||||
)
|
||||
|
||||
return [inspectTool, searchTool, readPagesTool, renderPagesTool]
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { z } from 'zod/v4'
|
||||
import type { WorkspaceCapabilityBuildInput } from '../types'
|
||||
import type {
|
||||
pdfInspectSchema,
|
||||
pdfReadPagesSchema,
|
||||
pdfRenderPagesSchema,
|
||||
pdfSearchSchema
|
||||
} from './schemas'
|
||||
|
||||
export type PdfCapabilityToolsInput = WorkspaceCapabilityBuildInput & {
|
||||
safePreviewChars: number
|
||||
modelContextWindowTokens: number
|
||||
}
|
||||
|
||||
export type PdfInspectRequest = z.infer<typeof pdfInspectSchema>
|
||||
export type PdfSearchRequest = z.infer<typeof pdfSearchSchema>
|
||||
export type PdfReadPagesRequest = z.infer<typeof pdfReadPagesSchema>
|
||||
export type PdfRenderPagesRequest = z.infer<typeof pdfRenderPagesSchema>
|
||||
@@ -1,6 +1,7 @@
|
||||
import { DOCX_TOOL_SPECS } from './docx/toolSpecs'
|
||||
import { IMAGE_RECOGNITION_TOOL_SPECS } from './imageRecognition/toolSpecs'
|
||||
import { KB_TOOL_SPECS } from './kb/toolSpecs'
|
||||
import { PDF_TOOL_SPECS } from './pdf/toolSpecs'
|
||||
import { UTILITY_TOOL_SPECS } from './utility/toolSpecs'
|
||||
import { WEB_TOOL_SPECS } from './web/toolSpecs'
|
||||
import {
|
||||
@@ -17,6 +18,7 @@ export const ALL_TOOL_SPECS = [
|
||||
...TEXT_FILE_TOOL_SPECS,
|
||||
...KB_TOOL_SPECS,
|
||||
...DOCX_TOOL_SPECS,
|
||||
...PDF_TOOL_SPECS,
|
||||
...WEB_TOOL_SPECS,
|
||||
...UTILITY_TOOL_SPECS,
|
||||
...IMAGE_RECOGNITION_TOOL_SPECS
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { JsonObject } from '../../../../shared/types'
|
||||
import type { z } from 'zod/v4'
|
||||
import type { AiToolFamilyId } from './tools/types'
|
||||
import type { DocxToolRuntime } from './tools/docx/runtime'
|
||||
import type { PdfToolRuntime } from './tools/pdf/runtime'
|
||||
import {
|
||||
HUMAN_ROUND_EXECUTION_KINDS,
|
||||
HUMAN_SKILL_SOURCES,
|
||||
@@ -105,6 +106,7 @@ export type HumanRuntimeCache = {
|
||||
familyToolsets: Partial<Record<AiToolFamilyId, StructuredToolInterface[]>>
|
||||
skillToolsets: Partial<Record<RuntimeSkillId, StructuredToolInterface[]>>
|
||||
docxRuntime?: DocxToolRuntime
|
||||
pdfRuntime?: PdfToolRuntime
|
||||
approvedTerminalCommandCounts?: Map<string, number>
|
||||
approvedBrowserUseTaskIds?: Set<string>
|
||||
toolCallCount: number
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import fs from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
|
||||
const WORKSPACE_NCURATOR_DIR = '.ncurator'
|
||||
const WORKSPACE_RUNTIME_DIR = 'runtime'
|
||||
const WORKSPACE_ASSETS_DIR = 'assets'
|
||||
const WORKSPACE_SCRIPTS_DIR = 'scripts'
|
||||
|
||||
const normalizeWorkspaceRoot = (workspaceRootPath: string): string => {
|
||||
const value = String(workspaceRootPath || '').trim()
|
||||
if (!value) throw new Error('缺少工作区根目录')
|
||||
return path.resolve(value)
|
||||
}
|
||||
|
||||
export const getWorkspaceRuntimeDir = (workspaceRootPath: string): string =>
|
||||
path.join(
|
||||
normalizeWorkspaceRoot(workspaceRootPath),
|
||||
WORKSPACE_NCURATOR_DIR,
|
||||
WORKSPACE_RUNTIME_DIR
|
||||
)
|
||||
|
||||
export const getWorkspaceAssetsRootDir = (workspaceRootPath: string): string =>
|
||||
path.join(getWorkspaceRuntimeDir(workspaceRootPath), WORKSPACE_ASSETS_DIR)
|
||||
|
||||
export const getWorkspaceAssetBucketDir = (
|
||||
workspaceRootPath: string,
|
||||
assetBucketId: string
|
||||
): string => {
|
||||
const normalizedBucketId = String(assetBucketId || '').trim()
|
||||
if (!normalizedBucketId || path.basename(normalizedBucketId) !== normalizedBucketId) {
|
||||
throw new Error('assetBucketId 无效')
|
||||
}
|
||||
return path.join(getWorkspaceAssetsRootDir(workspaceRootPath), normalizedBucketId)
|
||||
}
|
||||
|
||||
export const ensureWorkspaceRuntimeDirs = async (workspaceRootPath: string): Promise<void> => {
|
||||
const runtimeDir = getWorkspaceRuntimeDir(workspaceRootPath)
|
||||
await Promise.all([
|
||||
fs.mkdir(path.join(runtimeDir, WORKSPACE_ASSETS_DIR), { recursive: true }),
|
||||
fs.mkdir(path.join(runtimeDir, WORKSPACE_SCRIPTS_DIR), { recursive: true })
|
||||
])
|
||||
}
|
||||
|
||||
export const ensureWorkspaceAssetBucket = async (
|
||||
workspaceRootPath: string,
|
||||
assetBucketId: string
|
||||
): Promise<string> => {
|
||||
await ensureWorkspaceRuntimeDirs(workspaceRootPath)
|
||||
const directoryPath = getWorkspaceAssetBucketDir(workspaceRootPath, assetBucketId)
|
||||
await fs.mkdir(directoryPath, { recursive: true })
|
||||
return directoryPath
|
||||
}
|
||||
|
||||
export const createWorkspaceAssetBucket = async (
|
||||
workspaceRootPath: string
|
||||
): Promise<{ assetBucketId: string; directoryPath: string }> => {
|
||||
const assetBucketId = randomUUID()
|
||||
return {
|
||||
assetBucketId,
|
||||
directoryPath: await ensureWorkspaceAssetBucket(workspaceRootPath, assetBucketId)
|
||||
}
|
||||
}
|
||||
@@ -2,11 +2,7 @@ import fs from 'fs'
|
||||
import path from 'path'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import type { ChatAttachment } from '../../../shared/types'
|
||||
|
||||
const WORKSPACE_NCURATOR_DIR = '.ncurator'
|
||||
const WORKSPACE_RUNTIME_DIR = 'runtime'
|
||||
const WORKSPACE_ASSETS_DIR = 'assets'
|
||||
const WORKSPACE_SCRIPTS_DIR = 'scripts'
|
||||
import { ensureWorkspaceAssetBucket } from '../assets/WorkspaceAssetService'
|
||||
|
||||
const SUPPORTED_IMAGE_EXTENSIONS = new Set([
|
||||
'png',
|
||||
@@ -18,7 +14,7 @@ const SUPPORTED_IMAGE_EXTENSIONS = new Set([
|
||||
'tif',
|
||||
'tiff'
|
||||
])
|
||||
const SUPPORTED_FILE_EXTENSIONS = new Set(['docx', 'md', 'json'])
|
||||
const SUPPORTED_FILE_EXTENSIONS = new Set(['docx', 'pdf', 'md', 'json'])
|
||||
|
||||
const MIME_BY_EXTENSION: Record<string, string> = {
|
||||
png: 'image/png',
|
||||
@@ -31,6 +27,7 @@ const MIME_BY_EXTENSION: Record<string, string> = {
|
||||
tiff: 'image/tiff',
|
||||
md: 'text/markdown',
|
||||
json: 'application/json',
|
||||
pdf: 'application/pdf',
|
||||
docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
|
||||
}
|
||||
|
||||
@@ -85,38 +82,6 @@ const isSupportedAttachment = (input: {
|
||||
return SUPPORTED_FILE_EXTENSIONS.has(ext)
|
||||
}
|
||||
|
||||
const getWorkspaceRuntimeDir = (workspaceRootPath: string): string =>
|
||||
path.join(
|
||||
path.resolve(String(workspaceRootPath || '').trim()),
|
||||
WORKSPACE_NCURATOR_DIR,
|
||||
WORKSPACE_RUNTIME_DIR
|
||||
)
|
||||
|
||||
const getWorkspaceAssetsDir = (workspaceRootPath: string, assetBucketId: string): string =>
|
||||
path.join(
|
||||
getWorkspaceRuntimeDir(workspaceRootPath),
|
||||
WORKSPACE_ASSETS_DIR,
|
||||
String(assetBucketId || '').trim()
|
||||
)
|
||||
|
||||
const ensureWorkspaceAttachmentDirs = async (
|
||||
workspaceRootPath: string,
|
||||
assetBucketId: string
|
||||
): Promise<string> => {
|
||||
const rootPath = path.resolve(String(workspaceRootPath || '').trim())
|
||||
if (!rootPath) {
|
||||
throw new Error('缺少工作区根目录,无法保存附件')
|
||||
}
|
||||
const assetDir = getWorkspaceAssetsDir(rootPath, assetBucketId)
|
||||
await Promise.all([
|
||||
fs.promises.mkdir(assetDir, { recursive: true }),
|
||||
fs.promises.mkdir(path.join(getWorkspaceRuntimeDir(rootPath), WORKSPACE_SCRIPTS_DIR), {
|
||||
recursive: true
|
||||
})
|
||||
])
|
||||
return assetDir
|
||||
}
|
||||
|
||||
const splitFileName = (name: string): { stem: string; extWithDot: string } => {
|
||||
const safeName = path.basename(String(name || 'attachment').trim() || 'attachment')
|
||||
const parsed = path.parse(safeName)
|
||||
@@ -192,10 +157,7 @@ export class AttachmentService {
|
||||
throw new Error('暂不支持该文件格式')
|
||||
}
|
||||
|
||||
const dirPath = await ensureWorkspaceAttachmentDirs(
|
||||
input.workspaceRootPath,
|
||||
input.assetBucketId
|
||||
)
|
||||
const dirPath = await ensureWorkspaceAssetBucket(input.workspaceRootPath, input.assetBucketId)
|
||||
const targetPath = await resolveUniqueTargetPath(dirPath, input.name)
|
||||
await fs.promises.writeFile(targetPath, Buffer.from(input.bytes))
|
||||
return this.createAttachmentFromPath({
|
||||
@@ -229,10 +191,7 @@ export class AttachmentService {
|
||||
throw new Error('暂不支持该文件格式')
|
||||
}
|
||||
|
||||
const dirPath = await ensureWorkspaceAttachmentDirs(
|
||||
input.workspaceRootPath,
|
||||
input.assetBucketId
|
||||
)
|
||||
const dirPath = await ensureWorkspaceAssetBucket(input.workspaceRootPath, input.assetBucketId)
|
||||
const targetPath = await resolveUniqueTargetPath(dirPath, displayName)
|
||||
await fs.promises.copyFile(sourcePath, targetPath)
|
||||
return this.createAttachmentFromPath({
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
type PdfTextItemLike = {
|
||||
str: string
|
||||
transform: number[]
|
||||
width: number
|
||||
height: number
|
||||
hasEOL?: boolean
|
||||
}
|
||||
|
||||
const CJK_CHAR_PATTERN = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u
|
||||
// JavaScript 没有等价的 Unicode 属性可一次表达这组 PDF 控制字符。
|
||||
// eslint-disable-next-line no-control-regex
|
||||
const CONTROL_CHAR_PATTERN = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/gu
|
||||
const PRIVATE_USE_PATTERN = /[\uE000-\uF8FF]/gu
|
||||
|
||||
const isCjkBoundary = (left: string, right: string): boolean =>
|
||||
CJK_CHAR_PATTERN.test(left.slice(-1)) && CJK_CHAR_PATTERN.test(right.slice(0, 1))
|
||||
|
||||
const shouldInsertSpace = (input: {
|
||||
previousText: string
|
||||
nextText: string
|
||||
previousEndX?: number
|
||||
nextX: number
|
||||
fontSize: number
|
||||
}): boolean => {
|
||||
if (!input.previousText || !input.nextText) return false
|
||||
if (/\s$/u.test(input.previousText) || /^\s/u.test(input.nextText)) return false
|
||||
if (isCjkBoundary(input.previousText, input.nextText)) return false
|
||||
if (/^[,.;:!?,。;:!?、)\]}]/u.test(input.nextText)) return false
|
||||
if (/[([{(【《“‘]$/u.test(input.previousText)) return false
|
||||
if (typeof input.previousEndX !== 'number') return true
|
||||
return input.nextX - input.previousEndX > Math.max(1, input.fontSize * 0.18)
|
||||
}
|
||||
|
||||
export const normalizePdfText = (value: string): string => {
|
||||
const normalized = value
|
||||
.normalize('NFKC')
|
||||
.replace(/\r/gu, '')
|
||||
.replace(CONTROL_CHAR_PATTERN, '')
|
||||
.replace(/\u00a0/gu, ' ')
|
||||
|
||||
return normalized
|
||||
.split('\n')
|
||||
.map((line) => line.trim().replace(/[\t ]+/gu, ' '))
|
||||
.filter(Boolean)
|
||||
.join('\n')
|
||||
.replace(
|
||||
/([\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]) +(?=[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}])/gu,
|
||||
'$1'
|
||||
)
|
||||
}
|
||||
|
||||
export const buildPdfPageText = (
|
||||
rawItems: readonly unknown[]
|
||||
): { text: string; suspicious: boolean; warnings: string[] } => {
|
||||
const items = rawItems.filter((item): item is PdfTextItemLike =>
|
||||
Boolean(
|
||||
item &&
|
||||
typeof item === 'object' &&
|
||||
'str' in item &&
|
||||
typeof (item as { str?: unknown }).str === 'string' &&
|
||||
'transform' in item &&
|
||||
Array.isArray((item as { transform?: unknown }).transform)
|
||||
)
|
||||
)
|
||||
const rawJoined = items.map((item) => item.str).join('')
|
||||
const replacementCount = (rawJoined.match(/\uFFFD/gu) || []).length
|
||||
const privateUseCount = (rawJoined.match(PRIVATE_USE_PATTERN) || []).length
|
||||
const controlCount = (rawJoined.match(CONTROL_CHAR_PATTERN) || []).length
|
||||
const suspiciousCount = replacementCount + privateUseCount + controlCount
|
||||
const suspicious = rawJoined.length > 0 && suspiciousCount / rawJoined.length > 0.01
|
||||
|
||||
const lines: string[] = []
|
||||
let line = ''
|
||||
let currentY: number | undefined
|
||||
let previousEndX: number | undefined
|
||||
let previousFontSize = 10
|
||||
|
||||
const flush = () => {
|
||||
const normalizedLine = normalizePdfText(line)
|
||||
if (normalizedLine) lines.push(normalizedLine)
|
||||
line = ''
|
||||
currentY = undefined
|
||||
previousEndX = undefined
|
||||
}
|
||||
|
||||
for (const item of items) {
|
||||
const text = item.str
|
||||
if (!text) {
|
||||
if (item.hasEOL) flush()
|
||||
continue
|
||||
}
|
||||
const x = Number(item.transform[4] || 0)
|
||||
const y = Number(item.transform[5] || 0)
|
||||
const fontSize = Math.max(1, Math.abs(Number(item.transform[3] || item.height || 10)))
|
||||
const isNewLine =
|
||||
typeof currentY === 'number' && Math.abs(y - currentY) > Math.max(2, previousFontSize * 0.55)
|
||||
if (isNewLine) flush()
|
||||
if (
|
||||
shouldInsertSpace({
|
||||
previousText: line,
|
||||
nextText: text,
|
||||
previousEndX,
|
||||
nextX: x,
|
||||
fontSize
|
||||
})
|
||||
) {
|
||||
line += ' '
|
||||
}
|
||||
line += text
|
||||
currentY = y
|
||||
previousFontSize = fontSize
|
||||
previousEndX = x + Math.max(0, Number(item.width || 0))
|
||||
if (item.hasEOL) flush()
|
||||
}
|
||||
flush()
|
||||
|
||||
const warnings: string[] = []
|
||||
if (replacementCount > 0) warnings.push(`检测到 ${String(replacementCount)} 个替换字符`)
|
||||
if (privateUseCount > 0) warnings.push(`检测到 ${String(privateUseCount)} 个私用区字符`)
|
||||
if (controlCount > 0) warnings.push(`已清理 ${String(controlCount)} 个控制字符`)
|
||||
return {
|
||||
text: normalizePdfText(lines.join('\n')),
|
||||
suspicious,
|
||||
warnings
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
export type PdfDocumentMetadata = {
|
||||
title?: string
|
||||
author?: string
|
||||
subject?: string
|
||||
keywords?: string
|
||||
creator?: string
|
||||
producer?: string
|
||||
creationDate?: string
|
||||
modificationDate?: string
|
||||
}
|
||||
|
||||
export type PdfOutlineItem = {
|
||||
title: string
|
||||
pageNumber?: number
|
||||
depth: number
|
||||
}
|
||||
|
||||
export type PdfOpenResult = {
|
||||
docId: string
|
||||
sourcePath: string
|
||||
pageCount: number
|
||||
metadata: PdfDocumentMetadata
|
||||
outline: PdfOutlineItem[]
|
||||
encrypted: boolean
|
||||
warnings: string[]
|
||||
}
|
||||
|
||||
export type PdfPageTextStatus = 'available' | 'empty' | 'suspicious'
|
||||
|
||||
export type PdfPageReadResult = {
|
||||
pageNumber: number
|
||||
text: string
|
||||
charCount: number
|
||||
textStatus: PdfPageTextStatus
|
||||
imageCount: number
|
||||
width: number
|
||||
height: number
|
||||
rotation: number
|
||||
warnings: string[]
|
||||
}
|
||||
|
||||
export type PdfSearchMode = 'literal' | 'regex'
|
||||
|
||||
export type PdfSearchMatch = {
|
||||
pageNumber: number
|
||||
matchedText: string
|
||||
snippet: string
|
||||
start: number
|
||||
end: number
|
||||
}
|
||||
|
||||
export type PdfRenderedImage = {
|
||||
pageNumber: number
|
||||
path: string
|
||||
mimeType: 'image/png'
|
||||
width: number
|
||||
height: number
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import fs from 'node:fs/promises'
|
||||
import { parentPort } from 'node:worker_threads'
|
||||
import { createCanvas } from '@napi-rs/canvas'
|
||||
import * as pdfjs from 'pdfjs-dist/legacy/build/pdf.mjs'
|
||||
import type { PDFDocumentProxy, PDFPageProxy } from 'pdfjs-dist/types/src/display/api'
|
||||
import { createWorkspaceAssetBucket } from '../../assets/WorkspaceAssetService'
|
||||
import { buildPdfPageText, normalizePdfText } from '../textNormalizer'
|
||||
import type {
|
||||
PdfDocumentMetadata,
|
||||
PdfOutlineItem,
|
||||
PdfPageReadResult,
|
||||
PdfSearchMatch
|
||||
} from '../types'
|
||||
import type { PdfWorkerRequest, PdfWorkerResponse } from './types'
|
||||
|
||||
type PdfSnapshot = {
|
||||
docId: string
|
||||
sourcePath: string
|
||||
document: PDFDocumentProxy
|
||||
pageCount: number
|
||||
pageByNumber: Map<number, PDFPageProxy>
|
||||
pageTextByNumber: Map<number, PdfPageReadResult>
|
||||
}
|
||||
|
||||
const runSessions = new Map<string, Map<string, PdfSnapshot>>()
|
||||
|
||||
const getRunSession = (runId: string, create = false): Map<string, PdfSnapshot> | undefined => {
|
||||
const existing = runSessions.get(runId)
|
||||
if (existing || !create) return existing
|
||||
const next = new Map<string, PdfSnapshot>()
|
||||
runSessions.set(runId, next)
|
||||
return next
|
||||
}
|
||||
|
||||
const getSnapshot = (runId: string, docId: string): PdfSnapshot => {
|
||||
const snapshot = getRunSession(runId)?.get(docId)
|
||||
if (!snapshot) throw new Error(`pdf session not found: ${docId}`)
|
||||
return snapshot
|
||||
}
|
||||
|
||||
const readInfoString = (info: Record<string, unknown>, key: string): string | undefined => {
|
||||
const value = info[key]
|
||||
return typeof value === 'string' && value.trim() ? value.trim() : undefined
|
||||
}
|
||||
|
||||
const buildMetadata = (infoValue: unknown): PdfDocumentMetadata => {
|
||||
const info =
|
||||
infoValue && typeof infoValue === 'object' ? (infoValue as Record<string, unknown>) : {}
|
||||
return {
|
||||
title: readInfoString(info, 'Title'),
|
||||
author: readInfoString(info, 'Author'),
|
||||
subject: readInfoString(info, 'Subject'),
|
||||
keywords: readInfoString(info, 'Keywords'),
|
||||
creator: readInfoString(info, 'Creator'),
|
||||
producer: readInfoString(info, 'Producer'),
|
||||
creationDate: readInfoString(info, 'CreationDate'),
|
||||
modificationDate: readInfoString(info, 'ModDate')
|
||||
}
|
||||
}
|
||||
|
||||
const resolveOutlinePageNumber = async (
|
||||
document: PDFDocumentProxy,
|
||||
destination: string | unknown[] | null | undefined
|
||||
): Promise<number | undefined> => {
|
||||
if (!destination) return undefined
|
||||
const resolvedDestination =
|
||||
typeof destination === 'string' ? await document.getDestination(destination) : destination
|
||||
if (!Array.isArray(resolvedDestination) || !resolvedDestination[0]) return undefined
|
||||
const first = resolvedDestination[0]
|
||||
if (typeof first === 'number') return first + 1
|
||||
if (typeof first !== 'object') return undefined
|
||||
try {
|
||||
return (
|
||||
(await document.getPageIndex(first as Parameters<PDFDocumentProxy['getPageIndex']>[0])) + 1
|
||||
)
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
const buildOutline = async (document: PDFDocumentProxy): Promise<PdfOutlineItem[]> => {
|
||||
const rootItems = (await document.getOutline()) || []
|
||||
const output: PdfOutlineItem[] = []
|
||||
const visit = async (items: typeof rootItems, depth: number): Promise<void> => {
|
||||
for (const item of items) {
|
||||
if (output.length >= 500) return
|
||||
output.push({
|
||||
title: normalizePdfText(String(item.title || '')) || '未命名书签',
|
||||
pageNumber: await resolveOutlinePageNumber(document, item.dest),
|
||||
depth
|
||||
})
|
||||
if (Array.isArray(item.items) && item.items.length > 0) {
|
||||
await visit(item.items, depth + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
await visit(rootItems, 1)
|
||||
return output
|
||||
}
|
||||
|
||||
const getPage = async (snapshot: PdfSnapshot, pageNumber: number): Promise<PDFPageProxy> => {
|
||||
const cached = snapshot.pageByNumber.get(pageNumber)
|
||||
if (cached) return cached
|
||||
const page = await snapshot.document.getPage(pageNumber)
|
||||
snapshot.pageByNumber.set(pageNumber, page)
|
||||
return page
|
||||
}
|
||||
|
||||
const countPageImages = async (page: PDFPageProxy): Promise<number> => {
|
||||
try {
|
||||
const operators = await page.getOperatorList()
|
||||
const imageOps = new Set<number>([
|
||||
pdfjs.OPS.paintImageXObject,
|
||||
pdfjs.OPS.paintInlineImageXObject,
|
||||
pdfjs.OPS.paintImageMaskXObject
|
||||
])
|
||||
return operators.fnArray.filter((operator) => imageOps.has(operator)).length
|
||||
} catch {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
const readPage = async (snapshot: PdfSnapshot, pageNumber: number): Promise<PdfPageReadResult> => {
|
||||
const cached = snapshot.pageTextByNumber.get(pageNumber)
|
||||
if (cached) return cached
|
||||
const page = await getPage(snapshot, pageNumber)
|
||||
const viewport = page.getViewport({ scale: 1 })
|
||||
const content = await page.getTextContent()
|
||||
const normalized = buildPdfPageText(content.items)
|
||||
const result: PdfPageReadResult = {
|
||||
pageNumber,
|
||||
text: normalized.text,
|
||||
charCount: normalized.text.length,
|
||||
textStatus: normalized.text ? (normalized.suspicious ? 'suspicious' : 'available') : 'empty',
|
||||
imageCount: await countPageImages(page),
|
||||
width: Math.round(viewport.width),
|
||||
height: Math.round(viewport.height),
|
||||
rotation: viewport.rotation,
|
||||
warnings: normalized.warnings
|
||||
}
|
||||
snapshot.pageTextByNumber.set(pageNumber, result)
|
||||
return result
|
||||
}
|
||||
|
||||
const validatePageNumbers = (snapshot: PdfSnapshot, pageNumbers: number[]): void => {
|
||||
for (const pageNumber of pageNumbers) {
|
||||
if (!Number.isInteger(pageNumber) || pageNumber < 1 || pageNumber > snapshot.pageCount) {
|
||||
throw new Error(
|
||||
`pageNumber 超出范围:${String(pageNumber)},有效范围是 1-${String(snapshot.pageCount)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const buildSnippet = (text: string, start: number, end: number): string => {
|
||||
const from = Math.max(0, start - 80)
|
||||
const to = Math.min(text.length, end + 80)
|
||||
return text.slice(from, to).replace(/\s+/gu, ' ').trim()
|
||||
}
|
||||
|
||||
const findLiteralMatches = (text: string, query: string, pageNumber: number): PdfSearchMatch[] => {
|
||||
const matches: PdfSearchMatch[] = []
|
||||
const haystack = text.toLocaleLowerCase()
|
||||
const needle = query.toLocaleLowerCase()
|
||||
let fromIndex = 0
|
||||
while (needle && fromIndex <= haystack.length) {
|
||||
const start = haystack.indexOf(needle, fromIndex)
|
||||
if (start < 0) break
|
||||
const end = start + needle.length
|
||||
matches.push({
|
||||
pageNumber,
|
||||
matchedText: text.slice(start, end),
|
||||
snippet: buildSnippet(text, start, end),
|
||||
start,
|
||||
end
|
||||
})
|
||||
fromIndex = Math.max(end, start + 1)
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
const findRegexMatches = (text: string, query: string, pageNumber: number): PdfSearchMatch[] => {
|
||||
const expression = new RegExp(query, 'giu')
|
||||
const matches: PdfSearchMatch[] = []
|
||||
for (const match of text.matchAll(expression)) {
|
||||
const start = match.index ?? 0
|
||||
const matchedText = match[0] || ''
|
||||
if (!matchedText) continue
|
||||
const end = start + matchedText.length
|
||||
matches.push({
|
||||
pageNumber,
|
||||
matchedText,
|
||||
snippet: buildSnippet(text, start, end),
|
||||
start,
|
||||
end
|
||||
})
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
const renderPage = async (input: {
|
||||
snapshot: PdfSnapshot
|
||||
pageNumber: number
|
||||
directoryPath: string
|
||||
}) => {
|
||||
const page = await getPage(input.snapshot, input.pageNumber)
|
||||
const baseViewport = page.getViewport({ scale: 1 })
|
||||
const scale = Math.min(2, 3000 / Math.max(baseViewport.width, baseViewport.height))
|
||||
const viewport = page.getViewport({ scale })
|
||||
const width = Math.max(1, Math.ceil(viewport.width))
|
||||
const height = Math.max(1, Math.ceil(viewport.height))
|
||||
const canvas = createCanvas(width, height)
|
||||
const canvasContext = canvas.getContext('2d')
|
||||
await page.render({
|
||||
canvas: canvas as unknown as HTMLCanvasElement,
|
||||
canvasContext: canvasContext as unknown as CanvasRenderingContext2D,
|
||||
viewport
|
||||
}).promise
|
||||
const targetPath = `${input.directoryPath}/page-${String(input.pageNumber)}.png`
|
||||
await fs.writeFile(targetPath, canvas.toBuffer('image/png'))
|
||||
return {
|
||||
pageNumber: input.pageNumber,
|
||||
path: targetPath,
|
||||
mimeType: 'image/png' as const,
|
||||
width,
|
||||
height
|
||||
}
|
||||
}
|
||||
|
||||
const toPdfFailure = (requestId: string, error: unknown): PdfWorkerResponse => {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
const name = error instanceof Error ? error.name : ''
|
||||
if (name === 'PasswordException' || /password/iu.test(message)) {
|
||||
return { requestId, ok: false, code: 'password_required', error: 'PDF 需要密码才能读取' }
|
||||
}
|
||||
if (name === 'InvalidPDFException' || /invalid pdf/iu.test(message)) {
|
||||
return { requestId, ok: false, code: 'invalid_pdf', error: 'PDF 文件无效或已损坏' }
|
||||
}
|
||||
if (/pageNumber 超出范围|pageNumbers|query|regex|regular expression/iu.test(message)) {
|
||||
return { requestId, ok: false, code: 'invalid_input', error: message }
|
||||
}
|
||||
return { requestId, ok: false, code: 'failed', error: message }
|
||||
}
|
||||
|
||||
export const handlePdfWorkerRequest = async (
|
||||
request: PdfWorkerRequest
|
||||
): Promise<PdfWorkerResponse> => {
|
||||
try {
|
||||
if (request.action === 'open') {
|
||||
const bytes = new Uint8Array(await fs.readFile(request.payload.sourcePath))
|
||||
const document = await pdfjs.getDocument({ data: bytes }).promise
|
||||
const docId = randomUUID()
|
||||
const snapshot: PdfSnapshot = {
|
||||
docId,
|
||||
sourcePath: request.payload.sourcePath,
|
||||
document,
|
||||
pageCount: document.numPages,
|
||||
pageByNumber: new Map(),
|
||||
pageTextByNumber: new Map()
|
||||
}
|
||||
getRunSession(request.runId, true)?.set(docId, snapshot)
|
||||
const metadata = await document.getMetadata().catch(() => ({ info: {} }))
|
||||
return {
|
||||
requestId: request.requestId,
|
||||
ok: true,
|
||||
payload: {
|
||||
action: 'open',
|
||||
docId,
|
||||
sourcePath: snapshot.sourcePath,
|
||||
pageCount: snapshot.pageCount,
|
||||
metadata: buildMetadata(metadata.info),
|
||||
outline: await buildOutline(document),
|
||||
encrypted: false,
|
||||
warnings: []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (request.action === 'read_pages') {
|
||||
const snapshot = getSnapshot(request.runId, request.payload.docId)
|
||||
validatePageNumbers(snapshot, request.payload.pageNumbers)
|
||||
return {
|
||||
requestId: request.requestId,
|
||||
ok: true,
|
||||
payload: {
|
||||
action: 'read_pages',
|
||||
pages: await Promise.all(
|
||||
request.payload.pageNumbers.map((page) => readPage(snapshot, page))
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (request.action === 'search') {
|
||||
const snapshot = getSnapshot(request.runId, request.payload.docId)
|
||||
validatePageNumbers(snapshot, [request.payload.fromPage, request.payload.toPage])
|
||||
if (request.payload.fromPage > request.payload.toPage) {
|
||||
throw new Error('fromPage 不能大于 toPage')
|
||||
}
|
||||
if (request.payload.mode === 'regex') void new RegExp(request.payload.query, 'giu')
|
||||
const allMatches: PdfSearchMatch[] = []
|
||||
let pagesWithNoTextCount = 0
|
||||
let pagesWithSuspiciousTextCount = 0
|
||||
for (
|
||||
let pageNumber = request.payload.fromPage;
|
||||
pageNumber <= request.payload.toPage;
|
||||
pageNumber += 1
|
||||
) {
|
||||
const page = await readPage(snapshot, pageNumber)
|
||||
if (page.textStatus === 'empty') pagesWithNoTextCount += 1
|
||||
if (page.textStatus === 'suspicious') pagesWithSuspiciousTextCount += 1
|
||||
allMatches.push(
|
||||
...(request.payload.mode === 'regex'
|
||||
? findRegexMatches(page.text, request.payload.query, pageNumber)
|
||||
: findLiteralMatches(page.text, request.payload.query, pageNumber))
|
||||
)
|
||||
}
|
||||
return {
|
||||
requestId: request.requestId,
|
||||
ok: true,
|
||||
payload: {
|
||||
action: 'search',
|
||||
totalMatchCount: allMatches.length,
|
||||
matches: allMatches.slice(0, request.payload.maxMatches),
|
||||
pagesWithNoTextCount,
|
||||
pagesWithSuspiciousTextCount,
|
||||
warnings: []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (request.action === 'render_pages') {
|
||||
const snapshot = getSnapshot(request.runId, request.payload.docId)
|
||||
validatePageNumbers(snapshot, request.payload.pageNumbers)
|
||||
const bucket = await createWorkspaceAssetBucket(request.payload.workspaceRootPath)
|
||||
const images: Array<Awaited<ReturnType<typeof renderPage>>> = []
|
||||
for (const pageNumber of request.payload.pageNumbers) {
|
||||
images.push(
|
||||
await renderPage({
|
||||
snapshot,
|
||||
pageNumber,
|
||||
directoryPath: bucket.directoryPath
|
||||
})
|
||||
)
|
||||
}
|
||||
return {
|
||||
requestId: request.requestId,
|
||||
ok: true,
|
||||
payload: {
|
||||
action: 'render_pages',
|
||||
assetBucketId: bucket.assetBucketId,
|
||||
images,
|
||||
warnings: []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const released = getRunSession(request.runId)?.size || 0
|
||||
const session = getRunSession(request.runId)
|
||||
if (session) {
|
||||
await Promise.all([...session.values()].map((snapshot) => snapshot.document.destroy()))
|
||||
}
|
||||
runSessions.delete(request.runId)
|
||||
return {
|
||||
requestId: request.requestId,
|
||||
ok: true,
|
||||
payload: { action: 'release_run', releasedDocCount: released }
|
||||
}
|
||||
} catch (error) {
|
||||
return toPdfFailure(request.requestId, error)
|
||||
}
|
||||
}
|
||||
|
||||
const workerParentPort = parentPort
|
||||
if (workerParentPort) {
|
||||
workerParentPort.on('message', (request: PdfWorkerRequest) => {
|
||||
void handlePdfWorkerRequest(request).then((response) => workerParentPort.postMessage(response))
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import fs from 'node:fs'
|
||||
import { Worker } from 'node:worker_threads'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { handlePdfWorkerRequest } from './pdfWorker'
|
||||
import type {
|
||||
PdfWorkerAction,
|
||||
PdfWorkerRequest,
|
||||
PdfWorkerResponse,
|
||||
PendingPdfWorkerRequest
|
||||
} from './types'
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 45_000
|
||||
|
||||
class PdfWorkerClient {
|
||||
private worker: Worker | null = null
|
||||
private readonly pending = new Map<string, PendingPdfWorkerRequest>()
|
||||
private inProcessMode = false
|
||||
private readonly workerUrl = new URL('./pdfWorker.js', import.meta.url)
|
||||
|
||||
private ensureWorker(): Worker {
|
||||
if (this.inProcessMode) throw new Error('pdf worker is running in-process')
|
||||
if (this.worker) return this.worker
|
||||
const workerPath = fileURLToPath(this.workerUrl)
|
||||
if (!fs.existsSync(workerPath)) {
|
||||
this.inProcessMode = true
|
||||
throw new Error(`pdf worker entry not found: ${workerPath}`)
|
||||
}
|
||||
const worker = new Worker(this.workerUrl, { name: 'pdf-worker' })
|
||||
worker.on('message', (response: PdfWorkerResponse) => {
|
||||
const pending = this.pending.get(response.requestId)
|
||||
if (!pending) return
|
||||
clearTimeout(pending.timeout)
|
||||
this.pending.delete(response.requestId)
|
||||
pending.resolve(response)
|
||||
})
|
||||
worker.on('error', (error) => {
|
||||
this.failPending(new Error(`pdf worker error: ${error.message}`))
|
||||
this.worker = null
|
||||
})
|
||||
worker.on('exit', (code) => {
|
||||
if (code !== 0) this.failPending(new Error(`pdf worker exited with code ${String(code)}`))
|
||||
this.worker = null
|
||||
})
|
||||
this.worker = worker
|
||||
return worker
|
||||
}
|
||||
|
||||
private failPending(error: Error): void {
|
||||
for (const pending of this.pending.values()) {
|
||||
clearTimeout(pending.timeout)
|
||||
pending.reject(error)
|
||||
}
|
||||
this.pending.clear()
|
||||
}
|
||||
|
||||
public async call<TAction extends PdfWorkerAction>(
|
||||
input: Omit<Extract<PdfWorkerRequest, { action: TAction }>, 'requestId'>,
|
||||
timeoutMs = DEFAULT_TIMEOUT_MS
|
||||
): Promise<PdfWorkerResponse> {
|
||||
const request = { ...input, requestId: randomUUID() } as Extract<
|
||||
PdfWorkerRequest,
|
||||
{ action: TAction }
|
||||
>
|
||||
if (this.inProcessMode) return await handlePdfWorkerRequest(request)
|
||||
|
||||
let worker: Worker
|
||||
try {
|
||||
worker = this.ensureWorker()
|
||||
} catch {
|
||||
this.inProcessMode = true
|
||||
return await handlePdfWorkerRequest(request)
|
||||
}
|
||||
|
||||
return await new Promise<PdfWorkerResponse>((resolve, reject) => {
|
||||
const timeout = setTimeout(
|
||||
() => {
|
||||
this.pending.delete(request.requestId)
|
||||
reject(new Error(`pdf worker timeout (${input.action})`))
|
||||
},
|
||||
Math.max(1000, timeoutMs)
|
||||
)
|
||||
this.pending.set(request.requestId, { resolve, reject, timeout })
|
||||
worker.postMessage(request)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
let singleton: PdfWorkerClient | null = null
|
||||
|
||||
export const getPdfWorkerClient = (): PdfWorkerClient => {
|
||||
if (!singleton) singleton = new PdfWorkerClient()
|
||||
return singleton
|
||||
}
|
||||
|
||||
export const releasePdfRunSession = async (runId: string): Promise<void> => {
|
||||
if (!singleton) return
|
||||
try {
|
||||
await singleton.call({ runId, action: 'release_run', payload: {} })
|
||||
} catch {
|
||||
// ignore release failures
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import type {
|
||||
PdfOpenResult,
|
||||
PdfPageReadResult,
|
||||
PdfRenderedImage,
|
||||
PdfSearchMatch,
|
||||
PdfSearchMode
|
||||
} from '../types'
|
||||
|
||||
export type PdfWorkerAction = 'open' | 'read_pages' | 'search' | 'render_pages' | 'release_run'
|
||||
|
||||
export type PdfWorkerRequest =
|
||||
| {
|
||||
requestId: string
|
||||
runId: string
|
||||
action: 'open'
|
||||
payload: { sourcePath: string }
|
||||
}
|
||||
| {
|
||||
requestId: string
|
||||
runId: string
|
||||
action: 'read_pages'
|
||||
payload: { docId: string; pageNumbers: number[] }
|
||||
}
|
||||
| {
|
||||
requestId: string
|
||||
runId: string
|
||||
action: 'search'
|
||||
payload: {
|
||||
docId: string
|
||||
query: string
|
||||
mode: PdfSearchMode
|
||||
fromPage: number
|
||||
toPage: number
|
||||
maxMatches: number
|
||||
}
|
||||
}
|
||||
| {
|
||||
requestId: string
|
||||
runId: string
|
||||
action: 'render_pages'
|
||||
payload: {
|
||||
docId: string
|
||||
workspaceRootPath: string
|
||||
pageNumbers: number[]
|
||||
}
|
||||
}
|
||||
| {
|
||||
requestId: string
|
||||
runId: string
|
||||
action: 'release_run'
|
||||
payload: Record<string, never>
|
||||
}
|
||||
|
||||
export type PdfWorkerSuccessPayload =
|
||||
| ({ action: 'open' } & PdfOpenResult)
|
||||
| { action: 'read_pages'; pages: PdfPageReadResult[] }
|
||||
| {
|
||||
action: 'search'
|
||||
totalMatchCount: number
|
||||
matches: PdfSearchMatch[]
|
||||
pagesWithNoTextCount: number
|
||||
pagesWithSuspiciousTextCount: number
|
||||
warnings: string[]
|
||||
}
|
||||
| {
|
||||
action: 'render_pages'
|
||||
assetBucketId: string
|
||||
images: PdfRenderedImage[]
|
||||
warnings: string[]
|
||||
}
|
||||
| { action: 'release_run'; releasedDocCount: number }
|
||||
|
||||
export type PdfWorkerResponse =
|
||||
| { requestId: string; ok: true; payload: PdfWorkerSuccessPayload }
|
||||
| {
|
||||
requestId: string
|
||||
ok: false
|
||||
code:
|
||||
| 'invalid_input'
|
||||
| 'invalid_pdf'
|
||||
| 'password_required'
|
||||
| 'encrypted_pdf_unsupported'
|
||||
| 'failed'
|
||||
error: string
|
||||
warnings?: string[]
|
||||
}
|
||||
|
||||
export type PendingPdfWorkerRequest = {
|
||||
resolve: (response: PdfWorkerResponse) => void
|
||||
reject: (reason: Error) => void
|
||||
timeout: NodeJS.Timeout
|
||||
}
|
||||
@@ -92,7 +92,7 @@ type WorkbenchComposerProps = {
|
||||
const isTextNode = (value: Node | null): value is Text => value instanceof Text
|
||||
const CARET_ANCHOR_ROLE = 'composer-caret-anchor'
|
||||
const LARGE_FILE_WARNING_BYTES = 10 * 1024 * 1024
|
||||
const SUPPORTED_ATTACHMENT_FILE_EXTENSIONS = new Set(['md', 'json', 'docx'])
|
||||
const SUPPORTED_ATTACHMENT_FILE_EXTENSIONS = new Set(['md', 'json', 'docx', 'pdf'])
|
||||
|
||||
const getAttachmentFileExt = (name: string): string => {
|
||||
const last =
|
||||
|
||||
@@ -100,6 +100,10 @@ export type AssistantToolName =
|
||||
| 'docx_search'
|
||||
| 'docx_read_blocks'
|
||||
| 'docx_edit'
|
||||
| 'pdf_inspect'
|
||||
| 'pdf_search'
|
||||
| 'pdf_read_pages'
|
||||
| 'pdf_render_pages'
|
||||
| 'web_search'
|
||||
| 'web_crawl'
|
||||
| 'browser_use'
|
||||
@@ -137,6 +141,7 @@ export type AssistantProcessSkillKey =
|
||||
| 'docx-edit'
|
||||
| 'file-read'
|
||||
| 'file-write'
|
||||
| 'pdf-read'
|
||||
| 'kb'
|
||||
| 'terminal'
|
||||
| 'web-search'
|
||||
|
||||
@@ -80,6 +80,10 @@ const isAssistantToolName = (toolName: string): toolName is AssistantToolName =>
|
||||
toolName === 'docx_search' ||
|
||||
toolName === 'docx_read_blocks' ||
|
||||
toolName === 'docx_edit' ||
|
||||
toolName === 'pdf_inspect' ||
|
||||
toolName === 'pdf_search' ||
|
||||
toolName === 'pdf_read_pages' ||
|
||||
toolName === 'pdf_render_pages' ||
|
||||
toolName === 'web_search' ||
|
||||
toolName === 'web_crawl' ||
|
||||
toolName === 'browser_use' ||
|
||||
@@ -208,6 +212,7 @@ const getSkillDisplayName = (skillId?: RuntimeSkillId): string => {
|
||||
if (skillId === 'kb-retrieval') return '资料查阅器'
|
||||
if (skillId === 'docx-read') return 'docx 阅读器'
|
||||
if (skillId === 'docx-write') return 'docx 写作器'
|
||||
if (skillId === 'pdf-read') return 'PDF 阅读器'
|
||||
if (skillId === 'image-recognition') return '图像识别'
|
||||
return '新能力'
|
||||
}
|
||||
@@ -223,6 +228,14 @@ const getToolUiKind = (toolName?: AssistantToolName): AssistantProcessSkillKey =
|
||||
if (toolName === 'docx_edit') return 'docx-edit'
|
||||
if (toolName === 'file_read') return 'file-read'
|
||||
if (toolName === 'file_write') return 'file-write'
|
||||
if (
|
||||
toolName === 'pdf_inspect' ||
|
||||
toolName === 'pdf_search' ||
|
||||
toolName === 'pdf_read_pages' ||
|
||||
toolName === 'pdf_render_pages'
|
||||
) {
|
||||
return 'pdf-read'
|
||||
}
|
||||
if (toolName === 'kb_search' || toolName === 'kb_list_documents') return 'kb'
|
||||
if (toolName === 'terminal_run' || toolName === 'terminal_read' || toolName === 'terminal_stop') {
|
||||
return 'terminal'
|
||||
@@ -242,7 +255,7 @@ const getDisplaySkillKey = (display: AssistantProcessDisplay): AssistantProcessS
|
||||
|
||||
const getToolIconKind = (skillKey: AssistantProcessSkillKey): AssistantProcessIconKind => {
|
||||
if (skillKey === 'docx-edit') return 'edit'
|
||||
if (skillKey === 'docx-read') return 'document'
|
||||
if (skillKey === 'docx-read' || skillKey === 'pdf-read') return 'document'
|
||||
if (skillKey === 'file-read') return 'document'
|
||||
if (skillKey === 'file-write') return 'edit'
|
||||
if (skillKey === 'kb') return 'knowledge'
|
||||
@@ -317,6 +330,13 @@ const getToolSummaryText = (
|
||||
}
|
||||
return sentence(text(`编辑 ${normalizedCount} 个文件失败`))
|
||||
}
|
||||
if (skillKey === 'pdf-read') {
|
||||
if (status === 'running') return sentence(text(`正在读取 ${normalizedCount} 个 PDF 文件`))
|
||||
if (status === 'completed') return sentence(text(`已读取 ${normalizedCount} 个 PDF 文件`))
|
||||
if (status === 'partial_failed')
|
||||
return sentence(text(`部分读取 ${normalizedCount} 个 PDF 文件失败`))
|
||||
return sentence(text(`读取 ${normalizedCount} 个 PDF 文件失败`))
|
||||
}
|
||||
if (skillKey === 'kb') {
|
||||
if (status === 'running') return sentence(text('正在检索知识库'))
|
||||
if (status === 'completed') return sentence(text('已检索知识库'))
|
||||
@@ -394,6 +414,15 @@ const getToolItemText = (
|
||||
if (toolName === 'file_write') {
|
||||
return sentence(text('编辑文件 '), text(fileLabel))
|
||||
}
|
||||
if (toolName === 'pdf_inspect') {
|
||||
return sentence(text('查看 PDF 信息 '), text(fileLabel))
|
||||
}
|
||||
if (toolName === 'pdf_read_pages') {
|
||||
return sentence(text('读取 PDF 页面 '), text(fileLabel))
|
||||
}
|
||||
if (toolName === 'pdf_render_pages') {
|
||||
return sentence(text('渲染 PDF 页面 '), text(fileLabel))
|
||||
}
|
||||
if (toolName === 'terminal_run') {
|
||||
return sentence(text('执行命令 '), text(commandLabel))
|
||||
}
|
||||
@@ -406,9 +435,9 @@ const getToolItemText = (
|
||||
if (toolName === 'terminal_stop') {
|
||||
return sentence(text('停止终端 '), text(readStringValue(input || {}, 'terminalId') || '任务'))
|
||||
}
|
||||
if (toolName === 'kb_search' || toolName === 'docx_search') {
|
||||
if (toolName === 'kb_search' || toolName === 'docx_search' || toolName === 'pdf_search') {
|
||||
return sentence(
|
||||
text(toolName === 'docx_search' ? '在文档中搜索 ' : '在知识库中搜索 '),
|
||||
text(toolName === 'kb_search' ? '在知识库中搜索 ' : '在文档中搜索 '),
|
||||
text(queryLabel)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -821,6 +821,7 @@ export type RuntimeSkillId =
|
||||
| 'text-file'
|
||||
| 'docx-read'
|
||||
| 'docx-write'
|
||||
| 'pdf-read'
|
||||
| 'image-recognition'
|
||||
| 'kb-retrieval'
|
||||
| 'web-browsing'
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import fs from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { describe, it } from 'vitest'
|
||||
import {
|
||||
hasLiveLlmConfigInEnv,
|
||||
loadLiveLlmConfigFromEnv,
|
||||
loadLiveVisionLlmConfigFromEnv
|
||||
} from '../../../../harness/backend/llmConfig'
|
||||
import { createHumanTestRunner } from '../../../../harness/backend/humanHarness'
|
||||
import { runBackendCase } from '../../../../harness/backend/run-backend-case'
|
||||
import {
|
||||
buildSelectedPdfFixtureReference,
|
||||
HUMAN_PDF_VISUAL_READING_USER_INPUT,
|
||||
PDF_READING_FIXTURE_PATH
|
||||
} from '../../../../harness/backend/aiLiveTestData'
|
||||
import { filterAiRunEventsByType, runWithStructuredOutputRetry } from '../testUtils'
|
||||
import type { JsonObject } from '../../../../../src/shared/types'
|
||||
|
||||
const fileHash = async (filePath: string): Promise<string> =>
|
||||
createHash('sha256')
|
||||
.update(await fs.readFile(filePath))
|
||||
.digest('hex')
|
||||
|
||||
const readRenderedImagePaths = (data: unknown): string[] => {
|
||||
if (!data || typeof data !== 'object' || Array.isArray(data)) return []
|
||||
const images = (data as JsonObject).images
|
||||
if (!Array.isArray(images)) return []
|
||||
return images
|
||||
.map((image) =>
|
||||
image && typeof image === 'object' && !Array.isArray(image) && typeof image.path === 'string'
|
||||
? image.path
|
||||
: ''
|
||||
)
|
||||
.filter(Boolean)
|
||||
}
|
||||
|
||||
describe('ai human live backend', () => {
|
||||
it('human-pdf-visual-reading-live', async () => {
|
||||
await runWithStructuredOutputRetry(async () => {
|
||||
await runBackendCase({
|
||||
caseId: 'human-pdf-visual-reading-live',
|
||||
harnessMode: 'live',
|
||||
domain: 'ai',
|
||||
suite: 'human',
|
||||
groups: ['live'],
|
||||
description:
|
||||
'验证 live 模型能自主组合 PDF 文本读取、页面渲染和通用图像识别,回答图文结合问题。',
|
||||
fixtures: [PDF_READING_FIXTURE_PATH],
|
||||
runMode: 'single',
|
||||
runtimeModelName: hasLiveLlmConfigInEnv() ? loadLiveLlmConfigFromEnv().model_name : null,
|
||||
execute: async (runtime) => {
|
||||
if (!hasLiveLlmConfigInEnv()) {
|
||||
return { skipped: true, output: null, events: [], beforeHash: '', afterHash: '' }
|
||||
}
|
||||
|
||||
const primaryConfig = loadLiveLlmConfigFromEnv()
|
||||
const visionConfig = loadLiveVisionLlmConfigFromEnv()
|
||||
await runtime.seedLlmConfig(primaryConfig)
|
||||
await runtime.seedLlmConfig(visionConfig)
|
||||
const reference = buildSelectedPdfFixtureReference(runtime)
|
||||
const beforeHash = await fileHash(reference.absolutePath || '')
|
||||
const runner = createHumanTestRunner(runtime, {
|
||||
selectedReferences: [reference],
|
||||
visionModel: {
|
||||
configId: visionConfig.id,
|
||||
modelName: visionConfig.model_name
|
||||
},
|
||||
allowedToolNames: [
|
||||
'pdf_inspect',
|
||||
'pdf_search',
|
||||
'pdf_read_pages',
|
||||
'pdf_render_pages',
|
||||
'llm_recognize'
|
||||
]
|
||||
})
|
||||
const output = await runner.run({
|
||||
taskId: runtime.taskId,
|
||||
userInput: HUMAN_PDF_VISUAL_READING_USER_INPUT
|
||||
})
|
||||
return {
|
||||
skipped: false,
|
||||
output,
|
||||
events: runtime.eventCollector.snapshot(),
|
||||
beforeHash,
|
||||
afterHash: await fileHash(reference.absolutePath || '')
|
||||
}
|
||||
},
|
||||
assert: async (result, runtime) => {
|
||||
if (result.skipped) {
|
||||
runtime.assertThat(true, {
|
||||
name: 'live-config-not-provided',
|
||||
message: '未提供 live 模型配置,已跳过 human PDF 图文阅读 live case。'
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
const starts = filterAiRunEventsByType(result.events, 'tool_call_started')
|
||||
const results = filterAiRunEventsByType(result.events, 'tool_call_result')
|
||||
const toolNames = starts.map((event) => event.toolName)
|
||||
const renderPaths = results
|
||||
.filter((event) => event.toolName === 'pdf_render_pages' && event.result?.ok === true)
|
||||
.flatMap((event) => readRenderedImagePaths(event.result?.data))
|
||||
const recognizePaths = starts
|
||||
.filter((event) => event.toolName === 'llm_recognize')
|
||||
.map((event) => event.input.path)
|
||||
.filter((value): value is string => typeof value === 'string')
|
||||
const successfulRecognizeCallIds = new Set(
|
||||
results
|
||||
.filter((event) => event.toolName === 'llm_recognize' && event.result?.ok === true)
|
||||
.map((event) => event.toolCallId)
|
||||
)
|
||||
const successfulRecognizePaths = starts
|
||||
.filter(
|
||||
(event) =>
|
||||
event.toolName === 'llm_recognize' &&
|
||||
successfulRecognizeCallIds.has(event.toolCallId)
|
||||
)
|
||||
.map((event) => event.input.path)
|
||||
.filter((value): value is string => typeof value === 'string')
|
||||
const failedRecognizeCount = results.filter(
|
||||
(event) => event.toolName === 'llm_recognize' && event.result?.ok === false
|
||||
).length
|
||||
const delivery = String(result.output?.deliveryMarkdown || '')
|
||||
const page9Path = renderPaths.find(
|
||||
(imagePath) => path.basename(imagePath) === 'page-9.png'
|
||||
)
|
||||
const expectedAssetRoot = path.join(
|
||||
runtime.workspaceRoot,
|
||||
'.ncurator',
|
||||
'runtime',
|
||||
'assets'
|
||||
)
|
||||
|
||||
runtime.assertThat(
|
||||
result.output?.status === 'completed' &&
|
||||
toolNames.includes('pdf_read_pages') &&
|
||||
toolNames.includes('pdf_render_pages') &&
|
||||
toolNames.includes('llm_recognize') &&
|
||||
Boolean(page9Path) &&
|
||||
recognizePaths.some((imagePath) => renderPaths.includes(imagePath)) &&
|
||||
successfulRecognizePaths.some(
|
||||
(imagePath) => path.basename(imagePath) === 'page-9.png'
|
||||
) &&
|
||||
failedRecognizeCount === 0 &&
|
||||
Boolean(
|
||||
page9Path && path.relative(expectedAssetRoot, page9Path).split(path.sep).length >= 2
|
||||
) &&
|
||||
!toolNames.some(
|
||||
(name) =>
|
||||
name === 'terminal_run' || name.startsWith('kb_') || name.startsWith('docx_')
|
||||
) &&
|
||||
/(inputs|输入)/i.test(delivery) &&
|
||||
/(parameters|参数)/i.test(delivery) &&
|
||||
/(loss|损失)/i.test(delivery) &&
|
||||
/(update|更新)/i.test(delivery) &&
|
||||
/Proper fit/i.test(delivery) &&
|
||||
/Overfit/i.test(delivery) &&
|
||||
/(训练.*验证|验证.*训练|泛化|间距|差距)/i.test(delivery) &&
|
||||
!/(无法|不能|不支持).{0,12}(看到|识别|读取).{0,8}(图片|图像|曲线)/i.test(delivery) &&
|
||||
result.beforeHash === result.afterHash,
|
||||
{
|
||||
name: 'human-live-composes-pdf-render-and-image-recognition',
|
||||
message:
|
||||
'live human 应自主组合 PDF 阅读、页面渲染和图像识别,回答训练闭环及过度拟合图形差异。',
|
||||
data: {
|
||||
toolNames,
|
||||
renderPaths,
|
||||
recognizePaths,
|
||||
successfulRecognizePaths,
|
||||
failedRecognizeCount,
|
||||
deliveryMarkdown: result.output?.deliveryMarkdown
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
+207
@@ -0,0 +1,207 @@
|
||||
import { createServer } from 'node:http'
|
||||
import type { AddressInfo, Socket } from 'node:net'
|
||||
import path from 'node:path'
|
||||
import { describe, it } from 'vitest'
|
||||
import { createHumanTestRunner } from '../../../../harness/backend/humanHarness'
|
||||
import { runBackendCase } from '../../../../harness/backend/run-backend-case'
|
||||
import {
|
||||
buildSelectedPdfFixtureReference,
|
||||
PDF_READING_FIXTURE_PATH
|
||||
} from '../../../../harness/backend/aiLiveTestData'
|
||||
import {
|
||||
buildCompleteTaskDecision,
|
||||
buildLearnSkillsDecision,
|
||||
buildToolDecision,
|
||||
filterAiRunEventsByType
|
||||
} from '../testUtils'
|
||||
import type { JsonObject } from '../../../../../src/shared/types'
|
||||
|
||||
const startVisionServer = async (): Promise<{ baseUrl: string; close: () => Promise<void> }> => {
|
||||
const sockets = new Set<Socket>()
|
||||
const server = createServer((request, response) => {
|
||||
request.resume()
|
||||
response.writeHead(200, { 'content-type': 'application/json; charset=utf-8' })
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
id: 'chatcmpl-pdf-vision',
|
||||
object: 'chat.completion',
|
||||
created: Math.floor(Date.now() / 1000),
|
||||
model: 'gpt-5',
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: JSON.stringify({
|
||||
supports_image_recognition: true,
|
||||
result: '第9页右侧 Overfit 图中训练曲线继续改善,但验证曲线变差,二者差距明显。'
|
||||
})
|
||||
},
|
||||
finish_reason: 'stop'
|
||||
}
|
||||
]
|
||||
})
|
||||
)
|
||||
})
|
||||
server.on('connection', (socket) => {
|
||||
sockets.add(socket)
|
||||
socket.on('close', () => sockets.delete(socket))
|
||||
})
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once('error', reject)
|
||||
server.listen(0, '127.0.0.1', () => resolve())
|
||||
})
|
||||
const address = server.address() as AddressInfo
|
||||
return {
|
||||
baseUrl: `http://127.0.0.1:${String(address.port)}/v1`,
|
||||
close: async () => {
|
||||
sockets.forEach((socket) => socket.destroy())
|
||||
await new Promise<void>((resolve, reject) =>
|
||||
server.close((error) => (error ? reject(error) : resolve()))
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const renderedPathsFromData = (data: unknown): string[] => {
|
||||
if (!data || typeof data !== 'object' || Array.isArray(data)) return []
|
||||
const images = (data as JsonObject).images
|
||||
if (!Array.isArray(images)) return []
|
||||
return images.flatMap((image) => {
|
||||
if (!image || typeof image !== 'object' || Array.isArray(image)) return []
|
||||
return typeof image.path === 'string' ? [image.path] : []
|
||||
})
|
||||
}
|
||||
|
||||
describe('ai human mock-llm backend', () => {
|
||||
it('human-pdf-render-image-recognition-collaboration', async () => {
|
||||
await runBackendCase({
|
||||
caseId: 'human-pdf-render-image-recognition-collaboration',
|
||||
harnessMode: 'mock-llm',
|
||||
domain: 'ai',
|
||||
suite: 'human',
|
||||
groups: ['mock-llm'],
|
||||
description: '验证 PDF 页面图片路径会回流给 brain,并可继续交给通用图像识别工具。',
|
||||
fixtures: [PDF_READING_FIXTURE_PATH],
|
||||
runMode: 'single',
|
||||
execute: async (runtime) => {
|
||||
const visionServer = await startVisionServer()
|
||||
try {
|
||||
await runtime.seedLlmConfig({
|
||||
id: 'mock-llm-scripted',
|
||||
name: 'PDF 图像识别测试配置',
|
||||
providerKind: 'openai-compatible',
|
||||
base_url: visionServer.baseUrl,
|
||||
model_name: 'gpt-5',
|
||||
api_key: 'test-key'
|
||||
})
|
||||
const reference = buildSelectedPdfFixtureReference(runtime)
|
||||
const runner = createHumanTestRunner(runtime, {
|
||||
modelName: 'gpt-5',
|
||||
selectedReferences: [reference],
|
||||
allowedToolNames: ['pdf_read_pages', 'pdf_render_pages', 'llm_recognize'],
|
||||
structuredOutputScript: {
|
||||
'human.brain': [
|
||||
{
|
||||
output: buildLearnSkillsDecision({
|
||||
skillIds: ['pdf-read', 'image-recognition'],
|
||||
summary: '补齐 PDF 阅读和图像识别能力。'
|
||||
})
|
||||
},
|
||||
{
|
||||
output: {
|
||||
thought: { next: '读取文本并渲染需要观察的页面。' },
|
||||
nextAction: {
|
||||
action: 'hand',
|
||||
calls: [
|
||||
{
|
||||
toolName: 'pdf_read_pages',
|
||||
args: {
|
||||
sourcePath: reference.absolutePath,
|
||||
selection: { type: 'page_numbers', pageNumbers: [1, 9] }
|
||||
}
|
||||
},
|
||||
{
|
||||
toolName: 'pdf_render_pages',
|
||||
args: {
|
||||
sourcePath: reference.absolutePath,
|
||||
pageNumbers: [9]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
({ user }) => {
|
||||
const imagePath = user.match(
|
||||
/[^\s"<>]*\.ncurator\/runtime\/assets\/[^/\s"<>]+\/page-9\.png/u
|
||||
)?.[0]
|
||||
if (!imagePath)
|
||||
throw new Error('rendered page-9 image path missing from brain prompt')
|
||||
return buildToolDecision({
|
||||
toolName: 'llm_recognize',
|
||||
args: {
|
||||
path: imagePath,
|
||||
instruction: '说明 Proper fit 和 Overfit 两组曲线的直观差异。'
|
||||
},
|
||||
summary: '识别第9页对比图。'
|
||||
})
|
||||
},
|
||||
{
|
||||
output: buildCompleteTaskDecision({
|
||||
summary: 'PDF 文本和图片信息都已读取。'
|
||||
})
|
||||
}
|
||||
],
|
||||
'human.mouth': [
|
||||
{
|
||||
output: {
|
||||
summary: '已完成 PDF 图文阅读。',
|
||||
deliveryMarkdown:
|
||||
'训练闭环由 inputs、parameters、architecture、predictions、labels、loss 和 update 组成;Overfit 时训练与验证表现差距明显。'
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
const output = await runner.run({
|
||||
taskId: runtime.taskId,
|
||||
userInput: '结合第1页文字和第9页图形说明训练闭环与过度拟合。'
|
||||
})
|
||||
return { output, events: runtime.eventCollector.snapshot() }
|
||||
} finally {
|
||||
await visionServer.close()
|
||||
}
|
||||
},
|
||||
assert: async (result, runtime) => {
|
||||
const starts = filterAiRunEventsByType(result.events, 'tool_call_started')
|
||||
const toolResults = filterAiRunEventsByType(result.events, 'tool_call_result')
|
||||
const renderPaths = toolResults
|
||||
.filter((event) => event.toolName === 'pdf_render_pages' && event.result?.ok === true)
|
||||
.flatMap((event) => renderedPathsFromData(event.result?.data))
|
||||
const recognizePaths = starts
|
||||
.filter((event) => event.toolName === 'llm_recognize')
|
||||
.map((event) => event.input.path)
|
||||
.filter((value): value is string => typeof value === 'string')
|
||||
|
||||
runtime.assertThat(
|
||||
result.output.status === 'completed' &&
|
||||
starts.some((event) => event.toolName === 'pdf_read_pages') &&
|
||||
renderPaths.some((imagePath) => path.basename(imagePath) === 'page-9.png') &&
|
||||
recognizePaths.some((imagePath) => renderPaths.includes(imagePath)) &&
|
||||
toolResults.some(
|
||||
(event) =>
|
||||
event.toolName === 'llm_recognize' &&
|
||||
event.result?.ok === true &&
|
||||
JSON.stringify(event.result.data).includes('验证曲线')
|
||||
),
|
||||
{
|
||||
name: 'pdf-render-path-feeds-generic-image-recognition',
|
||||
message: 'PDF 渲染结果中的真实图片路径应进入下一轮 brain,并交给 llm_recognize。',
|
||||
data: { renderPaths, recognizePaths }
|
||||
}
|
||||
)
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,282 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import fs from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { describe, it } from 'vitest'
|
||||
import { createPdfToolHarness } from '../../../../harness/backend/aiToolHarness'
|
||||
import { runBackendCase } from '../../../../harness/backend/run-backend-case'
|
||||
|
||||
const PDF_FIXTURE = path.resolve('tests/fixtures/workspace/学习笔记/深度学习.pdf')
|
||||
|
||||
type PdfFailure = { ok: false; error: string; reason: string }
|
||||
type PdfInspectResult = {
|
||||
sourcePath: string
|
||||
pageCount: number
|
||||
metadata: { title?: string }
|
||||
warnings: string[]
|
||||
}
|
||||
type PdfReadResult = {
|
||||
sourcePath: string
|
||||
pageCount: number
|
||||
pages: Array<{
|
||||
pageNumber: number
|
||||
text: string
|
||||
textStatus: 'available' | 'empty' | 'suspicious'
|
||||
imageCount: number
|
||||
}>
|
||||
}
|
||||
type PdfSearchResult = {
|
||||
totalMatchCount: number
|
||||
returnedMatchCount: number
|
||||
matches: Array<{ pageNumber: number; matchedText: string; snippet: string }>
|
||||
}
|
||||
type PdfRenderResult = {
|
||||
sourcePath: string
|
||||
pageCount: number
|
||||
assetBucketId: string
|
||||
images: Array<{
|
||||
pageNumber: number
|
||||
path: string
|
||||
mimeType: 'image/png'
|
||||
width: number
|
||||
height: number
|
||||
}>
|
||||
}
|
||||
|
||||
const sha256 = async (filePath: string): Promise<string> =>
|
||||
createHash('sha256')
|
||||
.update(await fs.readFile(filePath))
|
||||
.digest('hex')
|
||||
|
||||
describe('ai tool pdf backend', () => {
|
||||
it('pdf-inspect-read-search-contract', async () => {
|
||||
await runBackendCase({
|
||||
caseId: 'pdf-inspect-read-search-contract',
|
||||
harnessMode: 'tool-real',
|
||||
domain: 'ai',
|
||||
suite: 'tool',
|
||||
groups: ['pdf'],
|
||||
description: '验证真实 PDF 的元数据、页级正文和搜索结果可稳定读取。',
|
||||
fixtures: [PDF_FIXTURE],
|
||||
runMode: 'single',
|
||||
execute: async (runtime) => {
|
||||
const sourcePath = runtime.getFixtureWorkspacePath(PDF_FIXTURE)
|
||||
const harness = createPdfToolHarness(runtime)
|
||||
try {
|
||||
const inspect = await harness.invoke<PdfInspectResult>('pdf_inspect', { sourcePath })
|
||||
const read = await harness.invoke<PdfReadResult>('pdf_read_pages', {
|
||||
sourcePath,
|
||||
selection: { type: 'page_numbers', pageNumbers: [1, 9] }
|
||||
})
|
||||
const search = await harness.invoke<PdfSearchResult>('pdf_search', {
|
||||
sourcePath,
|
||||
query: '过度拟合',
|
||||
mode: 'literal',
|
||||
maxMatches: 20
|
||||
})
|
||||
return { inspect, read, search }
|
||||
} finally {
|
||||
await harness.release()
|
||||
}
|
||||
},
|
||||
assert: async (result, runtime) => {
|
||||
runtime.assertThat(
|
||||
result.inspect.pageCount === 11 && result.inspect.metadata.title === '深度学习',
|
||||
{
|
||||
name: 'pdf-inspect-returns-real-metadata',
|
||||
message: 'pdf_inspect 应返回 fixture 的真实页数和标题。',
|
||||
data: result.inspect
|
||||
}
|
||||
)
|
||||
runtime.assertThat(
|
||||
result.read.pages.length === 2 &&
|
||||
result.read.pages[0]?.pageNumber === 1 &&
|
||||
/architecture/i.test(result.read.pages[0]?.text || '') &&
|
||||
!result.read.pages.some((page) => page.text.includes('\u0000')),
|
||||
{
|
||||
name: 'pdf-read-pages-returns-normalized-text',
|
||||
message: 'pdf_read_pages 应按页返回无 NUL 的真实文本层。',
|
||||
data: result.read
|
||||
}
|
||||
)
|
||||
runtime.assertThat(
|
||||
result.search.totalMatchCount > 0 &&
|
||||
result.search.matches.some((match) => match.pageNumber === 8 || match.pageNumber === 9),
|
||||
{
|
||||
name: 'pdf-search-locates-overfitting-pages',
|
||||
message: 'pdf_search 应在第 8 或第 9 页找到“过度拟合”。',
|
||||
data: result.search
|
||||
}
|
||||
)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('pdf-render-pages-runtime-assets-contract', async () => {
|
||||
await runBackendCase({
|
||||
caseId: 'pdf-render-pages-runtime-assets-contract',
|
||||
harnessMode: 'tool-real',
|
||||
domain: 'ai',
|
||||
suite: 'tool',
|
||||
groups: ['pdf'],
|
||||
description: '验证 PDF 页面图片写入 runtime assets,且不修改源 PDF、不产生普通 artifact。',
|
||||
fixtures: [PDF_FIXTURE],
|
||||
runMode: 'single',
|
||||
execute: async (runtime) => {
|
||||
const sourcePath = runtime.getFixtureWorkspacePath(PDF_FIXTURE)
|
||||
const beforeHash = await sha256(sourcePath)
|
||||
const harness = createPdfToolHarness(runtime)
|
||||
try {
|
||||
const render = await harness.invoke<PdfRenderResult>('pdf_render_pages', {
|
||||
sourcePath,
|
||||
pageNumbers: [1, 9]
|
||||
})
|
||||
const headers = await Promise.all(
|
||||
render.images.map(async (image) =>
|
||||
(await fs.readFile(image.path)).subarray(0, 8).toString('hex')
|
||||
)
|
||||
)
|
||||
return {
|
||||
render,
|
||||
headers,
|
||||
beforeHash,
|
||||
afterHash: await sha256(sourcePath),
|
||||
artifactCount: runtime.eventCollector
|
||||
.snapshot()
|
||||
.filter((event) => event.type === 'file_artifact').length
|
||||
}
|
||||
} finally {
|
||||
await harness.release()
|
||||
}
|
||||
},
|
||||
assert: async (result, runtime) => {
|
||||
const expectedParent = path.join(
|
||||
runtime.workspaceRoot,
|
||||
'.ncurator',
|
||||
'runtime',
|
||||
'assets',
|
||||
result.render.assetBucketId
|
||||
)
|
||||
runtime.assertThat(
|
||||
result.render.images.length === 2 &&
|
||||
result.render.images.every((image) => path.dirname(image.path) === expectedParent) &&
|
||||
result.render.images.map((image) => path.basename(image.path)).join(',') ===
|
||||
'page-1.png,page-9.png' &&
|
||||
result.headers.every((header) => header === '89504e470d0a1a0a'),
|
||||
{
|
||||
name: 'pdf-render-writes-valid-runtime-assets',
|
||||
message: '页面图片应写入 .ncurator/runtime/assets/<uuid>,并且是有效 PNG。',
|
||||
data: result.render
|
||||
}
|
||||
)
|
||||
runtime.assertThat(result.beforeHash === result.afterHash && result.artifactCount === 0, {
|
||||
name: 'pdf-render-does-not-mutate-source-or-emit-artifact',
|
||||
message: '渲染页面不能修改源 PDF,也不能产生普通 file artifact。',
|
||||
data: { artifactCount: result.artifactCount }
|
||||
})
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('pdf-failure-boundary-contract', async () => {
|
||||
await runBackendCase({
|
||||
caseId: 'pdf-failure-boundary-contract',
|
||||
harnessMode: 'tool-real',
|
||||
domain: 'ai',
|
||||
suite: 'tool',
|
||||
groups: ['pdf'],
|
||||
description: '验证越界页码、非法 regex、伪 PDF 和逃逸工作区的符号链接会明确失败。',
|
||||
fixtures: [PDF_FIXTURE],
|
||||
runMode: 'single',
|
||||
execute: async (runtime) => {
|
||||
const sourcePath = runtime.getFixtureWorkspacePath(PDF_FIXTURE)
|
||||
const corruptPath = path.join(runtime.workspaceRoot, 'corrupt.pdf')
|
||||
const outsidePath = path.join(path.dirname(runtime.workspaceRoot), 'outside-pdf.pdf')
|
||||
const symlinkPath = path.join(runtime.workspaceRoot, 'outside-link.pdf')
|
||||
await fs.writeFile(corruptPath, 'not a pdf')
|
||||
await fs.copyFile(sourcePath, outsidePath)
|
||||
await fs.symlink(outsidePath, symlinkPath)
|
||||
const harness = createPdfToolHarness(runtime)
|
||||
try {
|
||||
const outOfRange = await harness.invoke<PdfFailure>('pdf_read_pages', {
|
||||
sourcePath,
|
||||
selection: { type: 'page_numbers', pageNumbers: [99] }
|
||||
})
|
||||
const badRegex = await harness.invoke<PdfFailure>('pdf_search', {
|
||||
sourcePath,
|
||||
query: '[',
|
||||
mode: 'regex'
|
||||
})
|
||||
const oversizedRange = await harness.invoke<PdfFailure>('pdf_read_pages', {
|
||||
sourcePath,
|
||||
selection: { type: 'page_range', fromPage: 1, toPage: 1000000 }
|
||||
})
|
||||
const corrupt = await harness.invoke<PdfFailure>('pdf_inspect', {
|
||||
sourcePath: corruptPath
|
||||
})
|
||||
const outside = await harness.invoke<PdfFailure>('pdf_inspect', {
|
||||
sourcePath: symlinkPath
|
||||
})
|
||||
return { outOfRange, badRegex, oversizedRange, corrupt, outside }
|
||||
} finally {
|
||||
await harness.release()
|
||||
await fs.rm(outsidePath, { force: true })
|
||||
}
|
||||
},
|
||||
assert: async (result, runtime) => {
|
||||
runtime.assertThat(
|
||||
[
|
||||
result.outOfRange,
|
||||
result.badRegex,
|
||||
result.oversizedRange,
|
||||
result.corrupt,
|
||||
result.outside
|
||||
].every((failure) => failure.ok === false && failure.error.length > 0),
|
||||
{
|
||||
name: 'pdf-invalid-inputs-fail-explicitly',
|
||||
message: '所有 PDF 边界错误都应返回明确失败信息。',
|
||||
data: result
|
||||
}
|
||||
)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('pdf-result-size-contract', async () => {
|
||||
await runBackendCase({
|
||||
caseId: 'pdf-result-size-contract',
|
||||
harnessMode: 'tool-real',
|
||||
domain: 'ai',
|
||||
suite: 'tool',
|
||||
groups: ['pdf'],
|
||||
description: '验证 PDF 正文过大时返回统一 hand_tool_result_too_large 提示。',
|
||||
fixtures: [PDF_FIXTURE],
|
||||
runMode: 'single',
|
||||
execute: async (runtime) => {
|
||||
const sourcePath = runtime.getFixtureWorkspacePath(PDF_FIXTURE)
|
||||
const harness = createPdfToolHarness(runtime, {
|
||||
humanRuntimeOverrides: { modelContextWindowTokens: 100 }
|
||||
})
|
||||
try {
|
||||
return await harness.invoke<
|
||||
PdfFailure & { decisionHint?: { code?: string; detail?: unknown } }
|
||||
>('pdf_read_pages', {
|
||||
sourcePath,
|
||||
selection: { type: 'page_numbers', pageNumbers: [1] }
|
||||
})
|
||||
} finally {
|
||||
await harness.release()
|
||||
}
|
||||
},
|
||||
assert: async (result, runtime) => {
|
||||
runtime.assertThat(
|
||||
result.ok === false && result.decisionHint?.code === 'hand_tool_result_too_large',
|
||||
{
|
||||
name: 'pdf-large-result-returns-shared-decision-hint',
|
||||
message: 'PDF 工具结果过大时应复用 hand_tool_result_too_large。',
|
||||
data: result
|
||||
}
|
||||
)
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -284,6 +284,79 @@ describe('ai unit human memory renderers', () => {
|
||||
expect(rendered).toContain('"totalLines":1')
|
||||
})
|
||||
|
||||
it('keeps PDF page text and rendered image paths in brain history', () => {
|
||||
const initialState = createInitialHumanState({
|
||||
taskId: 'task-history-pdf',
|
||||
userInput: '阅读 PDF 图文'
|
||||
})
|
||||
const imagePath = '/workspace/.ncurator/runtime/assets/uuid/page-9.png'
|
||||
const calls = [
|
||||
{
|
||||
toolName: 'pdf_read_pages',
|
||||
args: {
|
||||
sourcePath: '/workspace/深度学习.pdf',
|
||||
selection: { type: 'page_numbers', pageNumbers: [9] }
|
||||
}
|
||||
},
|
||||
{
|
||||
toolName: 'pdf_render_pages',
|
||||
args: { sourcePath: '/workspace/深度学习.pdf', pageNumbers: [9] }
|
||||
}
|
||||
]
|
||||
const taskRun = appendRoundRecord({
|
||||
state: initialState,
|
||||
thought: {
|
||||
thought: { next: '读取 PDF 页面并渲染图片' },
|
||||
nextAction: { action: 'hand', calls }
|
||||
},
|
||||
execution: buildHandExecution({
|
||||
calls,
|
||||
toolResults: [
|
||||
{
|
||||
toolName: 'pdf_read_pages',
|
||||
ok: true,
|
||||
data: {
|
||||
sourcePath: '/workspace/深度学习.pdf',
|
||||
pageCount: 11,
|
||||
pages: [
|
||||
{
|
||||
pageNumber: 9,
|
||||
text: '过度拟合会让训练与验证表现差距变大',
|
||||
charCount: 19,
|
||||
textStatus: 'available',
|
||||
imageCount: 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
toolName: 'pdf_render_pages',
|
||||
ok: true,
|
||||
data: {
|
||||
sourcePath: '/workspace/深度学习.pdf',
|
||||
pageCount: 11,
|
||||
assetBucketId: 'uuid',
|
||||
images: [
|
||||
{
|
||||
pageNumber: 9,
|
||||
path: imagePath,
|
||||
mimeType: 'image/png',
|
||||
width: 1224,
|
||||
height: 1584
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
})
|
||||
})
|
||||
|
||||
const rendered = renderBrainRoundSection(taskRun.rounds)
|
||||
expect(rendered).toContain('过度拟合会让训练与验证表现差距变大')
|
||||
expect(rendered).toContain(imagePath)
|
||||
expect(rendered).toContain('"pageNumber":9')
|
||||
})
|
||||
|
||||
it('renders kb_list_documents result ids in round history', () => {
|
||||
const initialState = createInitialHumanState({
|
||||
taskId: 'task-history-kb-list-documents',
|
||||
|
||||
@@ -129,4 +129,51 @@ describe('ai unit human tool result renderers', () => {
|
||||
expect(rendered).not.toContain('currentRevisionRef')
|
||||
expect(rendered).not.toContain('createdBlocks')
|
||||
})
|
||||
|
||||
it('renders PDF page text, search snippets and rendered image paths', () => {
|
||||
const readRendered = renderToolResultDataText({
|
||||
toolName: 'pdf_read_pages',
|
||||
data: {
|
||||
sourcePath: '/workspace/深度学习.pdf',
|
||||
pageCount: 11,
|
||||
pages: [
|
||||
{
|
||||
pageNumber: 1,
|
||||
textStatus: 'available',
|
||||
charCount: 42,
|
||||
imageCount: 1,
|
||||
text: 'inputs parameters architecture predictions labels loss update'
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
const searchRendered = renderToolResultDataText({
|
||||
toolName: 'pdf_search',
|
||||
data: {
|
||||
sourcePath: '/workspace/深度学习.pdf',
|
||||
pageCount: 11,
|
||||
query: '过度拟合',
|
||||
mode: 'literal',
|
||||
totalMatchCount: 1,
|
||||
returnedMatchCount: 1,
|
||||
matches: [{ pageNumber: 9, matchedText: '过度拟合', snippet: '右图属于过度拟合' }]
|
||||
}
|
||||
})
|
||||
const imagePath = '/workspace/.ncurator/runtime/assets/uuid/page-9.png'
|
||||
const renderRendered = renderToolResultDataText({
|
||||
toolName: 'pdf_render_pages',
|
||||
data: {
|
||||
sourcePath: '/workspace/深度学习.pdf',
|
||||
pageCount: 11,
|
||||
assetBucketId: 'uuid',
|
||||
images: [{ pageNumber: 9, path: imagePath, mimeType: 'image/png', width: 100, height: 200 }]
|
||||
}
|
||||
})
|
||||
|
||||
expect(readRendered).toContain('page 1')
|
||||
expect(readRendered).toContain('architecture')
|
||||
expect(searchRendered).toContain('pageNumber=9')
|
||||
expect(searchRendered).toContain('右图属于过度拟合')
|
||||
expect(renderRendered).toContain(imagePath)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { SKILL_DEFINITIONS } from '../../../../src/main/services/ai/human/skills'
|
||||
import { PDF_TOOL_SPECS } from '../../../../src/main/services/ai/human/tools/pdf/toolSpecs'
|
||||
import {
|
||||
pdfReadPagesSchema,
|
||||
pdfRenderPagesSchema,
|
||||
pdfSearchSchema
|
||||
} from '../../../../src/main/services/ai/human/tools/pdf/schemas'
|
||||
|
||||
describe('ai unit PDF tool contract', () => {
|
||||
it('exposes only inspect, search, page reading and page rendering', () => {
|
||||
expect(PDF_TOOL_SPECS.map((spec) => spec.name)).toEqual([
|
||||
'pdf_inspect',
|
||||
'pdf_search',
|
||||
'pdf_read_pages',
|
||||
'pdf_render_pages'
|
||||
])
|
||||
expect(SKILL_DEFINITIONS['pdf-read'].toolNames).toEqual(PDF_TOOL_SPECS.map((spec) => spec.name))
|
||||
expect(PDF_TOOL_SPECS.map((spec) => spec.name)).not.toContain('pdf_recognize_pages')
|
||||
})
|
||||
|
||||
it('keeps selection, search and render limits in schemas', () => {
|
||||
expect(
|
||||
pdfReadPagesSchema.safeParse({
|
||||
sourcePath: '学习笔记/深度学习.pdf',
|
||||
selection: { type: 'page_range', fromPage: 1, toPage: 9 }
|
||||
}).success
|
||||
).toBe(true)
|
||||
expect(
|
||||
pdfSearchSchema.safeParse({
|
||||
sourcePath: '学习笔记/深度学习.pdf',
|
||||
query: '过度拟合',
|
||||
maxMatches: 51
|
||||
}).success
|
||||
).toBe(false)
|
||||
expect(
|
||||
pdfRenderPagesSchema.safeParse({
|
||||
sourcePath: '学习笔记/深度学习.pdf',
|
||||
pageNumbers: Array.from({ length: 21 }, (_, index) => index + 1)
|
||||
}).success
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -17,6 +17,7 @@ describe('ai unit runtime skill contract', () => {
|
||||
expect(listExternalSkillDefinitions().map((definition) => definition.skillId)).toEqual([
|
||||
'docx-read',
|
||||
'docx-write',
|
||||
'pdf-read',
|
||||
'image-recognition'
|
||||
])
|
||||
})
|
||||
|
||||
@@ -68,4 +68,19 @@ describe('attachment service runtime directories', () => {
|
||||
expect(second.path).toBe(path.join(bucketPath, 'reference-1.json'))
|
||||
await expect(fs.readFile(second.path, 'utf8')).resolves.toBe('{"ready":true}')
|
||||
})
|
||||
|
||||
it('accepts PDF attachments and assigns application/pdf', async () => {
|
||||
const attachment = await service.saveAttachment({
|
||||
workspaceRootPath,
|
||||
assetBucketId: 'bucket-pdf',
|
||||
name: 'lecture.pdf',
|
||||
bytes: Buffer.from('%PDF-1.7\n')
|
||||
})
|
||||
|
||||
expect(attachment.kind).toBe('file')
|
||||
expect(attachment.mimeType).toBe('application/pdf')
|
||||
expect(attachment.path).toBe(
|
||||
path.join(workspaceRootPath, '.ncurator', 'runtime', 'assets', 'bucket-pdf', 'lecture.pdf')
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -15,6 +15,11 @@ export const DOCX_POSITIONED_FIXTURE_PATHS = [
|
||||
`${WORKSPACE_LEARNING_FIXTURE_DIR}/图形学.docx`
|
||||
] as const
|
||||
|
||||
export const PDF_READING_FIXTURE_PATH = `${WORKSPACE_LEARNING_FIXTURE_DIR}/深度学习.pdf` as const
|
||||
|
||||
export const HUMAN_PDF_VISUAL_READING_USER_INPUT =
|
||||
'请只根据我选中的《深度学习.pdf》回答两个问题:1. 用通俗的话说明第1页流程图里 inputs、parameters、architecture、predictions、labels、loss 和 update 是怎么形成训练闭环的;2. 结合第9页 Proper fit 和 Overfit 对比图,说明两条曲线有什么直观区别,以及为什么右边属于过度拟合。'
|
||||
|
||||
export const DOCX_MULTI_EDIT_FIXTURE_PATHS = [
|
||||
`${SHARED_WORKSPACE_FIXTURE_PATH}/人工智能探究课堂PRD.docx`,
|
||||
`${SHARED_WORKSPACE_FIXTURE_PATH}/交互式探究学习案例:产业转移.docx`
|
||||
@@ -102,6 +107,25 @@ export const buildSelectedDocxFixtureReferences = (
|
||||
): SelectedReference[] =>
|
||||
buildSelectedDocxReferences(resolveSelectedDocxFixtures(runtime, fixturePaths), indexedFiles)
|
||||
|
||||
export const buildSelectedPdfFixtureReference = (
|
||||
runtime: BackendTestRuntime,
|
||||
fixturePath: string = PDF_READING_FIXTURE_PATH
|
||||
): SelectedReference => {
|
||||
const workspacePath = runtime.getFixtureWorkspacePath(fixturePath)
|
||||
const relativePath = path.relative(runtime.workspaceRoot, workspacePath).split(path.sep).join('/')
|
||||
return {
|
||||
kind: 'file',
|
||||
workspaceId: 'test-workspace',
|
||||
nodeId: 'fixture-pdf-1',
|
||||
name: path.basename(workspacePath),
|
||||
relativePath,
|
||||
absolutePath: workspacePath,
|
||||
fileExt: '.pdf',
|
||||
kbDocumentIds: [],
|
||||
kbAvailability: 'not_indexed'
|
||||
}
|
||||
}
|
||||
|
||||
export const captureDocxFixtureBytes = async (
|
||||
runtime: BackendTestRuntime,
|
||||
fixturePaths: readonly string[]
|
||||
|
||||
@@ -14,12 +14,15 @@ import {
|
||||
} from 'docx'
|
||||
import { buildDocxCapabilityTools } from '../../../src/main/services/ai/human/tools/docx'
|
||||
import { releaseDocxRunSession } from '../../../src/main/services/ai/human/tools/docx/worker/docxWorkerClient'
|
||||
import { AiRunResourceManager } from '../../../src/main/services/ai/run/AiRunResourceManager'
|
||||
import { DEFAULT_MODEL_CONTEXT_WINDOW_TOKENS } from '../../../src/main/services/ai/human/constants'
|
||||
import { buildKbCapabilityTools } from '../../../src/main/services/ai/human/tools/kb/retrievalTools'
|
||||
import { buildTerminalCapabilityTools } from '../../../src/main/services/ai/human/tools/terminal/terminalTools'
|
||||
import { TerminalSessionManager } from '../../../src/main/services/ai/human/tools/terminal/TerminalSessionManager'
|
||||
import { buildTextFileCapabilityTools } from '../../../src/main/services/ai/human/tools/textFile'
|
||||
import { imageRecognitionToolFamily } from '../../../src/main/services/ai/human/tools/imageRecognition'
|
||||
import { buildPdfCapabilityTools } from '../../../src/main/services/ai/human/tools/pdf'
|
||||
import { releasePdfRunSession } from '../../../src/main/services/pdf/worker/pdfWorkerClient'
|
||||
import { utilityToolFamily } from '../../../src/main/services/ai/human/tools/utility'
|
||||
import { webToolFamily } from '../../../src/main/services/ai/human/tools/web'
|
||||
import { createHumanRuntimeCache } from '../../../src/main/services/ai/human/node/utils/runtimeCache'
|
||||
@@ -79,6 +82,9 @@ export const createDocxToolHarness = (
|
||||
const ctx = runtime.createAiRunContext(contextInput)
|
||||
const cache = createHumanRuntimeCache()
|
||||
const activities: CapturedToolActivity[] = []
|
||||
const resourceManager = new AiRunResourceManager([
|
||||
{ id: 'docx', releaseRun: releaseDocxRunSession }
|
||||
])
|
||||
const built = buildDocxCapabilityTools({
|
||||
ctx,
|
||||
cache,
|
||||
@@ -99,7 +105,7 @@ export const createDocxToolHarness = (
|
||||
state: built.state,
|
||||
activities,
|
||||
async release(): Promise<void> {
|
||||
await releaseDocxRunSession(runtime.runId)
|
||||
await resourceManager.releaseRun(runtime.runId)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -169,6 +175,39 @@ export const createTextFileToolHarness = (
|
||||
}
|
||||
}
|
||||
|
||||
export const createPdfToolHarness = (
|
||||
runtime: BackendTestRuntime,
|
||||
contextInput: CreateAiRunContextInput = {}
|
||||
) => {
|
||||
const ctx = runtime.createAiRunContext(contextInput)
|
||||
const cache = createHumanRuntimeCache()
|
||||
const activities: CapturedToolActivity[] = []
|
||||
const resourceManager = new AiRunResourceManager([
|
||||
{ id: 'pdf', releaseRun: releasePdfRunSession }
|
||||
])
|
||||
const built = buildPdfCapabilityTools({
|
||||
ctx,
|
||||
cache,
|
||||
runId: runtime.runId,
|
||||
sessionId: contextInput.sessionId || 'test-session',
|
||||
taskId: runtime.taskId,
|
||||
workspaceRootPath: runtime.workspaceRoot,
|
||||
safePreviewChars: 1400,
|
||||
modelContextWindowTokens:
|
||||
ctx.humanRuntimeOverrides?.modelContextWindowTokens || DEFAULT_MODEL_CONTEXT_WINDOW_TOKENS,
|
||||
emitActivity: (activity) => activities.push(activity)
|
||||
})
|
||||
|
||||
return {
|
||||
...createToolInvoker(built.tools as readonly ToolLike[]),
|
||||
ctx,
|
||||
activities,
|
||||
async release(): Promise<void> {
|
||||
await resourceManager.releaseRun(runtime.runId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const createRetrievalToolHarness = (
|
||||
runtime: BackendTestRuntime,
|
||||
contextInput: CreateAiRunContextInput = {}
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import { createHumanGraphRunner } from '../../../src/main/services/ai/human/graph/humanGraph'
|
||||
import { runAiRuntimePreflight } from '../../../src/main/services/ai/runtimePreflight'
|
||||
import { AiRunResourceManager } from '../../../src/main/services/ai/run/AiRunResourceManager'
|
||||
import { releaseDocxRunSession } from '../../../src/main/services/ai/human/tools/docx/worker/docxWorkerClient'
|
||||
import { releasePdfRunSession } from '../../../src/main/services/pdf/worker/pdfWorkerClient'
|
||||
import type {
|
||||
HumanRunDispatchOptions,
|
||||
HumanTaskInput,
|
||||
@@ -15,6 +18,10 @@ export const createHumanTestRunner = (
|
||||
...contextInput
|
||||
})
|
||||
const run = createHumanGraphRunner({ ctx })
|
||||
const resourceManager = new AiRunResourceManager([
|
||||
{ id: 'docx', releaseRun: releaseDocxRunSession },
|
||||
{ id: 'pdf', releaseRun: releasePdfRunSession }
|
||||
])
|
||||
|
||||
return {
|
||||
ctx,
|
||||
@@ -23,7 +30,11 @@ export const createHumanTestRunner = (
|
||||
await runAiRuntimePreflight({
|
||||
ensureEmbeddingReady: runtime.ensureEmbeddingReady
|
||||
})
|
||||
return await run(taskInput, dispatchOptions) as HumanTaskOutput
|
||||
try {
|
||||
return (await run(taskInput, dispatchOptions)) as HumanTaskOutput
|
||||
} finally {
|
||||
await resourceManager.releaseRun(runtime.runId)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,19 @@ const resolveLiveBaseUrl = (modelName: string): string => {
|
||||
return normalizeOpenAIBaseUrl(rawBaseUrl)
|
||||
}
|
||||
|
||||
const resolveLiveVisionModelName = (primaryConfig: LiveLlmConfig): string => {
|
||||
const explicit = readEnv('AI_LIVE_VISION_MODEL_NAME')
|
||||
if (explicit) return explicit
|
||||
try {
|
||||
if (new URL(primaryConfig.base_url).hostname.endsWith('dashscope.aliyuncs.com')) {
|
||||
return 'qwen3-vl-flash'
|
||||
}
|
||||
} catch {
|
||||
// 交给主配置校验报告无效 URL。
|
||||
}
|
||||
return primaryConfig.model_name
|
||||
}
|
||||
|
||||
export const hasLiveLlmConfigInEnv = (): boolean => {
|
||||
const modelName = resolveLiveModelName()
|
||||
const baseUrl = resolveLiveBaseUrl(modelName)
|
||||
@@ -67,3 +80,23 @@ export const loadLiveLlmConfigFromEnv = (): LiveLlmConfig => {
|
||||
api_key: apiKey
|
||||
}
|
||||
}
|
||||
|
||||
export const loadLiveVisionLlmConfigFromEnv = (): LiveLlmConfig => {
|
||||
const primaryConfig = loadLiveLlmConfigFromEnv()
|
||||
const baseUrl = normalizeOpenAIBaseUrl(
|
||||
readEnv('AI_LIVE_VISION_BASE_URL') || primaryConfig.base_url
|
||||
)
|
||||
const apiKey = readEnv('AI_LIVE_VISION_API_KEY') || primaryConfig.api_key
|
||||
const modelName = resolveLiveVisionModelName(primaryConfig)
|
||||
return {
|
||||
id: 'ai-live-vision-config',
|
||||
name: 'AI Live Vision',
|
||||
providerKind: resolveLlmProviderKind({
|
||||
providerKind: readEnv('AI_LIVE_VISION_PROVIDER_KIND') || inferLlmProviderKind(baseUrl),
|
||||
baseUrl
|
||||
}),
|
||||
base_url: baseUrl,
|
||||
model_name: modelName,
|
||||
api_key: apiKey
|
||||
}
|
||||
}
|
||||
@@ -173,6 +173,7 @@ export const createRealAiDepsFactory = async (input: {
|
||||
defaultModelName: input.runtimeModelName
|
||||
}) || ''
|
||||
},
|
||||
visionModel: contextInput.visionModel,
|
||||
workspace: bindWorkspace
|
||||
? {
|
||||
workspaceId: 'test-workspace',
|
||||
|
||||
@@ -166,6 +166,10 @@ export type CreateAiRunContextInput = {
|
||||
allowedToolNames?: string[]
|
||||
runtimeSettingsOverride?: Partial<AiRuntimeSettings>
|
||||
modelName?: string
|
||||
visionModel?: {
|
||||
configId: string
|
||||
modelName: string
|
||||
}
|
||||
approvalPolicy?: ApprovalPolicy | ApprovalResolver
|
||||
checkCancelledOverride?: () => void
|
||||
humanRuntimeOverrides?: {
|
||||
|
||||
Reference in new issue
Block a user