diff --git a/build/package-size-budgets.json b/build/package-size-budgets.json index b4ca654c..765c28b5 100644 --- a/build/package-size-budgets.json +++ b/build/package-size-budgets.json @@ -4,10 +4,10 @@ "failureGrowthPercent": 8, "platforms": { "darwin-arm64": { - "installedApp": { "baselineMiB": 1113.3, "hardLimitMiB": 1200 }, - "appAsar": { "baselineMiB": 230.9, "hardLimitMiB": 252 }, - "dmg": { "baselineMiB": 432.9, "hardLimitMiB": 480 }, - "zip": { "baselineMiB": 418.5, "hardLimitMiB": 465 } + "installedApp": { "baselineMiB": 776.2, "hardLimitMiB": 870 }, + "appAsar": { "baselineMiB": 184.0, "hardLimitMiB": 205 }, + "dmg": { "baselineMiB": 273.9, "hardLimitMiB": 310 }, + "zip": { "baselineMiB": 289.1, "hardLimitMiB": 325 } } } } diff --git a/doc/architecture.md b/doc/architecture.md index a36f8cc7..d625fec5 100644 --- a/doc/architecture.md +++ b/doc/architecture.md @@ -42,7 +42,7 @@ NCurator-Desktop 是本地优先的 Electron 桌面应用。文档解析、嵌 工作区是默认执行范围而非硬边界：模型默认在工作区域内操作，系统级需求可访问工作区外绝对路径。文件工具的 `sourcePath` / `outputPath` 必须为本机绝对路径。内置浏览器由 Electron `WebContentsView` 承载，Stagehand 通过本机回环 CDP endpoint 精确连接对应 Tab。 -网页搜索继续由冻结的 Python sidecar 提供 DDG 搜索和 Crawl4AI 抓取。sidecar 不再携带 Playwright Chromium：首次使用网页工具时，Electron 创建专用、非持久化的 `WebContentsView` partition，sidecar 通过本机回环 CDP 连接 Electron Chromium，并按唯一 marker 选择该隔离 context。找不到唯一 context 时直接失败，禁止退回用户浏览器 context。 +网页搜索继续由冻结的 Python sidecar 提供 DDG 搜索和 Crawl4AI 抓取。sidecar 保留 Python Playwright 和 Playwright 驱动 JavaScript，但不再携带 Playwright Chromium 或 Node；驱动通过 `PLAYWRIGHT_NODEJS_PATH` 和 `ELECTRON_RUN_AS_NODE=1` 复用当前 Electron 可执行文件中的 Node。首次使用网页工具时，Electron 创建专用、非持久化的 `WebContentsView` partition，sidecar 通过本机回环 CDP 连接 Electron Chromium，并按唯一 marker 选择该隔离 context。找不到唯一 context 时直接失败，禁止退回用户浏览器 context。 开发版使用独立 `*-dev` user-data 目录；E2E 可通过 `NCURATOR_USER_DATA_PATH` 提供隔离目录。 diff --git a/doc/release.md b/doc/release.md index caff1802..182a5963 100644 --- a/doc/release.md +++ b/doc/release.md @@ -157,9 +157,9 @@ macOS 需要已安装并信任的 `Ncurator Update Signing` P12。当前使用 `pnpm run build` 主要完成： -1. 准备 native 模块和网页抓取 Python sidecar，完成类型检查与应用构建；sidecar 保留 DDG、Crawl4AI 和 Playwright 驱动，但复用 Electron Chromium，不下载或打包 Playwright 浏览器。 -2. 追踪生产依赖，生成最小化 `release-stage/app`。 -3. 验证 DOCX、PDF、PDF worker 和进程边界，混淆 `out/main`。 +1. 准备 native 模块和网页抓取 Python sidecar，完成类型检查与应用构建；sidecar 保留 DDG、Crawl4AI、Python Playwright 和 Playwright 驱动 JavaScript，但复用 Electron 的 Chromium 与 Node，不下载或打包 Playwright 浏览器和 Node。 +2. 追踪生产依赖，生成最小化 `release-stage/app`；只在 trace 未触达待删目录时，定向移除 MathJax 重复发行物、`pdf-parse` 未使用的 PDF.js 版本、已进入 Renderer bundle 的 ExcelJS browser dist，以及非 Windows 包中的 ConPTY 源码。 +3. 在裁剪后真实验证 DOCX/PDF 读取、MathJax 公式转 SVG、ExcelJS 写入回读、PDF worker 和进程边界，再混淆 `out/main`。 4. 审计 staging，再由 electron-builder 生成安装包。 5. 审计真实 `win-unpacked` 或 `.app` 的文件、依赖、敏感信息和体积。 6. macOS 额外执行签名和真实启动验证。 @@ -177,7 +177,9 @@ release-stage/ └── resources/{native,web_crawler_sidecar,open-with}/ ``` -安装包不得包含源码、测试、文档、开发配置、`.env`、锁文件、Source Map、测试日志、fixture、私钥或错误平台文件。网页抓取 sidecar 不得包含 `.local-browsers` 或 `chromium_headless_shell-*`，且解包体积不得超过 400 MiB；staging 和正式包审计都会硬校验。`app.asar` 不是加密，所有 `VITE_*` 变量都按公开信息处理。 +安装包不得包含源码、测试、文档、开发配置、`.env`、锁文件、Source Map、测试日志、fixture、私钥或错误平台文件。网页抓取 sidecar 必须保留 Playwright 驱动 JavaScript，不得包含 Playwright Node、`.local-browsers` 或 `chromium_headless_shell-*`，且解包体积不得超过 400 MiB；staging 和正式包审计都会硬校验。被定向裁剪的 Node 目录同样属于审计禁区，未来 trace 如果触达这些目录会在删除前直接使构建失败。`app.asar` 不是加密，所有 `VITE_*` 变量都按公开信息处理。 + +electron-builder 使用 `maximum` 压缩，以更慢的发布构建换取更小的安装包，不改变安装后的文件和运行逻辑。2026-09-01 的 macOS arm64 实测基线为：`.app` 776.2 MiB、`app.asar` 184.0 MiB、DMG 273.9 MiB、ZIP 289.1 MiB；同一 staging 下，`maximum` 相比默认压缩使 DMG 减少 26.1 MiB，ZIP 减少约 1.0 MiB。 候选 bundle 绑定 Tag、commit、release fragment、安装包、更新文件、大小、sha256 和 sha512。测试和发布前都会重新验证这些内容。 diff --git a/doc/test-plan/ai-test.md b/doc/test-plan/ai-test.md index c3e611f5..385013ce 100644 --- a/doc/test-plan/ai-test.md +++ b/doc/test-plan/ai-test.md @@ -50,7 +50,7 @@ - 全部 29 个工具均要求绑定专用 `modelResult.renderSuccess`（[`registry.ts`](src/main/services/ai/human/tools/registry.ts) compile-time 完整性检查）；**不对 prompt 措辞或 renderer 投影文字做 unit 单测**。工具契约由 **family unit contract**（如 `docx-tool-contract`、`terminal-tool-contract`、`text-file-tool-contract`）与 **backend tool integration** 保障。 - 目标：验证 tool 与 `hand` 执行守卫的生产输入输出。 - 约束：使用真实工具、store、docx apply service 和 artifact 流；禁止 fake `documentsStore`、`embeddingService`、`resolveDocxOpenTarget` 或 tool result。 -- `web-browsing` 工具测试先启动测试 Electron CDP host 和专用非持久化 partition，再覆盖冻结 sidecar bin 通过隔离 context 抓取，以及 `browser_use` 服务不可用诊断；禁止使用 sidecar 内置浏览器或 Electron 默认 context。 +- `web-browsing` 工具测试先启动测试 Electron CDP host 和专用非持久化 partition，再让冻结 sidecar bin 的 Python Playwright 显式复用该测试 Electron 的 Node，通过隔离 context 抓取；另覆盖 `browser_use` 服务不可用诊断。禁止使用 sidecar 内置 Node/浏览器或 Electron 默认 context。 ### 2.2 `human` @@ -495,7 +495,7 @@ DOCX tool 回归以 7 个公开工具、story 内容树、稳定 `nodeId`、segm - 测试类型：`backend` - What：`web_search` 传 `{ type: 'fetch', url }` 会返回扁平结构的 `requestedUrl/finalUrl`、标题、`markdown`、发布时间与分组 links；其中 `markdown` 内容来源于 crawl4ai 的 `fit_markdown`。 - Why：单页抓取是网页正文入口，必须稳定返回可被 brain 继续消费的网页事实和导航线索。 -- How：启动本地 HTTP server 和测试 Electron CDP host，创建专用非持久化 partition，通过真实冻结 sidecar bin 连接唯一 marker 对应的隔离 context 返回抓取结果，调用 `web_search({ type: 'fetch', url })`。 +- How：启动本地 HTTP server 和测试 Electron CDP host，创建专用非持久化 partition，通过 `PLAYWRIGHT_NODEJS_PATH` 让真实冻结 sidecar bin 的 Python Playwright 复用测试 Electron Node，再连接唯一 marker 对应的隔离 context 返回抓取结果，调用 `web_search({ type: 'fetch', url })`。 - Oracle：结果包含 `requestedUrl`、`finalUrl`、`title`、`markdown`、`publishedAt`、`links.internal/external`。 - Not：不验证复杂浏览器交互或外网可达性。 diff --git a/electron-builder.yml b/electron-builder.yml index 2b2b18b1..070931f2 100644 --- a/electron-builder.yml +++ b/electron-builder.yml @@ -1,6 +1,7 @@ appId: com.yoan.ncurator productName: 馆长 artifactName: Ncurator-${os}-${arch}.${ext} +compression: maximum directories: app: release-stage/app buildResources: build diff --git a/scripts/build/package/content-policy.mjs b/scripts/build/package/content-policy.mjs index 52097db3..3a9b96b8 100644 --- a/scripts/build/package/content-policy.mjs +++ b/scripts/build/package/content-policy.mjs @@ -163,6 +163,34 @@ export const assertSafePackagePaths = (paths) => { continue } + if ( + /(?:^|\/)node_modules\/mathjax-full\/(?:components|es5|ts)(?:\/|$)/u.test( + relativePath.toLowerCase() + ) + ) { + violations.push(`${relativePath}: removable MathJax distribution`) + continue + } + if ( + /(?:^|\/)node_modules\/pdf-parse\/lib\/pdf\.js\/(?:v1\.9\.426|v1\.10\.88|v2\.0\.550)(?:\/|$)/u.test( + relativePath.toLowerCase() + ) + ) { + violations.push(`${relativePath}: unused PDF.js version`) + continue + } + if (/(?:^|\/)node_modules\/exceljs\/dist(?:\/|$)/u.test(relativePath.toLowerCase())) { + violations.push(`${relativePath}: bundled ExcelJS browser distribution`) + continue + } + if ( + process.platform !== 'win32' && + /(?:^|\/)node_modules\/node-pty\/third_party(?:\/|$)/u.test(relativePath.toLowerCase()) + ) { + violations.push(`${relativePath}: Windows ConPTY sources on a non-Windows target`) + continue + } + if (FORBIDDEN_ROOT_SEGMENTS.has(lowerSegments[0])) { violations.push(`${relativePath}: forbidden project directory ${lowerSegments[0]}`) continue @@ -217,6 +245,22 @@ export const assertWebCrawlerSidecarPackage = (files, sourceLabel) => { if (sidecarFiles.length === 0) { throw new Error(`${sourceLabel} is missing the web crawler sidecar`) } + const bundledNode = sidecarFiles.find((item) => { + const relativePath = normalizeRelativePath(item.relativePath).toLowerCase() + return /\/playwright\/driver\/node(?:\.exe)?$/u.test(relativePath) + }) + if (bundledNode) { + throw new Error( + `${sourceLabel} includes a bundled Playwright Node runtime: ${bundledNode.relativePath}` + ) + } + const playwrightDriver = sidecarFiles.find((item) => { + const relativePath = normalizeRelativePath(item.relativePath).toLowerCase() + return /\/playwright\/driver\/package\/cli\.js$/u.test(relativePath) + }) + if (!playwrightDriver) { + throw new Error(`${sourceLabel} is missing the Python Playwright driver JavaScript`) + } const bytes = sidecarFiles.reduce((sum, item) => sum + item.size, 0) if (bytes > WEB_CRAWLER_SIDECAR_HARD_LIMIT_BYTES) { throw new Error( diff --git a/scripts/build/package/prepare-stage.mjs b/scripts/build/package/prepare-stage.mjs index a60d6e76..7b9da3ea 100644 --- a/scripts/build/package/prepare-stage.mjs +++ b/scripts/build/package/prepare-stage.mjs @@ -8,6 +8,7 @@ import { collectRuntimePackageNames } from '../runtime/bundle-graph.mjs' import { traceRuntimeDependencies, writeRuntimeTraceReport } from '../runtime/trace.mjs' import { verifyStagedRuntime } from '../runtime/verify-stage.mjs' import { obfuscatePackageMain, verifyPackageMainObfuscation } from './obfuscation.mjs' +import { pruneRuntimePackages } from './prune-runtime.mjs' import { installWindowsVcRuntime } from './vc-runtime.mjs' import { assertNodePtyNativeReleaseReady, @@ -299,7 +300,7 @@ const stripSourceMapReferences = () => { walk(STAGE_DIR) } -const pruneNodeModules = () => { +const pruneNodeModules = (runtimeTrace) => { const nodeModules = path.join(STAGE_DIR, 'node_modules') const targetPrebuildDirectory = `${process.platform}-${process.arch}` const isRequiredRuntimeDirectory = (directory) => @@ -399,6 +400,8 @@ const pruneNodeModules = () => { } } } + + return pruneRuntimePackages({ nodeModulesDirectory: nodeModules, runtimeTrace }) } const pruneStagedResources = () => { @@ -512,7 +515,7 @@ const main = async () => { if (runtimePackageNames.includes('node-pty')) installPreparedNodePtyNative() fs.rmSync(path.join(STAGE_DIR, 'pnpm-lock.yaml'), { force: true }) pruneCompiledOutput() - pruneNodeModules() + const runtimePruning = pruneNodeModules(runtimeTrace) installWindowsVcRuntime({ stageDir: STAGE_DIR, vcRedistDir: VC_REDIST_DIR }) pruneStagedResources() const packagedDependencies = collectInstalledTopLevelDependencies() @@ -545,7 +548,7 @@ const main = async () => { if (audit.status !== 0) throw new Error('staging package security audit failed') process.stdout.write( - `[package] staging ready for ${process.platform}-${process.arch}; runtime roots: ${runtimePackageNames.join(', ')}; packaged dependencies: ${String(Object.keys(packagedDependencies).length)}; loader smoke: ${String(runtimeVerification.docxDocuments)} DOCX / ${String(runtimeVerification.pdfDocuments)} PDF document(s); obfuscated main: ${String(obfuscation.totals.files)} file(s)\n` + `[package] staging ready for ${process.platform}-${process.arch}; runtime roots: ${runtimePackageNames.join(', ')}; packaged dependencies: ${String(Object.keys(packagedDependencies).length)}; targeted pruning: ${(runtimePruning.totalBytes / 1024 / 1024).toFixed(1)} MiB; runtime smoke: ${String(runtimeVerification.docxDocuments)} DOCX / ${String(runtimeVerification.pdfDocuments)} PDF document(s), MathJax SVG, ExcelJS round trip; obfuscated main: ${String(obfuscation.totals.files)} file(s)\n` ) } diff --git a/scripts/build/package/prune-runtime.mjs b/scripts/build/package/prune-runtime.mjs new file mode 100644 index 00000000..32108538 --- /dev/null +++ b/scripts/build/package/prune-runtime.mjs @@ -0,0 +1,124 @@ +import fs from 'node:fs' +import path from 'node:path' + +const normalizePath = (value) => String(value).replaceAll('\\', '/') + +const collectPackageTracePaths = (runtimeTrace, packageName) => { + const packageSegments = packageName.split('/') + const result = new Set() + for (const rawPath of runtimeTrace.trace?.files || []) { + const segments = normalizePath(rawPath).split('/') + for (let index = 0; index < segments.length; index += 1) { + if (segments[index] !== 'node_modules') continue + const packageStart = index + 1 + if ( + packageSegments.every( + (segment, segmentIndex) => segments[packageStart + segmentIndex] === segment + ) + ) { + result.add(segments.slice(packageStart + packageSegments.length).join('/')) + } + } + } + return [...result].sort() +} + +const assertTracedPackage = (runtimeTrace, packageName) => { + if (!(runtimeTrace.packages || []).some((item) => item.name === packageName)) { + throw new Error(`runtime pruning requires traced package ${packageName}`) + } + const paths = collectPackageTracePaths(runtimeTrace, packageName) + if (paths.length === 0) { + throw new Error(`runtime pruning has no traced files for ${packageName}`) + } + return paths +} + +export const assertRuntimePruningTrace = (runtimeTrace, platform = process.platform) => { + const mathJaxPaths = assertTracedPackage(runtimeTrace, 'mathjax-full') + const unsafeMathJaxPath = mathJaxPaths.find( + (relativePath) => relativePath !== 'package.json' && !relativePath.startsWith('js/') + ) + if (unsafeMathJaxPath) { + throw new Error(`mathjax-full trace reaches a removable path: ${unsafeMathJaxPath}`) + } + + const pdfParsePaths = assertTracedPackage(runtimeTrace, 'pdf-parse') + const unsafePdfVersion = pdfParsePaths.find((relativePath) => { + const match = relativePath.match(/^lib\/pdf\.js\/(v[^/]+)(?:\/|$)/u) + return match && match[1] !== 'v1.10.100' + }) + if (unsafePdfVersion) { + throw new Error(`pdf-parse trace reaches a removable PDF.js version: ${unsafePdfVersion}`) + } + + const excelJsPaths = assertTracedPackage(runtimeTrace, 'exceljs') + const unsafeExcelJsPath = excelJsPaths.find( + (relativePath) => relativePath === 'dist' || relativePath.startsWith('dist/') + ) + if (unsafeExcelJsPath) { + throw new Error(`exceljs trace reaches removable browser dist: ${unsafeExcelJsPath}`) + } + + if (platform !== 'win32') { + const nodePtyPaths = assertTracedPackage(runtimeTrace, 'node-pty') + const unsafeNodePtyPath = nodePtyPaths.find( + (relativePath) => relativePath === 'third_party' || relativePath.startsWith('third_party/') + ) + if (unsafeNodePtyPath) { + throw new Error(`node-pty trace reaches removable Windows sources: ${unsafeNodePtyPath}`) + } + } +} + +const directorySize = (targetPath) => { + if (!fs.existsSync(targetPath)) return 0 + const stat = fs.lstatSync(targetPath) + if (!stat.isDirectory()) return stat.size + return fs + .readdirSync(targetPath) + .reduce((total, name) => total + directorySize(path.join(targetPath, name)), 0) +} + +export const pruneRuntimePackages = ({ + nodeModulesDirectory, + runtimeTrace, + platform = process.platform +}) => { + assertRuntimePruningTrace(runtimeTrace, platform) + + const targets = [ + ['mathjax-full/es5', path.join(nodeModulesDirectory, 'mathjax-full', 'es5')], + ['mathjax-full/components', path.join(nodeModulesDirectory, 'mathjax-full', 'components')], + ['mathjax-full/ts', path.join(nodeModulesDirectory, 'mathjax-full', 'ts')], + [ + 'pdf-parse PDF.js v1.9.426', + path.join(nodeModulesDirectory, 'pdf-parse', 'lib', 'pdf.js', 'v1.9.426') + ], + [ + 'pdf-parse PDF.js v1.10.88', + path.join(nodeModulesDirectory, 'pdf-parse', 'lib', 'pdf.js', 'v1.10.88') + ], + [ + 'pdf-parse PDF.js v2.0.550', + path.join(nodeModulesDirectory, 'pdf-parse', 'lib', 'pdf.js', 'v2.0.550') + ], + ['exceljs/dist', path.join(nodeModulesDirectory, 'exceljs', 'dist')] + ] + if (platform !== 'win32') { + targets.push([ + 'node-pty Windows ConPTY', + path.join(nodeModulesDirectory, 'node-pty', 'third_party') + ]) + } + + const removed = targets.map(([label, targetPath]) => { + const bytes = directorySize(targetPath) + fs.rmSync(targetPath, { recursive: true, force: true }) + return { label, bytes } + }) + return { + removed, + totalBytes: removed.reduce((total, item) => total + item.bytes, 0) + } +} diff --git a/scripts/build/runtime/verify-stage.mjs b/scripts/build/runtime/verify-stage.mjs index c0cdc513..7c543385 100644 --- a/scripts/build/runtime/verify-stage.mjs +++ b/scripts/build/runtime/verify-stage.mjs @@ -21,6 +21,12 @@ const loadStageExport = async (stageRequire, stageNodeModules, specifier, export return value } +const loadStageModule = (stageRequire, stageNodeModules, specifier) => { + const resolved = stageRequire.resolve(specifier) + assertInside(stageNodeModules, resolved, specifier) + return stageRequire(resolved) +} + export const verifyStagedRuntime = async ({ stageDirectory, reportPath, rootDirectory }) => { const stageRoot = path.resolve(stageDirectory) const root = path.resolve(rootDirectory) @@ -116,11 +122,59 @@ export const verifyStagedRuntime = async ({ stageDirectory, reportPath, rootDire assertDocuments(docxDocuments, 'DOCX') assertDocuments(pdfDocuments, 'PDF') + const { mathjax } = loadStageModule(stageRequire, stageNodeModules, 'mathjax-full/js/mathjax.js') + const { TeX } = loadStageModule(stageRequire, stageNodeModules, 'mathjax-full/js/input/tex.js') + const { AllPackages } = loadStageModule( + stageRequire, + stageNodeModules, + 'mathjax-full/js/input/tex/AllPackages.js' + ) + const { SVG } = loadStageModule(stageRequire, stageNodeModules, 'mathjax-full/js/output/svg.js') + const { liteAdaptor } = loadStageModule( + stageRequire, + stageNodeModules, + 'mathjax-full/js/adaptors/liteAdaptor.js' + ) + const { RegisterHTMLHandler } = loadStageModule( + stageRequire, + stageNodeModules, + 'mathjax-full/js/handlers/html.js' + ) + const adaptor = liteAdaptor() + RegisterHTMLHandler(adaptor) + const mathDocument = mathjax.document('', { + InputJax: new TeX({ packages: AllPackages }), + OutputJax: new SVG({ fontCache: 'none' }) + }) + const formulaMarkup = adaptor.outerHTML( + mathDocument.convert('x^2 + y^2 = z^2', { + display: true, + em: 16, + ex: 8, + containerWidth: 1200 + }) + ) + if (!formulaMarkup.includes('')) { + throw new Error('MathJax staged runtime returned no SVG') + } + + const ExcelJS = loadStageModule(stageRequire, stageNodeModules, 'exceljs') + const workbook = new ExcelJS.Workbook() + workbook.addWorksheet('Smoke').getCell('A1').value = 'NCurator' + const workbookBytes = await workbook.xlsx.writeBuffer() + const loadedWorkbook = new ExcelJS.Workbook() + await loadedWorkbook.xlsx.load(workbookBytes) + if (loadedWorkbook.getWorksheet('Smoke')?.getCell('A1').value !== 'NCurator') { + throw new Error('ExcelJS staged runtime round trip failed') + } + return { packages: report.packages.length, requiredFiles: report.requiredFiles.length, docxDocuments: docxDocuments.length, pdfDocuments: pdfDocuments.length, + formulaSvgBytes: Buffer.byteLength(formulaMarkup), + excelWorkbookBytes: workbookBytes.byteLength, suppressedPdfWarnings } } diff --git a/scripts/build/sidecar/build.py b/scripts/build/sidecar/build.py index 17c285e8..f415416a 100644 --- a/scripts/build/sidecar/build.py +++ b/scripts/build/sidecar/build.py @@ -50,6 +50,21 @@ def copy_playwright_driver_bundle(sidecar_dist_dir: Path) -> None: ignore=shutil.ignore_patterns(".local-browsers"), ) + bundled_node = target / ( + "node.exe" if platform.system().lower().startswith("windows") else "node" + ) + if bundled_node.exists(): + bundled_node.unlink() + + +def assert_no_playwright_bundled_node(sidecar_dist_dir: Path) -> None: + driver_dir = sidecar_dist_dir / "_internal" / "playwright" / "driver" + bundled_nodes = [driver_dir / "node", driver_dir / "node.exe"] + forbidden = [candidate for candidate in bundled_nodes if candidate.exists()] + if forbidden: + paths = ", ".join(str(item.relative_to(sidecar_dist_dir)) for item in forbidden) + raise RuntimeError(f"Playwright must reuse the Electron Node runtime: {paths}") + def assert_no_playwright_browser_bundle(sidecar_dist_dir: Path) -> None: forbidden = [] @@ -148,6 +163,7 @@ def main() -> int: copy_playwright_driver_bundle(sidecar_dist_dir) prune_non_runtime_files(sidecar_dist_dir) assert_no_playwright_browser_bundle(sidecar_dist_dir) + assert_no_playwright_bundled_node(sidecar_dist_dir) return 0 diff --git a/src/main/services/ai/human/prompts/brain.ts b/src/main/services/ai/human/prompts/brain.ts index ddd1af4c..3c73c535 100644 --- a/src/main/services/ai/human/prompts/brain.ts +++ b/src/main/services/ai/human/prompts/brain.ts @@ -76,6 +76,7 @@ export const buildBrainSystemPrompt = (input: { ctx: AiRunContext }): string => - 当 terminal_run 返回命令仍在执行时，多尝试等待终端命令完成后（再读取结果或继续），不要在命令未完成时空转轮询；仅当判断等待不合理时才中断并分析原因。 + - 当需要等待操作时，使用终端指令来进行等待。 @@ -123,9 +124,7 @@ export const buildBrainUserMessage = (input: { : '' const renderActiveState = (): string => { const skillsLine = - learnedSkillIds.length > 0 - ? `已学习技能：${learnedSkillIds.join('、')}` - : '已学习技能：无' + learnedSkillIds.length > 0 ? `已学习技能：${learnedSkillIds.join('、')}` : '已学习技能：无' const planLine = input.injectPlan && plan.length > 0 ? `计划进度：${plan diff --git a/src/main/services/webCrawler/WebCrawlerService.ts b/src/main/services/webCrawler/WebCrawlerService.ts index a2071f50..610da04c 100644 --- a/src/main/services/webCrawler/WebCrawlerService.ts +++ b/src/main/services/webCrawler/WebCrawlerService.ts @@ -32,6 +32,7 @@ type PendingRequest = { export type WebCrawlerServiceOptions = { sidecarCommand?: SidecarCommand sidecarEnv?: NodeJS.ProcessEnv + playwrightNodeExecutablePath?: string resolveCdpConnection?: () => Promise appRootPath?: string isPackaged?: boolean @@ -45,6 +46,8 @@ const PACKAGED_SIDECAR_BIN_DIR = 'bin' const PYTHON_SIDECAR_DEBUG_ENV = 'NCURATOR_WEB_CRAWLER_USE_PYTHON_SIDECAR' const SIDECAR_CDP_URL_ENV = 'NCURATOR_WEB_CRAWLER_CDP_URL' const SIDECAR_CDP_CONTEXT_MARKER_ENV = 'NCURATOR_WEB_CRAWLER_CDP_CONTEXT_MARKER_URL' +const PLAYWRIGHT_NODEJS_PATH_ENV = 'PLAYWRIGHT_NODEJS_PATH' +const ELECTRON_RUN_AS_NODE_ENV = 'ELECTRON_RUN_AS_NODE' const LOOPBACK_HOSTNAMES = new Set(['127.0.0.1', 'localhost', '::1', '[::1]']) const DEV_SIDECAR_SCRIPT_PATH = path.join( 'resources', @@ -437,6 +440,7 @@ const buildSidecarError = (response: WebCrawlerSidecarResponse): Error => { export class ManagedWebCrawlerService implements WebCrawlerServiceContract { private readonly sidecarCommand: SidecarCommand + private readonly playwrightNodeExecutablePath: string private readonly resolveCdpConnection?: () => Promise private readonly requestTimeoutMs: number private child?: ChildProcessWithoutNullStreams @@ -448,6 +452,13 @@ export class ManagedWebCrawlerService implements WebCrawlerServiceContract { public constructor(options: WebCrawlerServiceOptions = {}) { this.sidecarCommand = options.sidecarCommand || resolveDefaultSidecarCommand(options) + this.playwrightNodeExecutablePath = path.resolve( + options.playwrightNodeExecutablePath || process.execPath + ) + assertExecutableExists( + this.playwrightNodeExecutablePath, + 'missing Node runtime for Python Playwright' + ) this.resolveCdpConnection = options.resolveCdpConnection this.requestTimeoutMs = Math.max( 1_000, @@ -523,6 +534,8 @@ export class ManagedWebCrawlerService implements WebCrawlerServiceContract { env: { ...process.env, ...this.sidecarCommand.env, + [PLAYWRIGHT_NODEJS_PATH_ENV]: this.playwrightNodeExecutablePath, + [ELECTRON_RUN_AS_NODE_ENV]: '1', [SIDECAR_CDP_URL_ENV]: cdpUrl, [SIDECAR_CDP_CONTEXT_MARKER_ENV]: contextMarkerUrl }, diff --git a/tests/case/unit/release/package-content-policy.test.ts b/tests/case/unit/release/package-content-policy.test.ts index 5449947f..f65c0926 100644 --- a/tests/case/unit/release/package-content-policy.test.ts +++ b/tests/case/unit/release/package-content-policy.test.ts @@ -7,6 +7,7 @@ import { createPackage } from '@electron/asar' import { afterEach, describe, expect, it } from 'vitest' import { assertSafePackagePaths, + assertWebCrawlerSidecarPackage, normalizeRelativePath } from '../../../../scripts/build/package/content-policy.mjs' import { PROTECTED_MAIN_PLAINTEXT_MARKERS } from '../../../../scripts/build/package/obfuscation.mjs' @@ -31,12 +32,49 @@ describe('package content policy paths', () => { }) it('permits ExcelJS runtime modules while rejecting other doc directories', () => { + expect(() => assertSafePackagePaths(['node_modules/exceljs/lib/doc/workbook.js'])).not.toThrow() expect(() => - assertSafePackagePaths(['node_modules/exceljs/lib/doc/workbook.js']) - ).not.toThrow() - expect(() => assertSafePackagePaths(['node_modules/runtime-package/lib/doc/readme.js'])).toThrow( - 'forbidden directory doc' + assertSafePackagePaths(['node_modules/runtime-package/lib/doc/readme.js']) + ).toThrow('forbidden directory doc') + }) + + it('rejects runtime directories covered by deterministic package pruning', () => { + expect(() => assertSafePackagePaths(['node_modules/mathjax-full/es5/tex-svg.js'])).toThrow( + 'removable MathJax distribution' ) + expect(() => + assertSafePackagePaths(['node_modules/pdf-parse/lib/pdf.js/v2.0.550/build/pdf.js']) + ).toThrow('unused PDF.js version') + expect(() => assertSafePackagePaths(['node_modules/exceljs/dist/exceljs.js'])).toThrow( + 'bundled ExcelJS browser distribution' + ) + }) + + it('requires Playwright driver JavaScript and rejects its bundled Node runtime', () => { + const driver = { + relativePath: + 'resources/web_crawler_sidecar/bin/app/_internal/playwright/driver/package/cli.js', + size: 10 + } + expect(assertWebCrawlerSidecarPackage([driver], 'fixture')).toBe(10) + expect(() => + assertWebCrawlerSidecarPackage( + [ + driver, + { + relativePath: 'resources/web_crawler_sidecar/bin/app/_internal/playwright/driver/node', + size: 100 + } + ], + 'fixture' + ) + ).toThrow('bundled Playwright Node runtime') + expect(() => + assertWebCrawlerSidecarPackage( + [{ relativePath: 'resources/web_crawler_sidecar/bin/app/app', size: 10 }], + 'fixture' + ) + ).toThrow('missing the Python Playwright driver JavaScript') }) it('audits nested files in an asar archive on the current platform', async () => { @@ -54,10 +92,32 @@ describe('package content policy paths', () => { 'Release', 'spawn-helper' ) + const sidecarExecutablePath = path.join( + packagedApp, + 'resources', + 'web_crawler_sidecar', + 'bin', + 'web_crawler_sidecar' + ) + const playwrightDriverPath = path.join( + packagedApp, + 'resources', + 'web_crawler_sidecar', + 'bin', + '_internal', + 'playwright', + 'driver', + 'package', + 'cli.js' + ) await mkdir(path.join(appSource, 'out', 'main'), { recursive: true }) await mkdir(path.dirname(archivePath), { recursive: true }) await mkdir(path.dirname(nodePtyHelperPath), { recursive: true }) + await mkdir(path.dirname(sidecarExecutablePath), { recursive: true }) + await mkdir(path.dirname(playwrightDriverPath), { recursive: true }) await writeFile(nodePtyHelperPath, '', { mode: 0o755 }) + await writeFile(sidecarExecutablePath, '', { mode: 0o755 }) + await writeFile(playwrightDriverPath, 'module.exports = {}\n') const vcRuntimeDllDirectory = resolveVcRuntimeDllDirectory(packagedApp) await mkdir(vcRuntimeDllDirectory, { recursive: true }) for (const filename of VC_REDIST_DLL_NAMES) { diff --git a/tests/case/unit/release/package-size-budget.test.ts b/tests/case/unit/release/package-size-budget.test.ts index 030f97db..2d030fb7 100644 --- a/tests/case/unit/release/package-size-budget.test.ts +++ b/tests/case/unit/release/package-size-budget.test.ts @@ -49,10 +49,10 @@ describe('package size budget', () => { const config = loadPackageSizeBudgetConfig(path.resolve('build/package-size-budgets.json')) expect(config.platforms['darwin-arm64']).toMatchObject({ - installedApp: { baselineMiB: 1113.3, hardLimitMiB: 1200 }, - appAsar: { baselineMiB: 230.9, hardLimitMiB: 252 }, - dmg: { baselineMiB: 432.9, hardLimitMiB: 480 }, - zip: { baselineMiB: 418.5, hardLimitMiB: 465 } + installedApp: { baselineMiB: 776.2, hardLimitMiB: 870 }, + appAsar: { baselineMiB: 184.0, hardLimitMiB: 205 }, + dmg: { baselineMiB: 273.9, hardLimitMiB: 310 }, + zip: { baselineMiB: 289.1, hardLimitMiB: 325 } }) expect(config.platforms['darwin-x64']).toBeUndefined() }) diff --git a/tests/case/unit/release/runtime-package-pruning.test.ts b/tests/case/unit/release/runtime-package-pruning.test.ts new file mode 100644 index 00000000..2602e107 --- /dev/null +++ b/tests/case/unit/release/runtime-package-pruning.test.ts @@ -0,0 +1,109 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + assertRuntimePruningTrace, + pruneRuntimePackages +} from '../../../../scripts/build/package/prune-runtime.mjs' + +let temporaryDirectory = '' + +afterEach(async () => { + if (temporaryDirectory) await rm(temporaryDirectory, { recursive: true, force: true }) + temporaryDirectory = '' +}) + +const runtimeTrace = { + packages: [ + { name: 'mathjax-full' }, + { name: 'pdf-parse' }, + { name: 'exceljs' }, + { name: 'node-pty' } + ], + trace: { + files: [ + 'node_modules/.pnpm/mathjax-full@3.2.2/node_modules/mathjax-full/js/mathjax.js', + 'node_modules/.pnpm/mathjax-full@3.2.2/node_modules/mathjax-full/package.json', + 'node_modules/.pnpm/pdf-parse@1.1.1/node_modules/pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js', + 'node_modules/.pnpm/exceljs@4.4.0/node_modules/exceljs/lib/doc/workbook.js', + 'node_modules/.pnpm/node-pty@1.2.0/node_modules/node-pty/lib/index.js' + ] + } +} + +const writeFixture = async (relativePath: string): Promise => { + const targetPath = path.join(temporaryDirectory, relativePath) + await mkdir(path.dirname(targetPath), { recursive: true }) + await writeFile(targetPath, 'runtime-fixture') +} + +describe('targeted runtime package pruning', () => { + it('removes only approved redundant distributions on non-Windows targets', async () => { + temporaryDirectory = await mkdtemp(path.join(os.tmpdir(), 'ncurator-runtime-pruning-')) + const removable = [ + 'mathjax-full/es5/tex-svg.js', + 'mathjax-full/components/src/component.js', + 'mathjax-full/ts/input/tex.ts', + 'pdf-parse/lib/pdf.js/v1.9.426/build/pdf.js', + 'pdf-parse/lib/pdf.js/v1.10.88/build/pdf.js', + 'pdf-parse/lib/pdf.js/v2.0.550/build/pdf.js', + 'exceljs/dist/exceljs.js', + 'node-pty/third_party/conpty/conpty.dll' + ] + await Promise.all( + [ + ...removable, + 'mathjax-full/js/mathjax.js', + 'pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js' + ].map(writeFixture) + ) + + const result = pruneRuntimePackages({ + nodeModulesDirectory: temporaryDirectory, + runtimeTrace, + platform: 'darwin' + }) + + for (const relativePath of removable) { + expect(fs.existsSync(path.join(temporaryDirectory, relativePath))).toBe(false) + } + expect(fs.existsSync(path.join(temporaryDirectory, 'mathjax-full/js/mathjax.js'))).toBe(true) + expect( + fs.existsSync(path.join(temporaryDirectory, 'pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js')) + ).toBe(true) + expect(result.totalBytes).toBeGreaterThan(0) + }) + + it('fails before pruning when trace reaches a removable distribution', () => { + for (const unsafePath of [ + 'node_modules/mathjax-full/es5/tex-svg.js', + 'node_modules/pdf-parse/lib/pdf.js/v2.0.550/build/pdf.js', + 'node_modules/exceljs/dist/exceljs.js', + 'node_modules/node-pty/third_party/conpty/conpty.dll' + ]) { + expect(() => + assertRuntimePruningTrace({ + ...runtimeTrace, + trace: { files: [...runtimeTrace.trace.files, unsafePath] } + }) + ).toThrow('trace reaches') + } + }) + + it('retains node-pty ConPTY on Windows targets', async () => { + temporaryDirectory = await mkdtemp(path.join(os.tmpdir(), 'ncurator-runtime-pruning-')) + await writeFixture('node-pty/third_party/conpty/conpty.dll') + + pruneRuntimePackages({ + nodeModulesDirectory: temporaryDirectory, + runtimeTrace, + platform: 'win32' + }) + + expect( + fs.existsSync(path.join(temporaryDirectory, 'node-pty/third_party/conpty/conpty.dll')) + ).toBe(true) + }) +}) diff --git a/tests/harness/backend/webCrawlerTestUtils.ts b/tests/harness/backend/webCrawlerTestUtils.ts index 1c053fdb..8a8dbf65 100644 --- a/tests/harness/backend/webCrawlerTestUtils.ts +++ b/tests/harness/backend/webCrawlerTestUtils.ts @@ -8,6 +8,9 @@ import { CdpEndpointService } from '../../../src/main/services/browser/CdpEndpoi import { ManagedWebCrawlerService } from '../../../src/main/services/webCrawler/WebCrawlerService' const electronExecutablePath = createRequire(import.meta.url)('electron') as string +const playwrightNodeExecutablePath = path.resolve( + String(process.env.NCURATOR_TEST_PLAYWRIGHT_NODEJS_PATH || electronExecutablePath) +) const stopProcess = async (child: ChildProcess): Promise => { if (child.exitCode !== null || child.signalCode !== null) return @@ -63,6 +66,7 @@ export const createWebCrawlerService = async (): Promise