feat: update package size budgets and enhance runtime package pruning logic for optimized builds

This commit is contained in:
hy committed 2026-09-01 18:03:14 +08:00
1 parent 7cd3782f1e
commit a2608a9f55
16 files changed
+454 -25

No files matched your search

+4 -4
View File
@@ -4,10 +4,10 @@
"failureGrowthPercent": 8, "failureGrowthPercent": 8,
"platforms": { "platforms": {
"darwin-arm64": { "darwin-arm64": {
"installedApp": { "baselineMiB": 1113.3, "hardLimitMiB": 1200 }, "installedApp": { "baselineMiB": 776.2, "hardLimitMiB": 870 },
"appAsar": { "baselineMiB": 230.9, "hardLimitMiB": 252 }, "appAsar": { "baselineMiB": 184.0, "hardLimitMiB": 205 },
"dmg": { "baselineMiB": 432.9, "hardLimitMiB": 480 }, "dmg": { "baselineMiB": 273.9, "hardLimitMiB": 310 },
"zip": { "baselineMiB": 418.5, "hardLimitMiB": 465 } "zip": { "baselineMiB": 289.1, "hardLimitMiB": 325 }
} }
} }
} }
+1 -1
View File
@@ -42,7 +42,7 @@ NCurator-Desktop 是本地优先的 Electron 桌面应用。文档解析、嵌
工作区是默认执行范围而非硬边界:模型默认在工作区域内操作,系统级需求可访问工作区外绝对路径。文件工具的 `sourcePath` / `outputPath` 必须为本机绝对路径。内置浏览器由 Electron `WebContentsView` 承载,Stagehand 通过本机回环 CDP endpoint 精确连接对应 Tab。 工作区是默认执行范围而非硬边界:模型默认在工作区域内操作,系统级需求可访问工作区外绝对路径。文件工具的 `sourcePath` / `outputPath` 必须为本机绝对路径。内置浏览器由 Electron `WebContentsView` 承载,Stagehand 通过本机回环 CDP endpoint 精确连接对应 Tab。
网页搜索继续由冻结的 Python sidecar 提供 DDG 搜索和 Crawl4AI 抓取。sidecar 不再携带 Playwright Chromium:首次使用网页工具时,Electron 创建专用、非持久化的 `WebContentsView` partition,sidecar 通过本机回环 CDP 连接 Electron Chromium,并按唯一 marker 选择该隔离 context。找不到唯一 context 时直接失败,禁止退回用户浏览器 context。 网页搜索继续由冻结的 Python sidecar 提供 DDG 搜索和 Crawl4AI 抓取。sidecar 保留 Python Playwright 和 Playwright 驱动 JavaScript,但不再携带 Playwright Chromium 或 Node;驱动通过 `PLAYWRIGHT_NODEJS_PATH` 和 `ELECTRON_RUN_AS_NODE=1` 复用当前 Electron 可执行文件中的 Node。首次使用网页工具时,Electron 创建专用、非持久化的 `WebContentsView` partition,sidecar 通过本机回环 CDP 连接 Electron Chromium,并按唯一 marker 选择该隔离 context。找不到唯一 context 时直接失败,禁止退回用户浏览器 context。
开发版使用独立 `*-dev` user-data 目录;E2E 可通过 `NCURATOR_USER_DATA_PATH` 提供隔离目录。 开发版使用独立 `*-dev` user-data 目录;E2E 可通过 `NCURATOR_USER_DATA_PATH` 提供隔离目录。
+6 -4
View File
@@ -157,9 +157,9 @@ macOS 需要已安装并信任的 `Ncurator Update Signing` P12。当前使用
`pnpm run build` 主要完成: `pnpm run build` 主要完成:
1. 准备 native 模块和网页抓取 Python sidecar,完成类型检查与应用构建;sidecar 保留 DDG、Crawl4AI 和 Playwright 驱动,但复用 Electron Chromium,不下载或打包 Playwright 浏览器。 1. 准备 native 模块和网页抓取 Python sidecar,完成类型检查与应用构建;sidecar 保留 DDG、Crawl4AI、Python Playwright 和 Playwright 驱动 JavaScript,但复用 Electron 的 Chromium 与 Node,不下载或打包 Playwright 浏览器和 Node。
2. 追踪生产依赖,生成最小化 `release-stage/app`。 2. 追踪生产依赖,生成最小化 `release-stage/app`;只在 trace 未触达待删目录时,定向移除 MathJax 重复发行物、`pdf-parse` 未使用的 PDF.js 版本、已进入 Renderer bundle 的 ExcelJS browser dist,以及非 Windows 包中的 ConPTY 源码。
3. 验证 DOCX、PDF、PDF worker 和进程边界,混淆 `out/main`。 3. 在裁剪后真实验证 DOCX/PDF 读取、MathJax 公式转 SVG、ExcelJS 写入回读、PDF worker 和进程边界,再混淆 `out/main`。
4. 审计 staging,再由 electron-builder 生成安装包。 4. 审计 staging,再由 electron-builder 生成安装包。
5. 审计真实 `win-unpacked` 或 `.app` 的文件、依赖、敏感信息和体积。 5. 审计真实 `win-unpacked` 或 `.app` 的文件、依赖、敏感信息和体积。
6. macOS 额外执行签名和真实启动验证。 6. macOS 额外执行签名和真实启动验证。
@@ -177,7 +177,9 @@ release-stage/
└── resources/{native,web_crawler_sidecar,open-with}/ └── resources/{native,web_crawler_sidecar,open-with}/
``` ```
安装包不得包含源码、测试、文档、开发配置、`.env`、锁文件、Source Map、测试日志、fixture、私钥或错误平台文件。网页抓取 sidecar 不得包含 `.local-browsers` 或 `chromium_headless_shell-*`,且解包体积不得超过 400 MiB;staging 和正式包审计都会硬校验。`app.asar` 不是加密,所有 `VITE_*` 变量都按公开信息处理。 安装包不得包含源码、测试、文档、开发配置、`.env`、锁文件、Source Map、测试日志、fixture、私钥或错误平台文件。网页抓取 sidecar 必须保留 Playwright 驱动 JavaScript,不得包含 Playwright Node、`.local-browsers` 或 `chromium_headless_shell-*`,且解包体积不得超过 400 MiB;staging 和正式包审计都会硬校验。被定向裁剪的 Node 目录同样属于审计禁区,未来 trace 如果触达这些目录会在删除前直接使构建失败。`app.asar` 不是加密,所有 `VITE_*` 变量都按公开信息处理。
electron-builder 使用 `maximum` 压缩,以更慢的发布构建换取更小的安装包,不改变安装后的文件和运行逻辑。2026-09-01 的 macOS arm64 实测基线为:`.app` 776.2 MiB、`app.asar` 184.0 MiB、DMG 273.9 MiB、ZIP 289.1 MiB;同一 staging 下,`maximum` 相比默认压缩使 DMG 减少 26.1 MiB,ZIP 减少约 1.0 MiB。
候选 bundle 绑定 Tag、commit、release fragment、安装包、更新文件、大小、sha256 和 sha512。测试和发布前都会重新验证这些内容。 候选 bundle 绑定 Tag、commit、release fragment、安装包、更新文件、大小、sha256 和 sha512。测试和发布前都会重新验证这些内容。
+2 -2
View File
@@ -50,7 +50,7 @@
- 全部 29 个工具均要求绑定专用 `modelResult.renderSuccess`([`registry.ts`](src/main/services/ai/human/tools/registry.ts) compile-time 完整性检查);**不对 prompt 措辞或 renderer 投影文字做 unit 单测**。工具契约由 **family unit contract**(如 `docx-tool-contract`、`terminal-tool-contract`、`text-file-tool-contract`)与 **backend tool integration** 保障。 - 全部 29 个工具均要求绑定专用 `modelResult.renderSuccess`([`registry.ts`](src/main/services/ai/human/tools/registry.ts) compile-time 完整性检查);**不对 prompt 措辞或 renderer 投影文字做 unit 单测**。工具契约由 **family unit contract**(如 `docx-tool-contract`、`terminal-tool-contract`、`text-file-tool-contract`)与 **backend tool integration** 保障。
- 目标:验证 tool 与 `hand` 执行守卫的生产输入输出。 - 目标:验证 tool 与 `hand` 执行守卫的生产输入输出。
- 约束:使用真实工具、store、docx apply service 和 artifact 流;禁止 fake `documentsStore`、`embeddingService`、`resolveDocxOpenTarget` 或 tool result。 - 约束:使用真实工具、store、docx apply service 和 artifact 流;禁止 fake `documentsStore`、`embeddingService`、`resolveDocxOpenTarget` 或 tool result。
- `web-browsing` 工具测试先启动测试 Electron CDP host 和专用非持久化 partition,再覆盖冻结 sidecar bin 通过隔离 context 抓取,以及 `browser_use` 服务不可用诊断;禁止使用 sidecar 内置浏览器或 Electron 默认 context。 - `web-browsing` 工具测试先启动测试 Electron CDP host 和专用非持久化 partition,再让冻结 sidecar bin 的 Python Playwright 显式复用该测试 Electron 的 Node,通过隔离 context 抓取;另覆盖 `browser_use` 服务不可用诊断。禁止使用 sidecar 内置 Node/浏览器或 Electron 默认 context。
### 2.2 `human` ### 2.2 `human`
@@ -495,7 +495,7 @@ DOCX tool 回归以 7 个公开工具、story 内容树、稳定 `nodeId`、segm
- 测试类型:`backend` - 测试类型:`backend`
- What:`web_search` 传 `{ type: 'fetch', url }` 会返回扁平结构的 `requestedUrl/finalUrl`、标题、`markdown`、发布时间与分组 links;其中 `markdown` 内容来源于 crawl4ai 的 `fit_markdown`。 - What:`web_search` 传 `{ type: 'fetch', url }` 会返回扁平结构的 `requestedUrl/finalUrl`、标题、`markdown`、发布时间与分组 links;其中 `markdown` 内容来源于 crawl4ai 的 `fit_markdown`。
- Why:单页抓取是网页正文入口,必须稳定返回可被 brain 继续消费的网页事实和导航线索。 - Why:单页抓取是网页正文入口,必须稳定返回可被 brain 继续消费的网页事实和导航线索。
- How:启动本地 HTTP server 和测试 Electron CDP host,创建专用非持久化 partition,通过真实冻结 sidecar bin 连接唯一 marker 对应的隔离 context 返回抓取结果,调用 `web_search({ type: 'fetch', url })`。 - How:启动本地 HTTP server 和测试 Electron CDP host,创建专用非持久化 partition,通过 `PLAYWRIGHT_NODEJS_PATH` 让真实冻结 sidecar bin 的 Python Playwright 复用测试 Electron Node,再连接唯一 marker 对应的隔离 context 返回抓取结果,调用 `web_search({ type: 'fetch', url })`。
- Oracle:结果包含 `requestedUrl`、`finalUrl`、`title`、`markdown`、`publishedAt`、`links.internal/external`。 - Oracle:结果包含 `requestedUrl`、`finalUrl`、`title`、`markdown`、`publishedAt`、`links.internal/external`。
- Not:不验证复杂浏览器交互或外网可达性。 - Not:不验证复杂浏览器交互或外网可达性。
+1
View File
@@ -1,6 +1,7 @@
appId: com.yoan.ncurator appId: com.yoan.ncurator
productName: 馆长 productName: 馆长
artifactName: Ncurator-${os}-${arch}.${ext} artifactName: Ncurator-${os}-${arch}.${ext}
compression: maximum
directories: directories:
app: release-stage/app app: release-stage/app
buildResources: build buildResources: build
+44
View File
@@ -163,6 +163,34 @@ export const assertSafePackagePaths = (paths) => {
continue continue
} }
if (
/(?:^|\/)node_modules\/mathjax-full\/(?:components|es5|ts)(?:\/|$)/u.test(
relativePath.toLowerCase()
)
) {
violations.push(`${relativePath}: removable MathJax distribution`)
continue
}
if (
/(?:^|\/)node_modules\/pdf-parse\/lib\/pdf\.js\/(?:v1\.9\.426|v1\.10\.88|v2\.0\.550)(?:\/|$)/u.test(
relativePath.toLowerCase()
)
) {
violations.push(`${relativePath}: unused PDF.js version`)
continue
}
if (/(?:^|\/)node_modules\/exceljs\/dist(?:\/|$)/u.test(relativePath.toLowerCase())) {
violations.push(`${relativePath}: bundled ExcelJS browser distribution`)
continue
}
if (
process.platform !== 'win32' &&
/(?:^|\/)node_modules\/node-pty\/third_party(?:\/|$)/u.test(relativePath.toLowerCase())
) {
violations.push(`${relativePath}: Windows ConPTY sources on a non-Windows target`)
continue
}
if (FORBIDDEN_ROOT_SEGMENTS.has(lowerSegments[0])) { if (FORBIDDEN_ROOT_SEGMENTS.has(lowerSegments[0])) {
violations.push(`${relativePath}: forbidden project directory ${lowerSegments[0]}`) violations.push(`${relativePath}: forbidden project directory ${lowerSegments[0]}`)
continue continue
@@ -217,6 +245,22 @@ export const assertWebCrawlerSidecarPackage = (files, sourceLabel) => {
if (sidecarFiles.length === 0) { if (sidecarFiles.length === 0) {
throw new Error(`${sourceLabel} is missing the web crawler sidecar`) throw new Error(`${sourceLabel} is missing the web crawler sidecar`)
} }
const bundledNode = sidecarFiles.find((item) => {
const relativePath = normalizeRelativePath(item.relativePath).toLowerCase()
return /\/playwright\/driver\/node(?:\.exe)?$/u.test(relativePath)
})
if (bundledNode) {
throw new Error(
`${sourceLabel} includes a bundled Playwright Node runtime: ${bundledNode.relativePath}`
)
}
const playwrightDriver = sidecarFiles.find((item) => {
const relativePath = normalizeRelativePath(item.relativePath).toLowerCase()
return /\/playwright\/driver\/package\/cli\.js$/u.test(relativePath)
})
if (!playwrightDriver) {
throw new Error(`${sourceLabel} is missing the Python Playwright driver JavaScript`)
}
const bytes = sidecarFiles.reduce((sum, item) => sum + item.size, 0) const bytes = sidecarFiles.reduce((sum, item) => sum + item.size, 0)
if (bytes > WEB_CRAWLER_SIDECAR_HARD_LIMIT_BYTES) { if (bytes > WEB_CRAWLER_SIDECAR_HARD_LIMIT_BYTES) {
throw new Error( throw new Error(
+6 -3
View File
@@ -8,6 +8,7 @@ import { collectRuntimePackageNames } from '../runtime/bundle-graph.mjs'
import { traceRuntimeDependencies, writeRuntimeTraceReport } from '../runtime/trace.mjs' import { traceRuntimeDependencies, writeRuntimeTraceReport } from '../runtime/trace.mjs'
import { verifyStagedRuntime } from '../runtime/verify-stage.mjs' import { verifyStagedRuntime } from '../runtime/verify-stage.mjs'
import { obfuscatePackageMain, verifyPackageMainObfuscation } from './obfuscation.mjs' import { obfuscatePackageMain, verifyPackageMainObfuscation } from './obfuscation.mjs'
import { pruneRuntimePackages } from './prune-runtime.mjs'
import { installWindowsVcRuntime } from './vc-runtime.mjs' import { installWindowsVcRuntime } from './vc-runtime.mjs'
import { import {
assertNodePtyNativeReleaseReady, assertNodePtyNativeReleaseReady,
@@ -299,7 +300,7 @@ const stripSourceMapReferences = () => {
walk(STAGE_DIR) walk(STAGE_DIR)
} }
const pruneNodeModules = () => { const pruneNodeModules = (runtimeTrace) => {
const nodeModules = path.join(STAGE_DIR, 'node_modules') const nodeModules = path.join(STAGE_DIR, 'node_modules')
const targetPrebuildDirectory = `${process.platform}-${process.arch}` const targetPrebuildDirectory = `${process.platform}-${process.arch}`
const isRequiredRuntimeDirectory = (directory) => const isRequiredRuntimeDirectory = (directory) =>
@@ -399,6 +400,8 @@ const pruneNodeModules = () => {
} }
} }
} }
return pruneRuntimePackages({ nodeModulesDirectory: nodeModules, runtimeTrace })
} }
const pruneStagedResources = () => { const pruneStagedResources = () => {
@@ -512,7 +515,7 @@ const main = async () => {
if (runtimePackageNames.includes('node-pty')) installPreparedNodePtyNative() if (runtimePackageNames.includes('node-pty')) installPreparedNodePtyNative()
fs.rmSync(path.join(STAGE_DIR, 'pnpm-lock.yaml'), { force: true }) fs.rmSync(path.join(STAGE_DIR, 'pnpm-lock.yaml'), { force: true })
pruneCompiledOutput() pruneCompiledOutput()
pruneNodeModules() const runtimePruning = pruneNodeModules(runtimeTrace)
installWindowsVcRuntime({ stageDir: STAGE_DIR, vcRedistDir: VC_REDIST_DIR }) installWindowsVcRuntime({ stageDir: STAGE_DIR, vcRedistDir: VC_REDIST_DIR })
pruneStagedResources() pruneStagedResources()
const packagedDependencies = collectInstalledTopLevelDependencies() const packagedDependencies = collectInstalledTopLevelDependencies()
@@ -545,7 +548,7 @@ const main = async () => {
if (audit.status !== 0) throw new Error('staging package security audit failed') if (audit.status !== 0) throw new Error('staging package security audit failed')
process.stdout.write( process.stdout.write(
`[package] staging ready for ${process.platform}-${process.arch}; runtime roots: ${runtimePackageNames.join(', ')}; packaged dependencies: ${String(Object.keys(packagedDependencies).length)}; loader smoke: ${String(runtimeVerification.docxDocuments)} DOCX / ${String(runtimeVerification.pdfDocuments)} PDF document(s); obfuscated main: ${String(obfuscation.totals.files)} file(s)\n` `[package] staging ready for ${process.platform}-${process.arch}; runtime roots: ${runtimePackageNames.join(', ')}; packaged dependencies: ${String(Object.keys(packagedDependencies).length)}; targeted pruning: ${(runtimePruning.totalBytes / 1024 / 1024).toFixed(1)} MiB; runtime smoke: ${String(runtimeVerification.docxDocuments)} DOCX / ${String(runtimeVerification.pdfDocuments)} PDF document(s), MathJax SVG, ExcelJS round trip; obfuscated main: ${String(obfuscation.totals.files)} file(s)\n`
) )
} }
+124
View File
@@ -0,0 +1,124 @@
import fs from 'node:fs'
import path from 'node:path'
const normalizePath = (value) => String(value).replaceAll('\\', '/')
const collectPackageTracePaths = (runtimeTrace, packageName) => {
const packageSegments = packageName.split('/')
const result = new Set()
for (const rawPath of runtimeTrace.trace?.files || []) {
const segments = normalizePath(rawPath).split('/')
for (let index = 0; index < segments.length; index += 1) {
if (segments[index] !== 'node_modules') continue
const packageStart = index + 1
if (
packageSegments.every(
(segment, segmentIndex) => segments[packageStart + segmentIndex] === segment
)
) {
result.add(segments.slice(packageStart + packageSegments.length).join('/'))
}
}
}
return [...result].sort()
}
const assertTracedPackage = (runtimeTrace, packageName) => {
if (!(runtimeTrace.packages || []).some((item) => item.name === packageName)) {
throw new Error(`runtime pruning requires traced package ${packageName}`)
}
const paths = collectPackageTracePaths(runtimeTrace, packageName)
if (paths.length === 0) {
throw new Error(`runtime pruning has no traced files for ${packageName}`)
}
return paths
}
export const assertRuntimePruningTrace = (runtimeTrace, platform = process.platform) => {
const mathJaxPaths = assertTracedPackage(runtimeTrace, 'mathjax-full')
const unsafeMathJaxPath = mathJaxPaths.find(
(relativePath) => relativePath !== 'package.json' && !relativePath.startsWith('js/')
)
if (unsafeMathJaxPath) {
throw new Error(`mathjax-full trace reaches a removable path: ${unsafeMathJaxPath}`)
}
const pdfParsePaths = assertTracedPackage(runtimeTrace, 'pdf-parse')
const unsafePdfVersion = pdfParsePaths.find((relativePath) => {
const match = relativePath.match(/^lib\/pdf\.js\/(v[^/]+)(?:\/|$)/u)
return match && match[1] !== 'v1.10.100'
})
if (unsafePdfVersion) {
throw new Error(`pdf-parse trace reaches a removable PDF.js version: ${unsafePdfVersion}`)
}
const excelJsPaths = assertTracedPackage(runtimeTrace, 'exceljs')
const unsafeExcelJsPath = excelJsPaths.find(
(relativePath) => relativePath === 'dist' || relativePath.startsWith('dist/')
)
if (unsafeExcelJsPath) {
throw new Error(`exceljs trace reaches removable browser dist: ${unsafeExcelJsPath}`)
}
if (platform !== 'win32') {
const nodePtyPaths = assertTracedPackage(runtimeTrace, 'node-pty')
const unsafeNodePtyPath = nodePtyPaths.find(
(relativePath) => relativePath === 'third_party' || relativePath.startsWith('third_party/')
)
if (unsafeNodePtyPath) {
throw new Error(`node-pty trace reaches removable Windows sources: ${unsafeNodePtyPath}`)
}
}
}
const directorySize = (targetPath) => {
if (!fs.existsSync(targetPath)) return 0
const stat = fs.lstatSync(targetPath)
if (!stat.isDirectory()) return stat.size
return fs
.readdirSync(targetPath)
.reduce((total, name) => total + directorySize(path.join(targetPath, name)), 0)
}
export const pruneRuntimePackages = ({
nodeModulesDirectory,
runtimeTrace,
platform = process.platform
}) => {
assertRuntimePruningTrace(runtimeTrace, platform)
const targets = [
['mathjax-full/es5', path.join(nodeModulesDirectory, 'mathjax-full', 'es5')],
['mathjax-full/components', path.join(nodeModulesDirectory, 'mathjax-full', 'components')],
['mathjax-full/ts', path.join(nodeModulesDirectory, 'mathjax-full', 'ts')],
[
'pdf-parse PDF.js v1.9.426',
path.join(nodeModulesDirectory, 'pdf-parse', 'lib', 'pdf.js', 'v1.9.426')
],
[
'pdf-parse PDF.js v1.10.88',
path.join(nodeModulesDirectory, 'pdf-parse', 'lib', 'pdf.js', 'v1.10.88')
],
[
'pdf-parse PDF.js v2.0.550',
path.join(nodeModulesDirectory, 'pdf-parse', 'lib', 'pdf.js', 'v2.0.550')
],
['exceljs/dist', path.join(nodeModulesDirectory, 'exceljs', 'dist')]
]
if (platform !== 'win32') {
targets.push([
'node-pty Windows ConPTY',
path.join(nodeModulesDirectory, 'node-pty', 'third_party')
])
}
const removed = targets.map(([label, targetPath]) => {
const bytes = directorySize(targetPath)
fs.rmSync(targetPath, { recursive: true, force: true })
return { label, bytes }
})
return {
removed,
totalBytes: removed.reduce((total, item) => total + item.bytes, 0)
}
}
+54
View File
@@ -21,6 +21,12 @@ const loadStageExport = async (stageRequire, stageNodeModules, specifier, export
return value return value
} }
const loadStageModule = (stageRequire, stageNodeModules, specifier) => {
const resolved = stageRequire.resolve(specifier)
assertInside(stageNodeModules, resolved, specifier)
return stageRequire(resolved)
}
export const verifyStagedRuntime = async ({ stageDirectory, reportPath, rootDirectory }) => { export const verifyStagedRuntime = async ({ stageDirectory, reportPath, rootDirectory }) => {
const stageRoot = path.resolve(stageDirectory) const stageRoot = path.resolve(stageDirectory)
const root = path.resolve(rootDirectory) const root = path.resolve(rootDirectory)
@@ -116,11 +122,59 @@ export const verifyStagedRuntime = async ({ stageDirectory, reportPath, rootDire
assertDocuments(docxDocuments, 'DOCX') assertDocuments(docxDocuments, 'DOCX')
assertDocuments(pdfDocuments, 'PDF') assertDocuments(pdfDocuments, 'PDF')
const { mathjax } = loadStageModule(stageRequire, stageNodeModules, 'mathjax-full/js/mathjax.js')
const { TeX } = loadStageModule(stageRequire, stageNodeModules, 'mathjax-full/js/input/tex.js')
const { AllPackages } = loadStageModule(
stageRequire,
stageNodeModules,
'mathjax-full/js/input/tex/AllPackages.js'
)
const { SVG } = loadStageModule(stageRequire, stageNodeModules, 'mathjax-full/js/output/svg.js')
const { liteAdaptor } = loadStageModule(
stageRequire,
stageNodeModules,
'mathjax-full/js/adaptors/liteAdaptor.js'
)
const { RegisterHTMLHandler } = loadStageModule(
stageRequire,
stageNodeModules,
'mathjax-full/js/handlers/html.js'
)
const adaptor = liteAdaptor()
RegisterHTMLHandler(adaptor)
const mathDocument = mathjax.document('', {
InputJax: new TeX({ packages: AllPackages }),
OutputJax: new SVG({ fontCache: 'none' })
})
const formulaMarkup = adaptor.outerHTML(
mathDocument.convert('x^2 + y^2 = z^2', {
display: true,
em: 16,
ex: 8,
containerWidth: 1200
})
)
if (!formulaMarkup.includes('<svg') || !formulaMarkup.includes('</svg>')) {
throw new Error('MathJax staged runtime returned no SVG')
}
const ExcelJS = loadStageModule(stageRequire, stageNodeModules, 'exceljs')
const workbook = new ExcelJS.Workbook()
workbook.addWorksheet('Smoke').getCell('A1').value = 'NCurator'
const workbookBytes = await workbook.xlsx.writeBuffer()
const loadedWorkbook = new ExcelJS.Workbook()
await loadedWorkbook.xlsx.load(workbookBytes)
if (loadedWorkbook.getWorksheet('Smoke')?.getCell('A1').value !== 'NCurator') {
throw new Error('ExcelJS staged runtime round trip failed')
}
return { return {
packages: report.packages.length, packages: report.packages.length,
requiredFiles: report.requiredFiles.length, requiredFiles: report.requiredFiles.length,
docxDocuments: docxDocuments.length, docxDocuments: docxDocuments.length,
pdfDocuments: pdfDocuments.length, pdfDocuments: pdfDocuments.length,
formulaSvgBytes: Buffer.byteLength(formulaMarkup),
excelWorkbookBytes: workbookBytes.byteLength,
suppressedPdfWarnings suppressedPdfWarnings
} }
} }
+16
View File
@@ -50,6 +50,21 @@ def copy_playwright_driver_bundle(sidecar_dist_dir: Path) -> None:
ignore=shutil.ignore_patterns(".local-browsers"), ignore=shutil.ignore_patterns(".local-browsers"),
) )
bundled_node = target / (
"node.exe" if platform.system().lower().startswith("windows") else "node"
)
if bundled_node.exists():
bundled_node.unlink()
def assert_no_playwright_bundled_node(sidecar_dist_dir: Path) -> None:
driver_dir = sidecar_dist_dir / "_internal" / "playwright" / "driver"
bundled_nodes = [driver_dir / "node", driver_dir / "node.exe"]
forbidden = [candidate for candidate in bundled_nodes if candidate.exists()]
if forbidden:
paths = ", ".join(str(item.relative_to(sidecar_dist_dir)) for item in forbidden)
raise RuntimeError(f"Playwright must reuse the Electron Node runtime: {paths}")
def assert_no_playwright_browser_bundle(sidecar_dist_dir: Path) -> None: def assert_no_playwright_browser_bundle(sidecar_dist_dir: Path) -> None:
forbidden = [] forbidden = []
@@ -148,6 +163,7 @@ def main() -> int:
copy_playwright_driver_bundle(sidecar_dist_dir) copy_playwright_driver_bundle(sidecar_dist_dir)
prune_non_runtime_files(sidecar_dist_dir) prune_non_runtime_files(sidecar_dist_dir)
assert_no_playwright_browser_bundle(sidecar_dist_dir) assert_no_playwright_browser_bundle(sidecar_dist_dir)
assert_no_playwright_bundled_node(sidecar_dist_dir)
return 0 return 0
+2 -3
View File
@@ -76,6 +76,7 @@ export const buildBrainSystemPrompt = (input: { ctx: AiRunContext }): string =>
<terminal_wait_contract> <terminal_wait_contract>
- 当 terminal_run 返回命令仍在执行时,多尝试等待终端命令完成后(再读取结果或继续),不要在命令未完成时空转轮询;仅当判断等待不合理时才中断并分析原因。 - 当 terminal_run 返回命令仍在执行时,多尝试等待终端命令完成后(再读取结果或继续),不要在命令未完成时空转轮询;仅当判断等待不合理时才中断并分析原因。
- 当需要等待操作时,使用终端指令来进行等待。
</terminal_wait_contract> </terminal_wait_contract>
<calculation_contract> <calculation_contract>
@@ -123,9 +124,7 @@ export const buildBrainUserMessage = (input: {
: '' : ''
const renderActiveState = (): string => { const renderActiveState = (): string => {
const skillsLine = const skillsLine =
learnedSkillIds.length > 0 learnedSkillIds.length > 0 ? `已学习技能:${learnedSkillIds.join('、')}` : '已学习技能:无'
? `已学习技能:${learnedSkillIds.join('、')}`
: '已学习技能:无'
const planLine = const planLine =
input.injectPlan && plan.length > 0 input.injectPlan && plan.length > 0
? `计划进度:${plan ? `计划进度:${plan
@@ -32,6 +32,7 @@ type PendingRequest = {
export type WebCrawlerServiceOptions = { export type WebCrawlerServiceOptions = {
sidecarCommand?: SidecarCommand sidecarCommand?: SidecarCommand
sidecarEnv?: NodeJS.ProcessEnv sidecarEnv?: NodeJS.ProcessEnv
playwrightNodeExecutablePath?: string
resolveCdpConnection?: () => Promise<WebCrawlerCdpConnection> resolveCdpConnection?: () => Promise<WebCrawlerCdpConnection>
appRootPath?: string appRootPath?: string
isPackaged?: boolean isPackaged?: boolean
@@ -45,6 +46,8 @@ const PACKAGED_SIDECAR_BIN_DIR = 'bin'
const PYTHON_SIDECAR_DEBUG_ENV = 'NCURATOR_WEB_CRAWLER_USE_PYTHON_SIDECAR' const PYTHON_SIDECAR_DEBUG_ENV = 'NCURATOR_WEB_CRAWLER_USE_PYTHON_SIDECAR'
const SIDECAR_CDP_URL_ENV = 'NCURATOR_WEB_CRAWLER_CDP_URL' const SIDECAR_CDP_URL_ENV = 'NCURATOR_WEB_CRAWLER_CDP_URL'
const SIDECAR_CDP_CONTEXT_MARKER_ENV = 'NCURATOR_WEB_CRAWLER_CDP_CONTEXT_MARKER_URL' const SIDECAR_CDP_CONTEXT_MARKER_ENV = 'NCURATOR_WEB_CRAWLER_CDP_CONTEXT_MARKER_URL'
const PLAYWRIGHT_NODEJS_PATH_ENV = 'PLAYWRIGHT_NODEJS_PATH'
const ELECTRON_RUN_AS_NODE_ENV = 'ELECTRON_RUN_AS_NODE'
const LOOPBACK_HOSTNAMES = new Set(['127.0.0.1', 'localhost', '::1', '[::1]']) const LOOPBACK_HOSTNAMES = new Set(['127.0.0.1', 'localhost', '::1', '[::1]'])
const DEV_SIDECAR_SCRIPT_PATH = path.join( const DEV_SIDECAR_SCRIPT_PATH = path.join(
'resources', 'resources',
@@ -437,6 +440,7 @@ const buildSidecarError = (response: WebCrawlerSidecarResponse): Error => {
export class ManagedWebCrawlerService implements WebCrawlerServiceContract { export class ManagedWebCrawlerService implements WebCrawlerServiceContract {
private readonly sidecarCommand: SidecarCommand private readonly sidecarCommand: SidecarCommand
private readonly playwrightNodeExecutablePath: string
private readonly resolveCdpConnection?: () => Promise<WebCrawlerCdpConnection> private readonly resolveCdpConnection?: () => Promise<WebCrawlerCdpConnection>
private readonly requestTimeoutMs: number private readonly requestTimeoutMs: number
private child?: ChildProcessWithoutNullStreams private child?: ChildProcessWithoutNullStreams
@@ -448,6 +452,13 @@ export class ManagedWebCrawlerService implements WebCrawlerServiceContract {
public constructor(options: WebCrawlerServiceOptions = {}) { public constructor(options: WebCrawlerServiceOptions = {}) {
this.sidecarCommand = options.sidecarCommand || resolveDefaultSidecarCommand(options) this.sidecarCommand = options.sidecarCommand || resolveDefaultSidecarCommand(options)
this.playwrightNodeExecutablePath = path.resolve(
options.playwrightNodeExecutablePath || process.execPath
)
assertExecutableExists(
this.playwrightNodeExecutablePath,
'missing Node runtime for Python Playwright'
)
this.resolveCdpConnection = options.resolveCdpConnection this.resolveCdpConnection = options.resolveCdpConnection
this.requestTimeoutMs = Math.max( this.requestTimeoutMs = Math.max(
1_000, 1_000,
@@ -523,6 +534,8 @@ export class ManagedWebCrawlerService implements WebCrawlerServiceContract {
env: { env: {
...process.env, ...process.env,
...this.sidecarCommand.env, ...this.sidecarCommand.env,
[PLAYWRIGHT_NODEJS_PATH_ENV]: this.playwrightNodeExecutablePath,
[ELECTRON_RUN_AS_NODE_ENV]: '1',
[SIDECAR_CDP_URL_ENV]: cdpUrl, [SIDECAR_CDP_URL_ENV]: cdpUrl,
[SIDECAR_CDP_CONTEXT_MARKER_ENV]: contextMarkerUrl [SIDECAR_CDP_CONTEXT_MARKER_ENV]: contextMarkerUrl
}, },
@@ -7,6 +7,7 @@ import { createPackage } from '@electron/asar'
import { afterEach, describe, expect, it } from 'vitest' import { afterEach, describe, expect, it } from 'vitest'
import { import {
assertSafePackagePaths, assertSafePackagePaths,
assertWebCrawlerSidecarPackage,
normalizeRelativePath normalizeRelativePath
} from '../../../../scripts/build/package/content-policy.mjs' } from '../../../../scripts/build/package/content-policy.mjs'
import { PROTECTED_MAIN_PLAINTEXT_MARKERS } from '../../../../scripts/build/package/obfuscation.mjs' import { PROTECTED_MAIN_PLAINTEXT_MARKERS } from '../../../../scripts/build/package/obfuscation.mjs'
@@ -31,12 +32,49 @@ describe('package content policy paths', () => {
}) })
it('permits ExcelJS runtime modules while rejecting other doc directories', () => { it('permits ExcelJS runtime modules while rejecting other doc directories', () => {
expect(() => assertSafePackagePaths(['node_modules/exceljs/lib/doc/workbook.js'])).not.toThrow()
expect(() => expect(() =>
assertSafePackagePaths(['node_modules/exceljs/lib/doc/workbook.js']) assertSafePackagePaths(['node_modules/runtime-package/lib/doc/readme.js'])
).not.toThrow() ).toThrow('forbidden directory doc')
expect(() => assertSafePackagePaths(['node_modules/runtime-package/lib/doc/readme.js'])).toThrow( })
'forbidden directory doc'
it('rejects runtime directories covered by deterministic package pruning', () => {
expect(() => assertSafePackagePaths(['node_modules/mathjax-full/es5/tex-svg.js'])).toThrow(
'removable MathJax distribution'
) )
expect(() =>
assertSafePackagePaths(['node_modules/pdf-parse/lib/pdf.js/v2.0.550/build/pdf.js'])
).toThrow('unused PDF.js version')
expect(() => assertSafePackagePaths(['node_modules/exceljs/dist/exceljs.js'])).toThrow(
'bundled ExcelJS browser distribution'
)
})
it('requires Playwright driver JavaScript and rejects its bundled Node runtime', () => {
const driver = {
relativePath:
'resources/web_crawler_sidecar/bin/app/_internal/playwright/driver/package/cli.js',
size: 10
}
expect(assertWebCrawlerSidecarPackage([driver], 'fixture')).toBe(10)
expect(() =>
assertWebCrawlerSidecarPackage(
[
driver,
{
relativePath: 'resources/web_crawler_sidecar/bin/app/_internal/playwright/driver/node',
size: 100
}
],
'fixture'
)
).toThrow('bundled Playwright Node runtime')
expect(() =>
assertWebCrawlerSidecarPackage(
[{ relativePath: 'resources/web_crawler_sidecar/bin/app/app', size: 10 }],
'fixture'
)
).toThrow('missing the Python Playwright driver JavaScript')
}) })
it('audits nested files in an asar archive on the current platform', async () => { it('audits nested files in an asar archive on the current platform', async () => {
@@ -54,10 +92,32 @@ describe('package content policy paths', () => {
'Release', 'Release',
'spawn-helper' 'spawn-helper'
) )
const sidecarExecutablePath = path.join(
packagedApp,
'resources',
'web_crawler_sidecar',
'bin',
'web_crawler_sidecar'
)
const playwrightDriverPath = path.join(
packagedApp,
'resources',
'web_crawler_sidecar',
'bin',
'_internal',
'playwright',
'driver',
'package',
'cli.js'
)
await mkdir(path.join(appSource, 'out', 'main'), { recursive: true }) await mkdir(path.join(appSource, 'out', 'main'), { recursive: true })
await mkdir(path.dirname(archivePath), { recursive: true }) await mkdir(path.dirname(archivePath), { recursive: true })
await mkdir(path.dirname(nodePtyHelperPath), { recursive: true }) await mkdir(path.dirname(nodePtyHelperPath), { recursive: true })
await mkdir(path.dirname(sidecarExecutablePath), { recursive: true })
await mkdir(path.dirname(playwrightDriverPath), { recursive: true })
await writeFile(nodePtyHelperPath, '', { mode: 0o755 }) await writeFile(nodePtyHelperPath, '', { mode: 0o755 })
await writeFile(sidecarExecutablePath, '', { mode: 0o755 })
await writeFile(playwrightDriverPath, 'module.exports = {}\n')
const vcRuntimeDllDirectory = resolveVcRuntimeDllDirectory(packagedApp) const vcRuntimeDllDirectory = resolveVcRuntimeDllDirectory(packagedApp)
await mkdir(vcRuntimeDllDirectory, { recursive: true }) await mkdir(vcRuntimeDllDirectory, { recursive: true })
for (const filename of VC_REDIST_DLL_NAMES) { for (const filename of VC_REDIST_DLL_NAMES) {
@@ -49,10 +49,10 @@ describe('package size budget', () => {
const config = loadPackageSizeBudgetConfig(path.resolve('build/package-size-budgets.json')) const config = loadPackageSizeBudgetConfig(path.resolve('build/package-size-budgets.json'))
expect(config.platforms['darwin-arm64']).toMatchObject({ expect(config.platforms['darwin-arm64']).toMatchObject({
installedApp: { baselineMiB: 1113.3, hardLimitMiB: 1200 }, installedApp: { baselineMiB: 776.2, hardLimitMiB: 870 },
appAsar: { baselineMiB: 230.9, hardLimitMiB: 252 }, appAsar: { baselineMiB: 184.0, hardLimitMiB: 205 },
dmg: { baselineMiB: 432.9, hardLimitMiB: 480 }, dmg: { baselineMiB: 273.9, hardLimitMiB: 310 },
zip: { baselineMiB: 418.5, hardLimitMiB: 465 } zip: { baselineMiB: 289.1, hardLimitMiB: 325 }
}) })
expect(config.platforms['darwin-x64']).toBeUndefined() expect(config.platforms['darwin-x64']).toBeUndefined()
}) })
@@ -0,0 +1,109 @@
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
import fs from 'node:fs'
import os from 'node:os'
import path from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
import {
assertRuntimePruningTrace,
pruneRuntimePackages
} from '../../../../scripts/build/package/prune-runtime.mjs'
let temporaryDirectory = ''
afterEach(async () => {
if (temporaryDirectory) await rm(temporaryDirectory, { recursive: true, force: true })
temporaryDirectory = ''
})
const runtimeTrace = {
packages: [
{ name: 'mathjax-full' },
{ name: 'pdf-parse' },
{ name: 'exceljs' },
{ name: 'node-pty' }
],
trace: {
files: [
'node_modules/.pnpm/mathjax-full@3.2.2/node_modules/mathjax-full/js/mathjax.js',
'node_modules/.pnpm/mathjax-full@3.2.2/node_modules/mathjax-full/package.json',
'node_modules/.pnpm/pdf-parse@1.1.1/node_modules/pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js',
'node_modules/.pnpm/exceljs@4.4.0/node_modules/exceljs/lib/doc/workbook.js',
'node_modules/.pnpm/node-pty@1.2.0/node_modules/node-pty/lib/index.js'
]
}
}
const writeFixture = async (relativePath: string): Promise<void> => {
const targetPath = path.join(temporaryDirectory, relativePath)
await mkdir(path.dirname(targetPath), { recursive: true })
await writeFile(targetPath, 'runtime-fixture')
}
describe('targeted runtime package pruning', () => {
it('removes only approved redundant distributions on non-Windows targets', async () => {
temporaryDirectory = await mkdtemp(path.join(os.tmpdir(), 'ncurator-runtime-pruning-'))
const removable = [
'mathjax-full/es5/tex-svg.js',
'mathjax-full/components/src/component.js',
'mathjax-full/ts/input/tex.ts',
'pdf-parse/lib/pdf.js/v1.9.426/build/pdf.js',
'pdf-parse/lib/pdf.js/v1.10.88/build/pdf.js',
'pdf-parse/lib/pdf.js/v2.0.550/build/pdf.js',
'exceljs/dist/exceljs.js',
'node-pty/third_party/conpty/conpty.dll'
]
await Promise.all(
[
...removable,
'mathjax-full/js/mathjax.js',
'pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js'
].map(writeFixture)
)
const result = pruneRuntimePackages({
nodeModulesDirectory: temporaryDirectory,
runtimeTrace,
platform: 'darwin'
})
for (const relativePath of removable) {
expect(fs.existsSync(path.join(temporaryDirectory, relativePath))).toBe(false)
}
expect(fs.existsSync(path.join(temporaryDirectory, 'mathjax-full/js/mathjax.js'))).toBe(true)
expect(
fs.existsSync(path.join(temporaryDirectory, 'pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js'))
).toBe(true)
expect(result.totalBytes).toBeGreaterThan(0)
})
it('fails before pruning when trace reaches a removable distribution', () => {
for (const unsafePath of [
'node_modules/mathjax-full/es5/tex-svg.js',
'node_modules/pdf-parse/lib/pdf.js/v2.0.550/build/pdf.js',
'node_modules/exceljs/dist/exceljs.js',
'node_modules/node-pty/third_party/conpty/conpty.dll'
]) {
expect(() =>
assertRuntimePruningTrace({
...runtimeTrace,
trace: { files: [...runtimeTrace.trace.files, unsafePath] }
})
).toThrow('trace reaches')
}
})
it('retains node-pty ConPTY on Windows targets', async () => {
temporaryDirectory = await mkdtemp(path.join(os.tmpdir(), 'ncurator-runtime-pruning-'))
await writeFixture('node-pty/third_party/conpty/conpty.dll')
pruneRuntimePackages({
nodeModulesDirectory: temporaryDirectory,
runtimeTrace,
platform: 'win32'
})
expect(
fs.existsSync(path.join(temporaryDirectory, 'node-pty/third_party/conpty/conpty.dll'))
).toBe(true)
})
})
@@ -8,6 +8,9 @@ import { CdpEndpointService } from '../../../src/main/services/browser/CdpEndpoi
import { ManagedWebCrawlerService } from '../../../src/main/services/webCrawler/WebCrawlerService' import { ManagedWebCrawlerService } from '../../../src/main/services/webCrawler/WebCrawlerService'
const electronExecutablePath = createRequire(import.meta.url)('electron') as string const electronExecutablePath = createRequire(import.meta.url)('electron') as string
const playwrightNodeExecutablePath = path.resolve(
String(process.env.NCURATOR_TEST_PLAYWRIGHT_NODEJS_PATH || electronExecutablePath)
)
const stopProcess = async (child: ChildProcess): Promise<void> => { const stopProcess = async (child: ChildProcess): Promise<void> => {
if (child.exitCode !== null || child.signalCode !== null) return if (child.exitCode !== null || child.signalCode !== null) return
@@ -63,6 +66,7 @@ export const createWebCrawlerService = async (): Promise<ManagedWebCrawlerServic
const service = new ManagedWebCrawlerService({ const service = new ManagedWebCrawlerService({
appRootPath: process.cwd(), appRootPath: process.cwd(),
isPackaged: false, isPackaged: false,
playwrightNodeExecutablePath,
sidecarEnv: { sidecarEnv: {
CRAWL4_AI_BASE_DIRECTORY: baseDirectory CRAWL4_AI_BASE_DIRECTORY: baseDirectory
}, },