From cb36f9dc22d456d171eb9d6080fd0e397f2e32fe Mon Sep 17 00:00:00 2001 From: hy Date: Tue, 16 Jun 2026 12:29:46 +0800 Subject: [PATCH] feat: enhance E2E tests with new session creation and approval batch decision handling --- doc/test-plan/workbench-test.md | 12 +- .../blocker/approval-batch-flow.e2e.test.ts | 102 +++++- .../browser-use-task-panel.e2e.test.ts | 134 +++++++- tests/harness/core/write-run-log.ts | 2 +- tests/harness/e2e/electronApp.ts | 72 ++++- tests/harness/e2e/run-e2e-case.ts | 295 ++++++++++++++---- tests/harness/e2e/workbench.ts | 98 +++++- 7 files changed, 595 insertions(+), 120 deletions(-) diff --git a/doc/test-plan/workbench-test.md b/doc/test-plan/workbench-test.md index 7b900e6c..be14b7e3 100644 --- a/doc/test-plan/workbench-test.md +++ b/doc/test-plan/workbench-test.md @@ -15,8 +15,8 @@ ### 2.1 当前状态 - 总体登记：`13` 个 e2e case。 -- 当前有效结论：`10 / 13` 有效通过。 -- 当前实现状态：`tests/case/workbench/**` 已按 suite / group 落地 `13` 个独立 `.e2e.test.ts` 文件；当前已有 `10 / 13` 形成真实有效通过结论。 +- 当前有效结论：`11 / 13` 有效通过。 +- 当前实现状态：`tests/case/workbench/**` 已按 suite / group 落地 `13` 个独立 `.e2e.test.ts` 文件；当前已有 `11 / 13` 形成真实有效通过结论。 ### 2.2 按 Suite 汇总 @@ -27,9 +27,9 @@ - 备注：case 已实现；`session-initial-create-and-title`、`send-guard-mode-switch-and-preferences`、`workspace-root-path-required`、`reference-token-lifecycle` 均已通过真实 Electron UI；其中涉及 AI 的 case 使用真实模型，`reference-token-lifecycle` 不启动 AI。 - `run` - 登记 case：`4` - - 有效通过：`1 / 4` - - 检测时间：`2026-06-16 10:32:04 +0800` - - 备注：case 已实现；`ai-process-stream-and-final-answer` 已通过真实 Electron UI、真实模型、真实 workspace 索引和真实 AI run。 + - 有效通过：`2 / 4` + - 检测时间：`2026-06-16 12:04:17 +0800` + - 备注：case 已实现；`ai-process-stream-and-final-answer`、`approval-batch-flow` 已通过真实 Electron UI、真实模型、真实 workspace 索引和真实 AI run；`stop-running-or-approval-run` 当前暴露审批等待时缺少停止入口的生产行为缺口；`browser-use-task-panel` 当前暴露 `browser_use` / Stagehand 对当前 live 模型接口返回 `Bad Request` 的生产或配置阻塞。 - `result` - 登记 case：`5` - 有效通过：`5 / 5` @@ -44,7 +44,7 @@ - Workbench formal e2e 需要真实索引完成信号和 UI 可见状态作为等待条件；不得再读取内部 store 或注入 fixed document id 来制造搜索结果。 - `file-preview-actions` 当前按生产真实索引能力覆盖 `pdf/docx`：`src/main/services/ingestion/loader.ts` 只支持 `.pdf`、`.docx` 本地知识库索引，`tests/fixtures/workspace/workbench-preview.md` 无法通过真实搜索结果打开；如 PRD 需要 markdown 也进入 Workbench 搜索 / 预览链路，需要产品 / 生产实现决策。 - docx apply case 需要真实 docx 输入、真实 AI 编辑、真实写回记录、真实预览和真实回退。 -- `browser-use-task-panel` 需要真实 AI run 触发 `browser_use` 并驱动内置浏览器完成本地网页任务。 +- `browser-use-task-panel` 需要真实 AI run 触发 `browser_use` 并驱动内置浏览器完成本地网页任务；当前测试已在连续两次 `browser_use` 失败时快速失败并记录诊断，避免模型无效重试导致长时间无反馈。 ## 3. Suite 设计 diff --git a/tests/case/workbench/run/blocker/approval-batch-flow.e2e.test.ts b/tests/case/workbench/run/blocker/approval-batch-flow.e2e.test.ts index ed3ed919..f046e172 100644 --- a/tests/case/workbench/run/blocker/approval-batch-flow.e2e.test.ts +++ b/tests/case/workbench/run/blocker/approval-batch-flow.e2e.test.ts @@ -1,11 +1,15 @@ import { access } from 'node:fs/promises' import path from 'node:path' import { describe, it } from 'vitest' +import type { AiRunEvent } from '../../../../../src/shared/types' import { runE2ECase } from '../../../../harness/e2e/run-e2e-case' import { WORKBENCH_LIVE_AI_E2E_TIMEOUT_MS, assertNoWorkbenchFakeSignals, chatSelectors, + collectWorkbenchAiRunEvents, + createWorkbenchNewSession, + decideWorkbenchApprovalBatch, getApprovalCard, getWorkbenchLocator, recordWorkbenchAiEvidence, @@ -16,12 +20,63 @@ import { } from '../../../../harness/e2e/workbench' const CASE_ID = 'approval-batch-flow' +const DENIED_RUN_SETTLE_TIMEOUT_MS = 120_000 const exists = async (filePath: string): Promise => await access(filePath) .then(() => true) .catch(() => false) +const isRunStartedEvent = ( + event: AiRunEvent +): event is Extract => event.type === 'run_started' + +const isApprovalBatchRequiredEvent = ( + event: AiRunEvent +): event is Extract => + event.type === 'approval_batch_required' + +const waitForDeniedRunToSettle = async ( + runtime: Parameters[0], + page: Parameters[0], + runId: string, + rejectedBatchIds: Set +): Promise => { + const deadline = Date.now() + DENIED_RUN_SETTLE_TIMEOUT_MS + for (;;) { + const events = await collectWorkbenchAiRunEvents(runtime) + const settled = events.some( + (event) => + (event.type === 'run_completed' || event.type === 'run_cancelled') && + event.runId === runId + ) + if (settled) return + + const nextApproval = events + .filter( + (event): event is Extract => + isApprovalBatchRequiredEvent(event) && + event.runId === runId && + !rejectedBatchIds.has(event.batchId) + ) + .at(-1) + if (nextApproval) { + await decideWorkbenchApprovalBatch(runtime, page, { + event: nextApproval, + action: 'reject', + timeoutMs: 30_000 + }) + rejectedBatchIds.add(nextApproval.batchId) + continue + } + + if (Date.now() >= deadline) { + throw new Error(`denied run did not settle after repeated rejects: ${runId}`) + } + await page.waitForTimeout(500) + } +} + describe('workbench run e2e', () => { it(CASE_ID, async () => { await runE2ECase({ @@ -47,8 +102,17 @@ describe('workbench run e2e', () => { page, '请使用终端命令在当前工作区创建 approval-approved.txt，内容写 approved-by-workbench-e2e。' ) - await getApprovalCard(page).waitFor({ state: 'visible' }) - await getWorkbenchLocator(page, chatSelectors.approvalBatchApproveButton).click() + const approvedApproval = await waitForWorkbenchAiEvent(runtime, { + name: 'approval_approved_batch_required', + matches: isApprovalBatchRequiredEvent + }) + if (!isApprovalBatchRequiredEvent(approvedApproval)) { + throw new Error('approval approved batch was not required') + } + await decideWorkbenchApprovalBatch(runtime, page, { + event: approvedApproval, + action: 'approve' + }) const approveEvidence = await waitForWorkbenchAiRunCompletion( runtime, approveStartedAtMs, @@ -58,17 +122,37 @@ describe('workbench run e2e', () => { await sendWorkbenchMessage( page, - '请使用终端命令在当前工作区创建 approval-denied.txt，内容写 denied-by-workbench-e2e。' + '请使用终端命令在当前工作区创建 approval-denied.txt，内容写 denied-by-workbench-e2e；如果本次操作未获授权，请不要再次申请执行，直接说明未执行。' ) - await getApprovalCard(page).waitFor({ state: 'visible' }) - await getWorkbenchLocator(page, chatSelectors.approvalBatchDenyButton).click() - await waitForWorkbenchAiEvent(runtime, { - name: 'approval_batch_decision_reject', + const deniedRunStarted = await waitForWorkbenchAiEvent(runtime, { + name: 'approval_denied_run_started', matches: (event) => - event.type === 'approval_batch_decision' && - event.decisions.some((decision) => decision.action === 'reject') + isRunStartedEvent(event) && event.input.includes('approval-denied.txt') }) + if (!isRunStartedEvent(deniedRunStarted)) { + throw new Error('approval denied run did not start') + } + const firstDeniedApproval = await waitForWorkbenchAiEvent(runtime, { + name: 'approval_denied_batch_required', + matches: (event) => + isApprovalBatchRequiredEvent(event) && event.runId === deniedRunStarted.runId + }) + if (!isApprovalBatchRequiredEvent(firstDeniedApproval)) { + throw new Error('approval denied batch was not required') + } + const rejectedBatchIds = new Set([firstDeniedApproval.batchId]) + await decideWorkbenchApprovalBatch(runtime, page, { + event: firstDeniedApproval, + action: 'reject' + }) + await waitForDeniedRunToSettle( + runtime, + page, + deniedRunStarted.runId, + rejectedBatchIds + ) + await createWorkbenchNewSession(page) await sendWorkbenchMessage( page, '请使用终端命令在当前工作区创建 approval-guided.txt，内容写 guided-by-workbench-e2e。' diff --git a/tests/case/workbench/run/browser/browser-use-task-panel.e2e.test.ts b/tests/case/workbench/run/browser/browser-use-task-panel.e2e.test.ts index f613e49c..953e03b7 100644 --- a/tests/case/workbench/run/browser/browser-use-task-panel.e2e.test.ts +++ b/tests/case/workbench/run/browser/browser-use-task-panel.e2e.test.ts @@ -1,22 +1,133 @@ import { describe, it } from 'vitest' +import type { AiRunEvent } from '../../../../../src/shared/types' import { runE2ECase } from '../../../../harness/e2e/run-e2e-case' import { WORKBENCH_LIVE_AI_E2E_TIMEOUT_MS, + type WorkbenchAiCompletionEvidence, assertNoWorkbenchFakeSignals, - chatSelectors, - getApprovalCard, + collectWorkbenchAiRunEvents, + decideWorkbenchApprovalBatch, getWorkbenchLocator, recordWorkbenchAiEvidence, sendWorkbenchMessage, setupWorkbenchWithTempWorkspace, startLocalWorkbenchHttpServer, - waitForWorkbenchAiEvent, - waitForWorkbenchAiRunCompletion, workbenchSelectors } from '../../../../harness/e2e/workbench' const CASE_ID = 'browser-use-task-panel' const TASK_CODE = 'cedar-browser-731' +const BROWSER_PANEL_WAIT_TIMEOUT_MS = 180_000 +const BROWSER_TASK_COMPLETION_TIMEOUT_MS = 240_000 +const BROWSER_USE_FAILURE_LIMIT = 2 + +const isApprovalBatchRequiredEvent = ( + event: AiRunEvent +): event is Extract => + event.type === 'approval_batch_required' + +const waitForBrowserPanelOpenedWithApproval = async ( + runtime: Parameters[0] +): Promise => { + const handledBatchIds = new Set() + const deadline = Date.now() + BROWSER_PANEL_WAIT_TIMEOUT_MS + for (;;) { + const events = await collectWorkbenchAiRunEvents(runtime) + const failed = events.find((event) => event.type === 'run_failed') + if (failed?.type === 'run_failed') { + throw new Error(`AI run failed while waiting for browser panel: ${failed.error}`) + } + if (events.some((event) => event.type === 'browser_panel_opened')) return + + const approvalRequired = events + .filter( + (event): event is Extract => + isApprovalBatchRequiredEvent(event) && !handledBatchIds.has(event.batchId) + ) + .at(-1) + if (approvalRequired) { + handledBatchIds.add(approvalRequired.batchId) + await decideWorkbenchApprovalBatch(runtime, runtime.page, { + event: approvalRequired, + action: 'approve' + }) + continue + } + + if (Date.now() >= deadline) { + throw new Error('browser panel did not open before timeout') + } + await runtime.page.waitForTimeout(500) + } +} + +const waitForBrowserTaskCompletion = async ( + runtime: Parameters[0], + startedAtMs: number, + modelName: string +): Promise => { + const deadline = Date.now() + BROWSER_TASK_COMPLETION_TIMEOUT_MS + for (;;) { + const events = await collectWorkbenchAiRunEvents(runtime) + const failed = events.find((event) => event.type === 'run_failed') + if (failed?.type === 'run_failed') { + throw new Error(`AI run failed while waiting for browser task: ${failed.error}`) + } + + const browserUseFailures = events.filter( + (event): event is Extract => + event.type === 'tool_call_result' && + event.toolName === 'browser_use' && + (Boolean(event.error) || event.result?.ok === false) + ) + if (browserUseFailures.length >= BROWSER_USE_FAILURE_LIMIT) { + const latestFailure = browserUseFailures.at(-1) + runtime.signal({ + name: 'workbench-browser-use-repeated-failure', + status: 'failed', + message: 'browser_use 连续失败，停止等待模型重复重试。', + data: { + failureCount: browserUseFailures.length, + latestFailure: latestFailure + ? { + toolCallId: latestFailure.toolCallId, + error: latestFailure.error || '', + result: latestFailure.result || null + } + : null + } + }) + throw new Error( + `browser_use failed ${browserUseFailures.length} times; latest=${JSON.stringify( + latestFailure?.result || latestFailure?.error || null + )}` + ) + } + + const runCompleted = events.find((event) => event.type === 'run_completed') + if (runCompleted?.type === 'run_completed') { + const finalAnswerText = + events + .filter( + (event): event is Extract => + event.type === 'answer_completed' + ) + .map((event) => event.text) + .at(-1) || '' + return { + modelName, + durationMs: Date.now() - startedAtMs, + events, + finalAnswerText + } + } + + if (Date.now() >= deadline) { + throw new Error('browser task did not complete before timeout') + } + await runtime.page.waitForTimeout(500) + } +} describe('workbench run e2e', () => { it(CASE_ID, async () => { @@ -50,25 +161,14 @@ describe('workbench run e2e', () => { `请使用右侧内置浏览器打开 ${server.url}，在“任务口令”输入框填入 ${TASK_CODE}，点击“提交任务”，然后告诉我页面显示的完整结果。不要只抓取源码，必须实际操作页面。` ) - const approvalVisible = await getApprovalCard(page) - .waitFor({ state: 'visible', timeout: 30_000 }) - .then(() => true) - .catch(() => false) - if (approvalVisible) { - await getWorkbenchLocator(page, chatSelectors.approvalBatchApproveButton).click() - } - - await waitForWorkbenchAiEvent(runtime, { - name: 'browser_panel_opened', - matches: (event) => event.type === 'browser_panel_opened' - }) + await waitForBrowserPanelOpenedWithApproval(runtime) await getWorkbenchLocator(page, workbenchSelectors.browserPanel).waitFor({ state: 'visible' }) await getWorkbenchLocator(page, workbenchSelectors.browserViewport).waitFor({ state: 'visible' }) - const evidence = await waitForWorkbenchAiRunCompletion( + const evidence = await waitForBrowserTaskCompletion( runtime, startedAtMs, setup.env.modelName diff --git a/tests/harness/core/write-run-log.ts b/tests/harness/core/write-run-log.ts index e250226e..fec0dcaf 100644 --- a/tests/harness/core/write-run-log.ts +++ b/tests/harness/core/write-run-log.ts @@ -73,7 +73,7 @@ const buildEmptySeedManifest = (): PersistedSeedManifest => ({ kbDocuments: [] }) -type StreamingRunLogWriter = { +export type StreamingRunLogWriter = { logDir: string paths: { metaPath: string diff --git a/tests/harness/e2e/electronApp.ts b/tests/harness/e2e/electronApp.ts index 301041d1..55f24562 100644 --- a/tests/harness/e2e/electronApp.ts +++ b/tests/harness/e2e/electronApp.ts @@ -56,6 +56,7 @@ type E2EObserverState = { type E2EWindow = Window & { __NCURATOR_E2E_OBSERVER__?: E2EObserverState __NCURATOR_E2E_ELECTRON_PROCESS_ID__?: number + __NCURATOR_E2E_RECORD_AI_RUN_EVENT__?: (entry: E2EAiRunEventEntry) => void } export type E2ERuntimeSignals = { @@ -77,6 +78,12 @@ export type LaunchedElectronApp = { export type ElectronLaunchMode = 'fixture' | 'real' export type ElectronWindowMode = 'background' | 'visible' +export type ElectronSignalSink = { + onAiRunEvent?: (entry: E2EAiRunEventEntry) => void + onConsole?: (entry: E2EConsoleEntry) => void + onLifecycle?: (entry: E2ELifecycleEntry) => void +} + const OUT_MAIN_ENTRY = path.resolve(process.cwd(), 'out', 'main', 'index.js') const toConsoleLevel = (message: ConsoleMessage): E2EConsoleLevel => { @@ -89,15 +96,18 @@ const pushProcessLog = ( entries: E2EConsoleEntry[], level: E2EConsoleLevel, source: string, - chunk: Buffer | string + chunk: Buffer | string, + sink?: ElectronSignalSink ): void => { const text = String(chunk) if (!text.trim()) return - entries.push({ + const entry = { ts: new Date().toISOString(), level, text: `[${source}] ${text.trim()}` - }) + } + entries.push(entry) + sink?.onConsole?.(entry) } const ensureBuiltAppExists = async (): Promise => { @@ -166,8 +176,13 @@ const buildElectronLaunchEnv = ( return env } -const installPageObservers = async (page: Page): Promise => { +const installPageObservers = async (page: Page, sink?: ElectronSignalSink): Promise => { await page.waitForFunction(() => Boolean(window.api)) + if (sink?.onAiRunEvent) { + await page.exposeFunction('__NCURATOR_E2E_RECORD_AI_RUN_EVENT__', (entry: E2EAiRunEventEntry) => { + sink.onAiRunEvent?.(entry) + }) + } await page.evaluate(() => { const currentWindow = window as E2EWindow @@ -202,10 +217,12 @@ const installPageObservers = async (page: Page): Promise => { window.addEventListener('ncurator-e2e-ai-run-event', (event) => { const detail = (event as CustomEvent).detail - currentWindow.__NCURATOR_E2E_OBSERVER__?.aiRunEventEntries.push({ + const entry = { ts: now(), event: detail - }) + } + currentWindow.__NCURATOR_E2E_OBSERVER__?.aiRunEventEntries.push(entry) + currentWindow.__NCURATOR_E2E_RECORD_AI_RUN_EVENT__?.(entry) }) window.addEventListener('check-llm-config', () => { @@ -231,18 +248,24 @@ const focusVisibleElectronWindow = async (electronApp: ElectronApplication): Pro } export const launchElectronApp = async ( - input: { launchMode?: ElectronLaunchMode; windowMode?: ElectronWindowMode } = {} + input: { + launchMode?: ElectronLaunchMode + windowMode?: ElectronWindowMode + signalSink?: ElectronSignalSink + } = {} ): Promise => { await ensureBuiltAppExists() const consoleEntries: E2EConsoleEntry[] = [] const lifecycleEntries: E2ELifecycleEntry[] = [] const recordLifecycle = (name: string, data?: JsonValue): void => { - lifecycleEntries.push({ + const entry = { ts: new Date().toISOString(), name, data - }) + } + lifecycleEntries.push(entry) + input.signalSink?.onLifecycle?.(entry) } const userDataDir = await mkdtemp(path.join(os.tmpdir(), 'ncurator-e2e-user-data-')) const remoteDebuggingPort = await allocateFreePort() @@ -263,10 +286,10 @@ export const launchElectronApp = async ( pid: electronProcess?.pid || null }) electronProcess?.stdout?.on('data', (chunk) => { - pushProcessLog(consoleEntries, 'log', 'main:stdout', chunk) + pushProcessLog(consoleEntries, 'log', 'main:stdout', chunk, input.signalSink) }) electronProcess?.stderr?.on('data', (chunk) => { - pushProcessLog(consoleEntries, 'error', 'main:stderr', chunk) + pushProcessLog(consoleEntries, 'error', 'main:stderr', chunk, input.signalSink) }) electronProcess?.on('exit', (code, signal) => { recordLifecycle('electron-process-exit', { @@ -277,7 +300,8 @@ export const launchElectronApp = async ( consoleEntries, code === 0 ? 'log' : 'error', 'main:exit', - `code=${String(code)} signal=${String(signal)}` + `code=${String(code)} signal=${String(signal)}`, + input.signalSink ) }) @@ -300,15 +324,17 @@ export const launchElectronApp = async ( }) page.on('console', (message) => { - consoleEntries.push({ + const entry = { ts: new Date().toISOString(), level: toConsoleLevel(message), text: message.text() - }) + } + consoleEntries.push(entry) + input.signalSink?.onConsole?.(entry) }) await page.waitForLoadState('domcontentloaded') - await installPageObservers(page) + await installPageObservers(page, input.signalSink) await page.evaluate((processId) => { const currentWindow = window as E2EWindow currentWindow.__NCURATOR_E2E_ELECTRON_PROCESS_ID__ = processId || undefined @@ -361,13 +387,25 @@ export const launchElectronApp = async ( path: input.tracePath }) } catch (error) { - pushProcessLog(consoleEntries, 'warning', 'trace', error instanceof Error ? error.message : String(error)) + pushProcessLog( + consoleEntries, + 'warning', + 'trace', + error instanceof Error ? error.message : String(error), + input.signalSink + ) } } try { await electronApp.close() } catch (error) { - pushProcessLog(consoleEntries, 'warning', 'electron-close', error instanceof Error ? error.message : String(error)) + pushProcessLog( + consoleEntries, + 'warning', + 'electron-close', + error instanceof Error ? error.message : String(error), + input.signalSink + ) } await rm(userDataDir, { recursive: true, force: true }) } diff --git a/tests/harness/e2e/run-e2e-case.ts b/tests/harness/e2e/run-e2e-case.ts index 023b2706..ee23821b 100644 --- a/tests/harness/e2e/run-e2e-case.ts +++ b/tests/harness/e2e/run-e2e-case.ts @@ -13,12 +13,18 @@ import { type TestRunMeta, type TestRunStatus } from '../core/run-log-contract' -import { createRunId, resolveRunArtifactsDir, writeRunLog } from '../core/write-run-log' +import { + createRunId, + createStreamingRunLogWriter, + resolveRunArtifactsDir, + type StreamingRunLogWriter +} from '../core/write-run-log' import { launchElectronApp, type E2ERuntimeSignals, type ElectronLaunchMode, - type ElectronWindowMode + type ElectronWindowMode, + type LaunchedElectronApp } from './electronApp' export type E2ETestRunMode = 'single' | 'repeatable' @@ -113,6 +119,20 @@ const escapeHtml = (value: string): string => .replace(/"/g, '"') .replace(/'/g, ''') +const getRuntimeEventKey = (entry: E2ERuntimeSignals['aiRunEventEntries'][number]): string => { + const event = entry.event + const runId = 'runId' in event ? String(event.runId || '') : '' + const createdAt = 'createdAt' in event ? String(event.createdAt || '') : '' + return [entry.ts, event.type, runId, createdAt].join(':') +} + +const getTimelineSignalKey = (input: { + source: string + ts: string + name: string + data?: JsonValue +}): string => `${input.source}:${input.ts}:${input.name}:${JSON.stringify(input.data || null)}` + const buildPlaywrightReportHtml = (input: { caseId: string runId: string @@ -195,11 +215,19 @@ export const runE2ECase = async ( }) const events: TestLogEvent[] = [] const artifacts: TestArtifact[] = [] + const streamingLogWriter: StreamingRunLogWriter = createStreamingRunLogWriter({ + scope, + caseId: definition.caseId, + runId + }) const logDirArtifacts = resolveRunArtifactsDir({ scope, caseId: definition.caseId, runId }) + const emittedTimelineSignalKeys = new Set() + const emittedRuntimeEventKeys = new Set() + let launchedApp: LaunchedElectronApp | null = null let status: TestRunStatus = 'failed' let stepCounter = 0 let assertionCounter = 0 @@ -209,13 +237,34 @@ export const runE2ECase = async ( let failedAssertionId: string | null = null let executionErrorName: string | null = null let executionErrorMessage: string | null = null - const launchedApp = await launchElectronApp({ - launchMode: definition.launchMode, - windowMode: definition.windowMode - }) const pushEvent = (event: TestLogEvent): void => { events.push(event) + streamingLogWriter.appendTimelineEvent(event) + } + + const appendRuntimeEventOnce = ( + entry: E2ERuntimeSignals['aiRunEventEntries'][number] + ): void => { + const key = getRuntimeEventKey(entry) + if (emittedRuntimeEventKeys.has(key)) return + emittedRuntimeEventKeys.add(key) + streamingLogWriter.appendRuntimeEvent(entry.event) + } + + const pushSignalEventOnce = (input: { + source: string + event: TestLogEvent + }): void => { + const key = getTimelineSignalKey({ + source: input.source, + ts: input.event.ts, + name: input.event.name, + data: input.event.data + }) + if (emittedTimelineSignalKeys.has(key)) return + emittedTimelineSignalKeys.add(key) + pushEvent(input.event) } const recordEvent = (input: Omit): void => { @@ -261,18 +310,24 @@ export const runE2ECase = async ( const relativePath = path.posix.join('screenshots', `${input.name}.png`) const absolutePath = path.join(logDirArtifacts, relativePath) await mkdir(path.dirname(absolutePath), { recursive: true }) - if (launchedApp.page.isClosed()) { + if (!launchedApp || launchedApp.page.isClosed()) { recordEvent({ type: 'signal', name: 'screenshot-skipped', status: 'info', - message: '页面已关闭，跳过失败截图', + message: launchedApp ? '页面已关闭，跳过失败截图' : 'Electron 未启动，跳过失败截图', data: { name: input.name } }) const placeholderPath = path.join(logDirArtifacts, relativePath) - await writeFile(placeholderPath, 'page already closed; screenshot skipped', 'utf8') + await writeFile( + placeholderPath, + launchedApp + ? 'page already closed; screenshot skipped' + : 'electron not launched; screenshot skipped', + 'utf8' + ) return captureArtifact({ kind: 'other', relativePath, @@ -306,10 +361,10 @@ export const runE2ECase = async ( }) } - const runtime: E2ETestRuntime = { + const createRuntime = (app: LaunchedElectronApp): E2ETestRuntime => ({ runId, taskId, - page: launchedApp.page, + page: app.page, recordEvent, signal, step: async (input) => { @@ -374,8 +429,32 @@ export const runE2ECase = async ( }, captureArtifact, captureScreenshot, - collectSignals: launchedApp.collectSignals - } + collectSignals: app.collectSignals + }) + + streamingLogWriter.init({ + meta: { + schemaVersion: TEST_LOG_SCHEMA_VERSION, + runId, + caseId: definition.caseId, + scope, + startedAt: new Date(startedAtMs).toISOString(), + finishedAt: '', + status: 'running', + stats: { + durationMs: 0, + steps: 0, + assertions: 0, + errors: 0, + artifacts: 0 + }, + runtime: { + driver: 'playwright', + target: 'electron', + modelName: null + } + } + }) recordEvent({ type: 'run', @@ -390,8 +469,62 @@ export const runE2ECase = async ( runMode: definition.runMode } }) + await streamingLogWriter.flush() try { + launchedApp = await launchElectronApp({ + launchMode: definition.launchMode, + windowMode: definition.windowMode, + signalSink: { + onAiRunEvent: (entry) => { + appendRuntimeEventOnce(entry) + pushSignalEventOnce({ + source: 'ai-run-event', + event: { + ts: entry.ts, + type: 'signal', + name: 'ai-run-event', + status: 'info', + message: `收到 AI 事件 ${entry.event.type}`, + data: { + eventType: entry.event.type + } + } + }) + }, + onConsole: (entry) => { + if (entry.level === 'log') return + pushSignalEventOnce({ + source: 'console', + event: { + ts: entry.ts, + type: 'signal', + name: entry.level === 'warning' ? 'console-warning' : 'console-error', + status: 'info', + message: `捕获到 console ${entry.level}：${entry.text}`, + data: { + level: entry.level, + text: entry.text + } + } + }) + }, + onLifecycle: (entry) => { + pushSignalEventOnce({ + source: 'electron-lifecycle', + event: { + ts: entry.ts, + type: 'signal', + name: 'electron-lifecycle', + status: 'info', + message: `Electron lifecycle: ${entry.name}`, + data: entry.data + } + }) + } + } + }) + const runtime = createRuntime(launchedApp) const result = await definition.execute(runtime) await definition.assert(result, runtime) status = 'passed' @@ -422,44 +555,62 @@ export const runE2ECase = async ( } throw error } finally { - const signals = await launchedApp.collectSignals() - const runtimeEvents = signals.aiRunEventEntries.map((entry) => entry.event) + const signals: E2ERuntimeSignals = launchedApp + ? await launchedApp.collectSignals() + : { + routeEntries: [], + ipcInvokeEntries: [], + aiRunEventEntries: [], + domEventEntries: [], + consoleEntries: [], + lifecycleEntries: [] + } for (const routeEntry of signals.routeEntries) { - pushEvent({ - ts: routeEntry.ts, - type: 'signal', - name: 'route-change', - status: 'info', - message: `路由变更为 ${routeEntry.route}`, - data: { - route: routeEntry.route + pushSignalEventOnce({ + source: 'route-change', + event: { + ts: routeEntry.ts, + type: 'signal', + name: 'route-change', + status: 'info', + message: `路由变更为 ${routeEntry.route}`, + data: { + route: routeEntry.route + } } }) } for (const ipcEntry of signals.ipcInvokeEntries) { - pushEvent({ - ts: ipcEntry.ts, - type: 'signal', - name: 'ipc-call', - status: 'info', - message: `调用 IPC ${ipcEntry.name}`, - data: { - name: ipcEntry.name + pushSignalEventOnce({ + source: 'ipc-call', + event: { + ts: ipcEntry.ts, + type: 'signal', + name: 'ipc-call', + status: 'info', + message: `调用 IPC ${ipcEntry.name}`, + data: { + name: ipcEntry.name + } } }) } for (const aiRunEventEntry of signals.aiRunEventEntries) { - pushEvent({ - ts: aiRunEventEntry.ts, - type: 'signal', - name: 'ai-run-event', - status: 'info', - message: `收到 AI 事件 ${aiRunEventEntry.event.type}`, - data: { - eventType: aiRunEventEntry.event.type + appendRuntimeEventOnce(aiRunEventEntry) + pushSignalEventOnce({ + source: 'ai-run-event', + event: { + ts: aiRunEventEntry.ts, + type: 'signal', + name: 'ai-run-event', + status: 'info', + message: `收到 AI 事件 ${aiRunEventEntry.event.type}`, + data: { + eventType: aiRunEventEntry.event.type + } } }) } @@ -468,41 +619,49 @@ export const runE2ECase = async ( if (consoleEntry.level === 'log') { continue } - pushEvent({ - ts: consoleEntry.ts, - type: 'signal', - name: consoleEntry.level === 'warning' ? 'console-warning' : 'console-error', - status: 'info', - message: `捕获到 console ${consoleEntry.level}：${consoleEntry.text}`, - data: { - level: consoleEntry.level, - text: consoleEntry.text + pushSignalEventOnce({ + source: 'console', + event: { + ts: consoleEntry.ts, + type: 'signal', + name: consoleEntry.level === 'warning' ? 'console-warning' : 'console-error', + status: 'info', + message: `捕获到 console ${consoleEntry.level}：${consoleEntry.text}`, + data: { + level: consoleEntry.level, + text: consoleEntry.text + } } }) } for (const lifecycleEntry of signals.lifecycleEntries) { - pushEvent({ - ts: lifecycleEntry.ts, - type: 'signal', - name: 'electron-lifecycle', - status: 'info', - message: `Electron lifecycle: ${lifecycleEntry.name}`, - data: lifecycleEntry.data + pushSignalEventOnce({ + source: 'electron-lifecycle', + event: { + ts: lifecycleEntry.ts, + type: 'signal', + name: 'electron-lifecycle', + status: 'info', + message: `Electron lifecycle: ${lifecycleEntry.name}`, + data: lifecycleEntry.data + } }) } - const traceRelativePath = path.posix.join('trace', 'playwright-trace.zip') - const traceAbsolutePath = path.join(logDirArtifacts, traceRelativePath) - await mkdir(path.dirname(traceAbsolutePath), { recursive: true }) - await launchedApp.close({ - tracePath: traceAbsolutePath - }) - await captureArtifact({ - kind: 'trace', - relativePath: traceRelativePath, - message: '保存 Playwright trace' - }) + if (launchedApp) { + const traceRelativePath = path.posix.join('trace', 'playwright-trace.zip') + const traceAbsolutePath = path.join(logDirArtifacts, traceRelativePath) + await mkdir(path.dirname(traceAbsolutePath), { recursive: true }) + await launchedApp.close({ + tracePath: traceAbsolutePath + }) + await captureArtifact({ + kind: 'trace', + relativePath: traceRelativePath, + message: '保存 Playwright trace' + }) + } if (status === 'failed') { const reportRelativePath = path.posix.join('playwright', 'report', 'index.html') @@ -558,10 +717,8 @@ export const runE2ECase = async ( } } - await writeRunLog({ + await streamingLogWriter.finalize({ meta, - events, - runtimeEvents, artifacts }) } diff --git a/tests/harness/e2e/workbench.ts b/tests/harness/e2e/workbench.ts index 36c00c10..0ce01871 100644 --- a/tests/harness/e2e/workbench.ts +++ b/tests/harness/e2e/workbench.ts @@ -954,6 +954,25 @@ export const sendWorkbenchMessage = async (page: Page, value: string): Promise => { + await getWorkbenchLocator(page, workbenchSelectors.newSessionButton).click() + await getWorkbenchLocator(page, workbenchSelectors.input).waitFor({ state: 'visible' }) + await page.waitForFunction( + (input: { assistantTurnPrefix: string; approvalBatchCard: string }) => { + const assistantTurns = document.querySelectorAll( + `[data-testid^="${input.assistantTurnPrefix}-"]` + ) + const approvalCard = document.querySelector(`[data-testid="${input.approvalBatchCard}"]`) + return assistantTurns.length === 0 && !approvalCard + }, + { + assistantTurnPrefix: workbenchSelectors.assistantTurnPrefix, + approvalBatchCard: chatSelectors.approvalBatchCard + }, + { timeout: WORKBENCH_E2E_TIMEOUT_MS } + ) +} + export const switchWorkbenchMode = async ( page: Page, modeLabel: '大聪明' | '没头脑' @@ -1315,6 +1334,73 @@ export const waitForWorkbenchAiEvent = async ( } } +export const decideWorkbenchApprovalBatch = async ( + runtime: E2ETestRuntime, + page: Page, + input: { + event: Extract + action: 'approve' | 'reject' + timeoutMs?: number + } +): Promise> => { + const card = getApprovalCard(page) + const buttonTestId = + input.action === 'approve' + ? chatSelectors.approvalBatchApproveButton + : chatSelectors.approvalBatchDenyButton + const button = getWorkbenchLocator(page, buttonTestId) + await card.waitFor({ state: 'visible', timeout: WORKBENCH_E2E_TIMEOUT_MS }) + + for (let index = 0; index < input.event.items.length; index += 1) { + const cardText = await readLocatorText(card) + runtime.signal({ + name: 'workbench-approval-decision-click', + message: `提交审批决策 ${index + 1}/${input.event.items.length}`, + data: { + action: input.action, + batchId: input.event.batchId, + item: input.event.items[index] || null, + cardText + } + }) + await button.waitFor({ state: 'visible', timeout: WORKBENCH_E2E_TIMEOUT_MS }) + await button.scrollIntoViewIfNeeded() + await page.waitForFunction( + (testId: string) => { + const element = document.querySelector(`[data-testid="${testId}"]`) + return element instanceof HTMLButtonElement && !element.disabled + }, + buttonTestId, + { timeout: WORKBENCH_E2E_TIMEOUT_MS } + ) + await button.click() + if (index < input.event.items.length - 1) { + await page.waitForFunction( + (testId: string, expectedIndex: number) => { + const root = document.querySelector(`[data-testid="${testId}"]`) + return Boolean(root?.textContent?.includes(`审批 ${expectedIndex} /`)) + }, + chatSelectors.approvalBatchCard, + index + 2, + { timeout: WORKBENCH_E2E_TIMEOUT_MS } + ) + } + } + + const decision = await waitForWorkbenchAiEvent(runtime, { + name: `approval_batch_decision_${input.action}_${input.event.batchId}`, + timeoutMs: input.timeoutMs || WORKBENCH_E2E_TIMEOUT_MS, + matches: (event) => + event.type === 'approval_batch_decision' && + event.batchId === input.event.batchId && + event.decisions.some((item) => item.action === input.action) + }) + if (decision.type !== 'approval_batch_decision') { + throw new Error(`approval batch decision did not resolve for ${input.event.batchId}`) + } + return decision +} + export const waitForWorkbenchAiRunOutcome = async ( runtime: E2ETestRuntime, startedAtMs: number, @@ -1365,7 +1451,17 @@ export const waitForWorkbenchAiRunOutcome = async ( event.type === 'approval_batch_required' ) .at(-1) - if (approvalRequired && !approvalAutoHandledBatchIds.has(approvalRequired.batchId)) { + const approvalAlreadyDecided = approvalRequired + ? events.some( + (event) => + event.type === 'approval_batch_decision' && event.batchId === approvalRequired.batchId + ) + : false + if ( + approvalRequired && + !approvalAlreadyDecided && + !approvalAutoHandledBatchIds.has(approvalRequired.batchId) + ) { const assistantTextSoFar = await readAssistantTextSoFar(page) const cardText = await readLocatorText(getApprovalCard(page)) const blocked: WorkbenchRunBlockedResult = {