feat: enhance E2E tests with new session creation and approval batch decision handling

This commit is contained in:
hy committed 2026-06-16 12:29:46 +08:00
1 parent 9cde4b16dc
commit cb36f9dc22
7 files changed
+595 -120

No files matched your search

+6 -6
View File
@@ -15,8 +15,8 @@
### 2.1 当前状态
- 总体登记:`13` 个 e2e case。
- 当前有效结论:`10 / 13` 有效通过。
- 当前实现状态:`tests/case/workbench/**` 已按 suite / group 落地 `13` 个独立 `.e2e.test.ts` 文件;当前已有 `10 / 13` 形成真实有效通过结论。
- 当前有效结论:`11 / 13` 有效通过。
- 当前实现状态:`tests/case/workbench/**` 已按 suite / group 落地 `13` 个独立 `.e2e.test.ts` 文件;当前已有 `11 / 13` 形成真实有效通过结论。
### 2.2 按 Suite 汇总
@@ -27,9 +27,9 @@
- 备注:case 已实现;`session-initial-create-and-title`、`send-guard-mode-switch-and-preferences`、`workspace-root-path-required`、`reference-token-lifecycle` 均已通过真实 Electron UI;其中涉及 AI 的 case 使用真实模型,`reference-token-lifecycle` 不启动 AI。
- `run`
- 登记 case:`4`
- 有效通过:`1 / 4`
- 检测时间:`2026-06-16 10:32:04 +0800`
- 备注:case 已实现;`ai-process-stream-and-final-answer` 已通过真实 Electron UI、真实模型、真实 workspace 索引和真实 AI run。
- 有效通过:`2 / 4`
- 检测时间:`2026-06-16 12:04:17 +0800`
- 备注:case 已实现;`ai-process-stream-and-final-answer`、`approval-batch-flow` 已通过真实 Electron UI、真实模型、真实 workspace 索引和真实 AI run;`stop-running-or-approval-run` 当前暴露审批等待时缺少停止入口的生产行为缺口;`browser-use-task-panel` 当前暴露 `browser_use` / Stagehand 对当前 live 模型接口返回 `Bad Request` 的生产或配置阻塞。
- `result`
- 登记 case:`5`
- 有效通过:`5 / 5`
@@ -44,7 +44,7 @@
- Workbench formal e2e 需要真实索引完成信号和 UI 可见状态作为等待条件;不得再读取内部 store 或注入 fixed document id 来制造搜索结果。
- `file-preview-actions` 当前按生产真实索引能力覆盖 `pdf/docx`:`src/main/services/ingestion/loader.ts` 只支持 `.pdf`、`.docx` 本地知识库索引,`tests/fixtures/workspace/workbench-preview.md` 无法通过真实搜索结果打开;如 PRD 需要 markdown 也进入 Workbench 搜索 / 预览链路,需要产品 / 生产实现决策。
- docx apply case 需要真实 docx 输入、真实 AI 编辑、真实写回记录、真实预览和真实回退。
- `browser-use-task-panel` 需要真实 AI run 触发 `browser_use` 并驱动内置浏览器完成本地网页任务。
- `browser-use-task-panel` 需要真实 AI run 触发 `browser_use` 并驱动内置浏览器完成本地网页任务;当前测试已在连续两次 `browser_use` 失败时快速失败并记录诊断,避免模型无效重试导致长时间无反馈。
## 3. Suite 设计
@@ -1,11 +1,15 @@
import { access } from 'node:fs/promises'
import path from 'node:path'
import { describe, it } from 'vitest'
import type { AiRunEvent } from '../../../../../src/shared/types'
import { runE2ECase } from '../../../../harness/e2e/run-e2e-case'
import {
WORKBENCH_LIVE_AI_E2E_TIMEOUT_MS,
assertNoWorkbenchFakeSignals,
chatSelectors,
collectWorkbenchAiRunEvents,
createWorkbenchNewSession,
decideWorkbenchApprovalBatch,
getApprovalCard,
getWorkbenchLocator,
recordWorkbenchAiEvidence,
@@ -16,12 +20,63 @@ import {
} from '../../../../harness/e2e/workbench'
const CASE_ID = 'approval-batch-flow'
const DENIED_RUN_SETTLE_TIMEOUT_MS = 120_000
const exists = async (filePath: string): Promise<boolean> =>
await access(filePath)
.then(() => true)
.catch(() => false)
const isRunStartedEvent = (
event: AiRunEvent
): event is Extract<AiRunEvent, { type: 'run_started' }> => event.type === 'run_started'
const isApprovalBatchRequiredEvent = (
event: AiRunEvent
): event is Extract<AiRunEvent, { type: 'approval_batch_required' }> =>
event.type === 'approval_batch_required'
const waitForDeniedRunToSettle = async (
runtime: Parameters<typeof collectWorkbenchAiRunEvents>[0],
page: Parameters<typeof getApprovalCard>[0],
runId: string,
rejectedBatchIds: Set<string>
): Promise<void> => {
const deadline = Date.now() + DENIED_RUN_SETTLE_TIMEOUT_MS
for (;;) {
const events = await collectWorkbenchAiRunEvents(runtime)
const settled = events.some(
(event) =>
(event.type === 'run_completed' || event.type === 'run_cancelled') &&
event.runId === runId
)
if (settled) return
const nextApproval = events
.filter(
(event): event is Extract<AiRunEvent, { type: 'approval_batch_required' }> =>
isApprovalBatchRequiredEvent(event) &&
event.runId === runId &&
!rejectedBatchIds.has(event.batchId)
)
.at(-1)
if (nextApproval) {
await decideWorkbenchApprovalBatch(runtime, page, {
event: nextApproval,
action: 'reject',
timeoutMs: 30_000
})
rejectedBatchIds.add(nextApproval.batchId)
continue
}
if (Date.now() >= deadline) {
throw new Error(`denied run did not settle after repeated rejects: ${runId}`)
}
await page.waitForTimeout(500)
}
}
describe('workbench run e2e', () => {
it(CASE_ID, async () => {
await runE2ECase({
@@ -47,8 +102,17 @@ describe('workbench run e2e', () => {
page,
'请使用终端命令在当前工作区创建 approval-approved.txt,内容写 approved-by-workbench-e2e。'
)
await getApprovalCard(page).waitFor({ state: 'visible' })
await getWorkbenchLocator(page, chatSelectors.approvalBatchApproveButton).click()
const approvedApproval = await waitForWorkbenchAiEvent(runtime, {
name: 'approval_approved_batch_required',
matches: isApprovalBatchRequiredEvent
})
if (!isApprovalBatchRequiredEvent(approvedApproval)) {
throw new Error('approval approved batch was not required')
}
await decideWorkbenchApprovalBatch(runtime, page, {
event: approvedApproval,
action: 'approve'
})
const approveEvidence = await waitForWorkbenchAiRunCompletion(
runtime,
approveStartedAtMs,
@@ -58,17 +122,37 @@ describe('workbench run e2e', () => {
await sendWorkbenchMessage(
page,
'请使用终端命令在当前工作区创建 approval-denied.txt,内容写 denied-by-workbench-e2e。'
'请使用终端命令在当前工作区创建 approval-denied.txt,内容写 denied-by-workbench-e2e;如果本次操作未获授权,请不要再次申请执行,直接说明未执行。'
)
await getApprovalCard(page).waitFor({ state: 'visible' })
await getWorkbenchLocator(page, chatSelectors.approvalBatchDenyButton).click()
await waitForWorkbenchAiEvent(runtime, {
name: 'approval_batch_decision_reject',
const deniedRunStarted = await waitForWorkbenchAiEvent(runtime, {
name: 'approval_denied_run_started',
matches: (event) =>
event.type === 'approval_batch_decision' &&
event.decisions.some((decision) => decision.action === 'reject')
isRunStartedEvent(event) && event.input.includes('approval-denied.txt')
})
if (!isRunStartedEvent(deniedRunStarted)) {
throw new Error('approval denied run did not start')
}
const firstDeniedApproval = await waitForWorkbenchAiEvent(runtime, {
name: 'approval_denied_batch_required',
matches: (event) =>
isApprovalBatchRequiredEvent(event) && event.runId === deniedRunStarted.runId
})
if (!isApprovalBatchRequiredEvent(firstDeniedApproval)) {
throw new Error('approval denied batch was not required')
}
const rejectedBatchIds = new Set([firstDeniedApproval.batchId])
await decideWorkbenchApprovalBatch(runtime, page, {
event: firstDeniedApproval,
action: 'reject'
})
await waitForDeniedRunToSettle(
runtime,
page,
deniedRunStarted.runId,
rejectedBatchIds
)
await createWorkbenchNewSession(page)
await sendWorkbenchMessage(
page,
'请使用终端命令在当前工作区创建 approval-guided.txt,内容写 guided-by-workbench-e2e。'
@@ -1,22 +1,133 @@
import { describe, it } from 'vitest'
import type { AiRunEvent } from '../../../../../src/shared/types'
import { runE2ECase } from '../../../../harness/e2e/run-e2e-case'
import {
WORKBENCH_LIVE_AI_E2E_TIMEOUT_MS,
type WorkbenchAiCompletionEvidence,
assertNoWorkbenchFakeSignals,
chatSelectors,
getApprovalCard,
collectWorkbenchAiRunEvents,
decideWorkbenchApprovalBatch,
getWorkbenchLocator,
recordWorkbenchAiEvidence,
sendWorkbenchMessage,
setupWorkbenchWithTempWorkspace,
startLocalWorkbenchHttpServer,
waitForWorkbenchAiEvent,
waitForWorkbenchAiRunCompletion,
workbenchSelectors
} from '../../../../harness/e2e/workbench'
const CASE_ID = 'browser-use-task-panel'
const TASK_CODE = 'cedar-browser-731'
const BROWSER_PANEL_WAIT_TIMEOUT_MS = 180_000
const BROWSER_TASK_COMPLETION_TIMEOUT_MS = 240_000
const BROWSER_USE_FAILURE_LIMIT = 2
const isApprovalBatchRequiredEvent = (
event: AiRunEvent
): event is Extract<AiRunEvent, { type: 'approval_batch_required' }> =>
event.type === 'approval_batch_required'
const waitForBrowserPanelOpenedWithApproval = async (
runtime: Parameters<typeof collectWorkbenchAiRunEvents>[0]
): Promise<void> => {
const handledBatchIds = new Set<string>()
const deadline = Date.now() + BROWSER_PANEL_WAIT_TIMEOUT_MS
for (;;) {
const events = await collectWorkbenchAiRunEvents(runtime)
const failed = events.find((event) => event.type === 'run_failed')
if (failed?.type === 'run_failed') {
throw new Error(`AI run failed while waiting for browser panel: ${failed.error}`)
}
if (events.some((event) => event.type === 'browser_panel_opened')) return
const approvalRequired = events
.filter(
(event): event is Extract<AiRunEvent, { type: 'approval_batch_required' }> =>
isApprovalBatchRequiredEvent(event) && !handledBatchIds.has(event.batchId)
)
.at(-1)
if (approvalRequired) {
handledBatchIds.add(approvalRequired.batchId)
await decideWorkbenchApprovalBatch(runtime, runtime.page, {
event: approvalRequired,
action: 'approve'
})
continue
}
if (Date.now() >= deadline) {
throw new Error('browser panel did not open before timeout')
}
await runtime.page.waitForTimeout(500)
}
}
const waitForBrowserTaskCompletion = async (
runtime: Parameters<typeof collectWorkbenchAiRunEvents>[0],
startedAtMs: number,
modelName: string
): Promise<WorkbenchAiCompletionEvidence> => {
const deadline = Date.now() + BROWSER_TASK_COMPLETION_TIMEOUT_MS
for (;;) {
const events = await collectWorkbenchAiRunEvents(runtime)
const failed = events.find((event) => event.type === 'run_failed')
if (failed?.type === 'run_failed') {
throw new Error(`AI run failed while waiting for browser task: ${failed.error}`)
}
const browserUseFailures = events.filter(
(event): event is Extract<AiRunEvent, { type: 'tool_call_result' }> =>
event.type === 'tool_call_result' &&
event.toolName === 'browser_use' &&
(Boolean(event.error) || event.result?.ok === false)
)
if (browserUseFailures.length >= BROWSER_USE_FAILURE_LIMIT) {
const latestFailure = browserUseFailures.at(-1)
runtime.signal({
name: 'workbench-browser-use-repeated-failure',
status: 'failed',
message: 'browser_use 连续失败,停止等待模型重复重试。',
data: {
failureCount: browserUseFailures.length,
latestFailure: latestFailure
? {
toolCallId: latestFailure.toolCallId,
error: latestFailure.error || '',
result: latestFailure.result || null
}
: null
}
})
throw new Error(
`browser_use failed ${browserUseFailures.length} times; latest=${JSON.stringify(
latestFailure?.result || latestFailure?.error || null
)}`
)
}
const runCompleted = events.find((event) => event.type === 'run_completed')
if (runCompleted?.type === 'run_completed') {
const finalAnswerText =
events
.filter(
(event): event is Extract<AiRunEvent, { type: 'answer_completed' }> =>
event.type === 'answer_completed'
)
.map((event) => event.text)
.at(-1) || ''
return {
modelName,
durationMs: Date.now() - startedAtMs,
events,
finalAnswerText
}
}
if (Date.now() >= deadline) {
throw new Error('browser task did not complete before timeout')
}
await runtime.page.waitForTimeout(500)
}
}
describe('workbench run e2e', () => {
it(CASE_ID, async () => {
@@ -50,25 +161,14 @@ describe('workbench run e2e', () => {
`请使用右侧内置浏览器打开 ${server.url},在“任务口令”输入框填入 ${TASK_CODE},点击“提交任务”,然后告诉我页面显示的完整结果。不要只抓取源码,必须实际操作页面。`
)
const approvalVisible = await getApprovalCard(page)
.waitFor({ state: 'visible', timeout: 30_000 })
.then(() => true)
.catch(() => false)
if (approvalVisible) {
await getWorkbenchLocator(page, chatSelectors.approvalBatchApproveButton).click()
}
await waitForWorkbenchAiEvent(runtime, {
name: 'browser_panel_opened',
matches: (event) => event.type === 'browser_panel_opened'
})
await waitForBrowserPanelOpenedWithApproval(runtime)
await getWorkbenchLocator(page, workbenchSelectors.browserPanel).waitFor({
state: 'visible'
})
await getWorkbenchLocator(page, workbenchSelectors.browserViewport).waitFor({
state: 'visible'
})
const evidence = await waitForWorkbenchAiRunCompletion(
const evidence = await waitForBrowserTaskCompletion(
runtime,
startedAtMs,
setup.env.modelName
+1 -1
View File
@@ -73,7 +73,7 @@ const buildEmptySeedManifest = (): PersistedSeedManifest => ({
kbDocuments: []
})
type StreamingRunLogWriter = {
export type StreamingRunLogWriter = {
logDir: string
paths: {
metaPath: string
+55 -17
View File
@@ -56,6 +56,7 @@ type E2EObserverState = {
type E2EWindow = Window & {
__NCURATOR_E2E_OBSERVER__?: E2EObserverState
__NCURATOR_E2E_ELECTRON_PROCESS_ID__?: number
__NCURATOR_E2E_RECORD_AI_RUN_EVENT__?: (entry: E2EAiRunEventEntry) => void
}
export type E2ERuntimeSignals = {
@@ -77,6 +78,12 @@ export type LaunchedElectronApp = {
export type ElectronLaunchMode = 'fixture' | 'real'
export type ElectronWindowMode = 'background' | 'visible'
export type ElectronSignalSink = {
onAiRunEvent?: (entry: E2EAiRunEventEntry) => void
onConsole?: (entry: E2EConsoleEntry) => void
onLifecycle?: (entry: E2ELifecycleEntry) => void
}
const OUT_MAIN_ENTRY = path.resolve(process.cwd(), 'out', 'main', 'index.js')
const toConsoleLevel = (message: ConsoleMessage): E2EConsoleLevel => {
@@ -89,15 +96,18 @@ const pushProcessLog = (
entries: E2EConsoleEntry[],
level: E2EConsoleLevel,
source: string,
chunk: Buffer | string
chunk: Buffer | string,
sink?: ElectronSignalSink
): void => {
const text = String(chunk)
if (!text.trim()) return
entries.push({
const entry = {
ts: new Date().toISOString(),
level,
text: `[${source}] ${text.trim()}`
})
}
entries.push(entry)
sink?.onConsole?.(entry)
}
const ensureBuiltAppExists = async (): Promise<void> => {
@@ -166,8 +176,13 @@ const buildElectronLaunchEnv = (
return env
}
const installPageObservers = async (page: Page): Promise<void> => {
const installPageObservers = async (page: Page, sink?: ElectronSignalSink): Promise<void> => {
await page.waitForFunction(() => Boolean(window.api))
if (sink?.onAiRunEvent) {
await page.exposeFunction('__NCURATOR_E2E_RECORD_AI_RUN_EVENT__', (entry: E2EAiRunEventEntry) => {
sink.onAiRunEvent?.(entry)
})
}
await page.evaluate(() => {
const currentWindow = window as E2EWindow
@@ -202,10 +217,12 @@ const installPageObservers = async (page: Page): Promise<void> => {
window.addEventListener('ncurator-e2e-ai-run-event', (event) => {
const detail = (event as CustomEvent<AiRunEvent>).detail
currentWindow.__NCURATOR_E2E_OBSERVER__?.aiRunEventEntries.push({
const entry = {
ts: now(),
event: detail
})
}
currentWindow.__NCURATOR_E2E_OBSERVER__?.aiRunEventEntries.push(entry)
currentWindow.__NCURATOR_E2E_RECORD_AI_RUN_EVENT__?.(entry)
})
window.addEventListener('check-llm-config', () => {
@@ -231,18 +248,24 @@ const focusVisibleElectronWindow = async (electronApp: ElectronApplication): Pro
}
export const launchElectronApp = async (
input: { launchMode?: ElectronLaunchMode; windowMode?: ElectronWindowMode } = {}
input: {
launchMode?: ElectronLaunchMode
windowMode?: ElectronWindowMode
signalSink?: ElectronSignalSink
} = {}
): Promise<LaunchedElectronApp> => {
await ensureBuiltAppExists()
const consoleEntries: E2EConsoleEntry[] = []
const lifecycleEntries: E2ELifecycleEntry[] = []
const recordLifecycle = (name: string, data?: JsonValue): void => {
lifecycleEntries.push({
const entry = {
ts: new Date().toISOString(),
name,
data
})
}
lifecycleEntries.push(entry)
input.signalSink?.onLifecycle?.(entry)
}
const userDataDir = await mkdtemp(path.join(os.tmpdir(), 'ncurator-e2e-user-data-'))
const remoteDebuggingPort = await allocateFreePort()
@@ -263,10 +286,10 @@ export const launchElectronApp = async (
pid: electronProcess?.pid || null
})
electronProcess?.stdout?.on('data', (chunk) => {
pushProcessLog(consoleEntries, 'log', 'main:stdout', chunk)
pushProcessLog(consoleEntries, 'log', 'main:stdout', chunk, input.signalSink)
})
electronProcess?.stderr?.on('data', (chunk) => {
pushProcessLog(consoleEntries, 'error', 'main:stderr', chunk)
pushProcessLog(consoleEntries, 'error', 'main:stderr', chunk, input.signalSink)
})
electronProcess?.on('exit', (code, signal) => {
recordLifecycle('electron-process-exit', {
@@ -277,7 +300,8 @@ export const launchElectronApp = async (
consoleEntries,
code === 0 ? 'log' : 'error',
'main:exit',
`code=${String(code)} signal=${String(signal)}`
`code=${String(code)} signal=${String(signal)}`,
input.signalSink
)
})
@@ -300,15 +324,17 @@ export const launchElectronApp = async (
})
page.on('console', (message) => {
consoleEntries.push({
const entry = {
ts: new Date().toISOString(),
level: toConsoleLevel(message),
text: message.text()
})
}
consoleEntries.push(entry)
input.signalSink?.onConsole?.(entry)
})
await page.waitForLoadState('domcontentloaded')
await installPageObservers(page)
await installPageObservers(page, input.signalSink)
await page.evaluate((processId) => {
const currentWindow = window as E2EWindow
currentWindow.__NCURATOR_E2E_ELECTRON_PROCESS_ID__ = processId || undefined
@@ -361,13 +387,25 @@ export const launchElectronApp = async (
path: input.tracePath
})
} catch (error) {
pushProcessLog(consoleEntries, 'warning', 'trace', error instanceof Error ? error.message : String(error))
pushProcessLog(
consoleEntries,
'warning',
'trace',
error instanceof Error ? error.message : String(error),
input.signalSink
)
}
}
try {
await electronApp.close()
} catch (error) {
pushProcessLog(consoleEntries, 'warning', 'electron-close', error instanceof Error ? error.message : String(error))
pushProcessLog(
consoleEntries,
'warning',
'electron-close',
error instanceof Error ? error.message : String(error),
input.signalSink
)
}
await rm(userDataDir, { recursive: true, force: true })
}
+226 -69
View File
@@ -13,12 +13,18 @@ import {
type TestRunMeta,
type TestRunStatus
} from '../core/run-log-contract'
import { createRunId, resolveRunArtifactsDir, writeRunLog } from '../core/write-run-log'
import {
createRunId,
createStreamingRunLogWriter,
resolveRunArtifactsDir,
type StreamingRunLogWriter
} from '../core/write-run-log'
import {
launchElectronApp,
type E2ERuntimeSignals,
type ElectronLaunchMode,
type ElectronWindowMode
type ElectronWindowMode,
type LaunchedElectronApp
} from './electronApp'
export type E2ETestRunMode = 'single' | 'repeatable'
@@ -113,6 +119,20 @@ const escapeHtml = (value: string): string =>
.replace(/"/g, '&quot;')
.replace(/'/g, '&#39;')
const getRuntimeEventKey = (entry: E2ERuntimeSignals['aiRunEventEntries'][number]): string => {
const event = entry.event
const runId = 'runId' in event ? String(event.runId || '') : ''
const createdAt = 'createdAt' in event ? String(event.createdAt || '') : ''
return [entry.ts, event.type, runId, createdAt].join(':')
}
const getTimelineSignalKey = (input: {
source: string
ts: string
name: string
data?: JsonValue
}): string => `${input.source}:${input.ts}:${input.name}:${JSON.stringify(input.data || null)}`
const buildPlaywrightReportHtml = (input: {
caseId: string
runId: string
@@ -195,11 +215,19 @@ export const runE2ECase = async <TResult>(
})
const events: TestLogEvent[] = []
const artifacts: TestArtifact[] = []
const streamingLogWriter: StreamingRunLogWriter = createStreamingRunLogWriter({
scope,
caseId: definition.caseId,
runId
})
const logDirArtifacts = resolveRunArtifactsDir({
scope,
caseId: definition.caseId,
runId
})
const emittedTimelineSignalKeys = new Set<string>()
const emittedRuntimeEventKeys = new Set<string>()
let launchedApp: LaunchedElectronApp | null = null
let status: TestRunStatus = 'failed'
let stepCounter = 0
let assertionCounter = 0
@@ -209,13 +237,34 @@ export const runE2ECase = async <TResult>(
let failedAssertionId: string | null = null
let executionErrorName: string | null = null
let executionErrorMessage: string | null = null
const launchedApp = await launchElectronApp({
launchMode: definition.launchMode,
windowMode: definition.windowMode
})
const pushEvent = (event: TestLogEvent): void => {
events.push(event)
streamingLogWriter.appendTimelineEvent(event)
}
const appendRuntimeEventOnce = (
entry: E2ERuntimeSignals['aiRunEventEntries'][number]
): void => {
const key = getRuntimeEventKey(entry)
if (emittedRuntimeEventKeys.has(key)) return
emittedRuntimeEventKeys.add(key)
streamingLogWriter.appendRuntimeEvent(entry.event)
}
const pushSignalEventOnce = (input: {
source: string
event: TestLogEvent
}): void => {
const key = getTimelineSignalKey({
source: input.source,
ts: input.event.ts,
name: input.event.name,
data: input.event.data
})
if (emittedTimelineSignalKeys.has(key)) return
emittedTimelineSignalKeys.add(key)
pushEvent(input.event)
}
const recordEvent = (input: Omit<TestLogEvent, 'ts'>): void => {
@@ -261,18 +310,24 @@ export const runE2ECase = async <TResult>(
const relativePath = path.posix.join('screenshots', `${input.name}.png`)
const absolutePath = path.join(logDirArtifacts, relativePath)
await mkdir(path.dirname(absolutePath), { recursive: true })
if (launchedApp.page.isClosed()) {
if (!launchedApp || launchedApp.page.isClosed()) {
recordEvent({
type: 'signal',
name: 'screenshot-skipped',
status: 'info',
message: '页面已关闭,跳过失败截图',
message: launchedApp ? '页面已关闭,跳过失败截图' : 'Electron 未启动,跳过失败截图',
data: {
name: input.name
}
})
const placeholderPath = path.join(logDirArtifacts, relativePath)
await writeFile(placeholderPath, 'page already closed; screenshot skipped', 'utf8')
await writeFile(
placeholderPath,
launchedApp
? 'page already closed; screenshot skipped'
: 'electron not launched; screenshot skipped',
'utf8'
)
return captureArtifact({
kind: 'other',
relativePath,
@@ -306,10 +361,10 @@ export const runE2ECase = async <TResult>(
})
}
const runtime: E2ETestRuntime = {
const createRuntime = (app: LaunchedElectronApp): E2ETestRuntime => ({
runId,
taskId,
page: launchedApp.page,
page: app.page,
recordEvent,
signal,
step: async (input) => {
@@ -374,8 +429,32 @@ export const runE2ECase = async <TResult>(
},
captureArtifact,
captureScreenshot,
collectSignals: launchedApp.collectSignals
}
collectSignals: app.collectSignals
})
streamingLogWriter.init({
meta: {
schemaVersion: TEST_LOG_SCHEMA_VERSION,
runId,
caseId: definition.caseId,
scope,
startedAt: new Date(startedAtMs).toISOString(),
finishedAt: '',
status: 'running',
stats: {
durationMs: 0,
steps: 0,
assertions: 0,
errors: 0,
artifacts: 0
},
runtime: {
driver: 'playwright',
target: 'electron',
modelName: null
}
}
})
recordEvent({
type: 'run',
@@ -390,8 +469,62 @@ export const runE2ECase = async <TResult>(
runMode: definition.runMode
}
})
await streamingLogWriter.flush()
try {
launchedApp = await launchElectronApp({
launchMode: definition.launchMode,
windowMode: definition.windowMode,
signalSink: {
onAiRunEvent: (entry) => {
appendRuntimeEventOnce(entry)
pushSignalEventOnce({
source: 'ai-run-event',
event: {
ts: entry.ts,
type: 'signal',
name: 'ai-run-event',
status: 'info',
message: `收到 AI 事件 ${entry.event.type}`,
data: {
eventType: entry.event.type
}
}
})
},
onConsole: (entry) => {
if (entry.level === 'log') return
pushSignalEventOnce({
source: 'console',
event: {
ts: entry.ts,
type: 'signal',
name: entry.level === 'warning' ? 'console-warning' : 'console-error',
status: 'info',
message: `捕获到 console ${entry.level}:${entry.text}`,
data: {
level: entry.level,
text: entry.text
}
}
})
},
onLifecycle: (entry) => {
pushSignalEventOnce({
source: 'electron-lifecycle',
event: {
ts: entry.ts,
type: 'signal',
name: 'electron-lifecycle',
status: 'info',
message: `Electron lifecycle: ${entry.name}`,
data: entry.data
}
})
}
}
})
const runtime = createRuntime(launchedApp)
const result = await definition.execute(runtime)
await definition.assert(result, runtime)
status = 'passed'
@@ -422,44 +555,62 @@ export const runE2ECase = async <TResult>(
}
throw error
} finally {
const signals = await launchedApp.collectSignals()
const runtimeEvents = signals.aiRunEventEntries.map((entry) => entry.event)
const signals: E2ERuntimeSignals = launchedApp
? await launchedApp.collectSignals()
: {
routeEntries: [],
ipcInvokeEntries: [],
aiRunEventEntries: [],
domEventEntries: [],
consoleEntries: [],
lifecycleEntries: []
}
for (const routeEntry of signals.routeEntries) {
pushEvent({
ts: routeEntry.ts,
type: 'signal',
name: 'route-change',
status: 'info',
message: `路由变更为 ${routeEntry.route}`,
data: {
route: routeEntry.route
pushSignalEventOnce({
source: 'route-change',
event: {
ts: routeEntry.ts,
type: 'signal',
name: 'route-change',
status: 'info',
message: `路由变更为 ${routeEntry.route}`,
data: {
route: routeEntry.route
}
}
})
}
for (const ipcEntry of signals.ipcInvokeEntries) {
pushEvent({
ts: ipcEntry.ts,
type: 'signal',
name: 'ipc-call',
status: 'info',
message: `调用 IPC ${ipcEntry.name}`,
data: {
name: ipcEntry.name
pushSignalEventOnce({
source: 'ipc-call',
event: {
ts: ipcEntry.ts,
type: 'signal',
name: 'ipc-call',
status: 'info',
message: `调用 IPC ${ipcEntry.name}`,
data: {
name: ipcEntry.name
}
}
})
}
for (const aiRunEventEntry of signals.aiRunEventEntries) {
pushEvent({
ts: aiRunEventEntry.ts,
type: 'signal',
name: 'ai-run-event',
status: 'info',
message: `收到 AI 事件 ${aiRunEventEntry.event.type}`,
data: {
eventType: aiRunEventEntry.event.type
appendRuntimeEventOnce(aiRunEventEntry)
pushSignalEventOnce({
source: 'ai-run-event',
event: {
ts: aiRunEventEntry.ts,
type: 'signal',
name: 'ai-run-event',
status: 'info',
message: `收到 AI 事件 ${aiRunEventEntry.event.type}`,
data: {
eventType: aiRunEventEntry.event.type
}
}
})
}
@@ -468,41 +619,49 @@ export const runE2ECase = async <TResult>(
if (consoleEntry.level === 'log') {
continue
}
pushEvent({
ts: consoleEntry.ts,
type: 'signal',
name: consoleEntry.level === 'warning' ? 'console-warning' : 'console-error',
status: 'info',
message: `捕获到 console ${consoleEntry.level}:${consoleEntry.text}`,
data: {
level: consoleEntry.level,
text: consoleEntry.text
pushSignalEventOnce({
source: 'console',
event: {
ts: consoleEntry.ts,
type: 'signal',
name: consoleEntry.level === 'warning' ? 'console-warning' : 'console-error',
status: 'info',
message: `捕获到 console ${consoleEntry.level}:${consoleEntry.text}`,
data: {
level: consoleEntry.level,
text: consoleEntry.text
}
}
})
}
for (const lifecycleEntry of signals.lifecycleEntries) {
pushEvent({
ts: lifecycleEntry.ts,
type: 'signal',
name: 'electron-lifecycle',
status: 'info',
message: `Electron lifecycle: ${lifecycleEntry.name}`,
data: lifecycleEntry.data
pushSignalEventOnce({
source: 'electron-lifecycle',
event: {
ts: lifecycleEntry.ts,
type: 'signal',
name: 'electron-lifecycle',
status: 'info',
message: `Electron lifecycle: ${lifecycleEntry.name}`,
data: lifecycleEntry.data
}
})
}
const traceRelativePath = path.posix.join('trace', 'playwright-trace.zip')
const traceAbsolutePath = path.join(logDirArtifacts, traceRelativePath)
await mkdir(path.dirname(traceAbsolutePath), { recursive: true })
await launchedApp.close({
tracePath: traceAbsolutePath
})
await captureArtifact({
kind: 'trace',
relativePath: traceRelativePath,
message: '保存 Playwright trace'
})
if (launchedApp) {
const traceRelativePath = path.posix.join('trace', 'playwright-trace.zip')
const traceAbsolutePath = path.join(logDirArtifacts, traceRelativePath)
await mkdir(path.dirname(traceAbsolutePath), { recursive: true })
await launchedApp.close({
tracePath: traceAbsolutePath
})
await captureArtifact({
kind: 'trace',
relativePath: traceRelativePath,
message: '保存 Playwright trace'
})
}
if (status === 'failed') {
const reportRelativePath = path.posix.join('playwright', 'report', 'index.html')
@@ -558,10 +717,8 @@ export const runE2ECase = async <TResult>(
}
}
await writeRunLog({
await streamingLogWriter.finalize({
meta,
events,
runtimeEvents,
artifacts
})
}
+97 -1
View File
@@ -954,6 +954,25 @@ export const sendWorkbenchMessage = async (page: Page, value: string): Promise<v
await submitWorkbenchCurrentDraft(page)
}
export const createWorkbenchNewSession = async (page: Page): Promise<void> => {
await getWorkbenchLocator(page, workbenchSelectors.newSessionButton).click()
await getWorkbenchLocator(page, workbenchSelectors.input).waitFor({ state: 'visible' })
await page.waitForFunction(
(input: { assistantTurnPrefix: string; approvalBatchCard: string }) => {
const assistantTurns = document.querySelectorAll(
`[data-testid^="${input.assistantTurnPrefix}-"]`
)
const approvalCard = document.querySelector(`[data-testid="${input.approvalBatchCard}"]`)
return assistantTurns.length === 0 && !approvalCard
},
{
assistantTurnPrefix: workbenchSelectors.assistantTurnPrefix,
approvalBatchCard: chatSelectors.approvalBatchCard
},
{ timeout: WORKBENCH_E2E_TIMEOUT_MS }
)
}
export const switchWorkbenchMode = async (
page: Page,
modeLabel: '大聪明' | '没头脑'
@@ -1315,6 +1334,73 @@ export const waitForWorkbenchAiEvent = async (
}
}
export const decideWorkbenchApprovalBatch = async (
runtime: E2ETestRuntime,
page: Page,
input: {
event: Extract<AiRunEvent, { type: 'approval_batch_required' }>
action: 'approve' | 'reject'
timeoutMs?: number
}
): Promise<Extract<AiRunEvent, { type: 'approval_batch_decision' }>> => {
const card = getApprovalCard(page)
const buttonTestId =
input.action === 'approve'
? chatSelectors.approvalBatchApproveButton
: chatSelectors.approvalBatchDenyButton
const button = getWorkbenchLocator(page, buttonTestId)
await card.waitFor({ state: 'visible', timeout: WORKBENCH_E2E_TIMEOUT_MS })
for (let index = 0; index < input.event.items.length; index += 1) {
const cardText = await readLocatorText(card)
runtime.signal({
name: 'workbench-approval-decision-click',
message: `提交审批决策 ${index + 1}/${input.event.items.length}`,
data: {
action: input.action,
batchId: input.event.batchId,
item: input.event.items[index] || null,
cardText
}
})
await button.waitFor({ state: 'visible', timeout: WORKBENCH_E2E_TIMEOUT_MS })
await button.scrollIntoViewIfNeeded()
await page.waitForFunction(
(testId: string) => {
const element = document.querySelector(`[data-testid="${testId}"]`)
return element instanceof HTMLButtonElement && !element.disabled
},
buttonTestId,
{ timeout: WORKBENCH_E2E_TIMEOUT_MS }
)
await button.click()
if (index < input.event.items.length - 1) {
await page.waitForFunction(
(testId: string, expectedIndex: number) => {
const root = document.querySelector(`[data-testid="${testId}"]`)
return Boolean(root?.textContent?.includes(`审批 ${expectedIndex} /`))
},
chatSelectors.approvalBatchCard,
index + 2,
{ timeout: WORKBENCH_E2E_TIMEOUT_MS }
)
}
}
const decision = await waitForWorkbenchAiEvent(runtime, {
name: `approval_batch_decision_${input.action}_${input.event.batchId}`,
timeoutMs: input.timeoutMs || WORKBENCH_E2E_TIMEOUT_MS,
matches: (event) =>
event.type === 'approval_batch_decision' &&
event.batchId === input.event.batchId &&
event.decisions.some((item) => item.action === input.action)
})
if (decision.type !== 'approval_batch_decision') {
throw new Error(`approval batch decision did not resolve for ${input.event.batchId}`)
}
return decision
}
export const waitForWorkbenchAiRunOutcome = async (
runtime: E2ETestRuntime,
startedAtMs: number,
@@ -1365,7 +1451,17 @@ export const waitForWorkbenchAiRunOutcome = async (
event.type === 'approval_batch_required'
)
.at(-1)
if (approvalRequired && !approvalAutoHandledBatchIds.has(approvalRequired.batchId)) {
const approvalAlreadyDecided = approvalRequired
? events.some(
(event) =>
event.type === 'approval_batch_decision' && event.batchId === approvalRequired.batchId
)
: false
if (
approvalRequired &&
!approvalAlreadyDecided &&
!approvalAutoHandledBatchIds.has(approvalRequired.batchId)
) {
const assistantTextSoFar = await readAssistantTextSoFar(page)
const cardText = await readLocatorText(getApprovalCard(page))
const blocked: WorkbenchRunBlockedResult = {