From 5dfbec24ad1c0b2f1717d91bbc36a056c19cda62 Mon Sep 17 00:00:00 2001 From: hy Date: Mon, 21 Sep 2026 13:59:56 +0800 Subject: [PATCH] feat: add ask_user_question tool for user interactions - Introduced ASK_USER_QUESTION_TOOL_SPEC and related schemas for user questions. - Implemented agentControlToolFamily to include the new ask_user_question tool. - Enhanced AiRunStore to handle pending interactions and updated database schema. - Created HumanInteractionCard component for rendering user questions in the UI. - Updated ChatPage to manage human interaction states and submit answers. - Added tests for ask_user_question tool contract to ensure validation and constraints. --- doc/ai-harness/human-harness.md | 8 +- doc/ai-harness/tool-design.md | 1 + doc/deploy/release.md | 3 +- doc/prd/workbench.md | 5 +- doc/test-plan/ai-test.md | 6 + doc/test-plan/workbench-test.md | 6 + src/main/ipc/handlers.ts | 12 +- src/main/services/ai/ChatRunService.ts | 149 +++++++++++++++++- src/main/services/ai/human/constants.ts | 1 + .../services/ai/human/graph/humanGraph.ts | 7 + .../services/ai/human/node/dispatch/index.ts | 17 +- .../human/node/humanInteractionNode/index.ts | 54 +++++++ .../human/node/utils/stateTransitionState.ts | 8 +- src/main/services/ai/human/schema.ts | 8 +- .../ai/human/tools/agentControl/index.ts | 15 +- .../human/tools/agentControl/toolSchemas.ts | 27 ++++ .../ai/human/tools/agentControl/toolSpecs.ts | 26 ++- src/main/services/ai/runtime.ts | 4 + src/main/services/ai/types.ts | 16 ++ .../services/storage/domains/AiRunStore.ts | 19 ++- .../services/storage/sqlite/SqliteCore.ts | 6 +- .../013_ai_run_pending_interaction.ts | 7 + .../migrations/released-migrations.json | 8 +- src/preload/api.ts | 5 + src/preload/index.ts | 4 + .../workbench/AssistantTurnCard.tsx | 59 ++----- .../workbench/HumanInteractionCard.tsx | 75 +++++++++ .../workbench/messageLogProjection.ts | 13 +- .../src/components/workbench/types.ts | 4 + .../src/components/workbench/utils.ts | 1 + src/renderer/src/pages/ChatPage.tsx | 67 +++++++- src/renderer/src/testing/selectors.ts | 1 + src/shared/assistantExecutionSummary.ts | 3 +- src/shared/runtimeCapabilities.ts | 1 + src/shared/types.ts | 78 ++++++++- .../workbench/conversationProjection.ts | 2 +- .../ask-user-question-tool-contract.test.ts | 19 +++ 37 files changed, 663 insertions(+), 82 deletions(-) create mode 100644 src/main/services/ai/human/node/humanInteractionNode/index.ts create mode 100644 src/main/services/storage/sqlite/migrations/013_ai_run_pending_interaction.ts create mode 100644 src/renderer/src/components/workbench/HumanInteractionCard.tsx create mode 100644 tests/case/unit/ai/ask-user-question-tool-contract.test.ts diff --git a/doc/ai-harness/human-harness.md b/doc/ai-harness/human-harness.md index de4274a7..f2c0a24c 100644 --- a/doc/ai-harness/human-harness.md +++ b/doc/ai-harness/human-harness.md @@ -43,7 +43,7 @@ graph 从 `context_budget` 进入；首次进入才写本轮 `user`。 | `dispatch` | 否 | 否 | 消费 `tool_calls` 队列，路由下一 stage | | `learning` | 否 | `learning_*` + `tool` + **`skill_state_updated`** | 加载 skill 工具；**不**改 build user | | `plan` | 否 | `tool` + `plan_updated` | 更新 state.plan；**不**改 build user | -| `hand` | 否 | `tool`（+ 可选 `images[]`）+ 审批后 `approval_batch_resolved` | append ToolMessage；`read_image` 后 append 多模态 HumanMessage | +| `hand` / `human_interaction` | 否 | `tool`（+ 可选 `images[]`）+ `human_interaction_resolved` annotation | append ToolMessage；`read_image` 后 append 多模态 HumanMessage；问题答案作为结构化 tool result 回到 brain | 交付收尾由 brain 内的 `finalizeDelivery` 完成（**不是**独立图节点）：docx/ppt 落盘 side effect → `finalizing_started` → 写交付 `assistant`（`toolCalls: []`）→ 立即 `answer_completed` / `run_completed`。下一轮同伙伴发送可立刻创建。 无 `tool_calls` 的正常交付路径上，**brain 不向 log 写 `assistant`**（流式 Answer 只走事件 / `partial_answer`）。 @@ -128,7 +128,7 @@ append-only 伙伴消息转录；模型上下文、UI、resume 均由此投影 | `plan_updated` | `{ plan: [{ title, status }] }` | UI 计划条；新 run 从图启动扫描该伙伴剩余 log 最新一条 | | `compaction_checkpoint` | `{ compactBlockId, throughSeq, sourceMessageCount, summary }` | 压缩书签 + 摘要正文；`seq <= historyFloorSeq` 的原文不再进 brain；发给主模型时再合成 Compact User | | `finalizing_started` | `{}` | 投影时**清空** plan | -| `approval_batch_resolved` | `{ batchId, items: [{ toolCallId, toolName, command, commandExplanation?, action, message? }] }` | UI Activity：走过审批的工具行图标 / 悬停揭示文案；`hand` 在审批决策后写入 | +| `human_interaction_resolved` | `{ interactionId, kind, result }` | UI Activity：在对应工具行下按顺序展示问题答案或审批结果 | #### `entry_json` 示例 @@ -277,11 +277,11 @@ append-only 伙伴消息转录；模型上下文、UI、resume 均由此投影 | 字段（要点） | 用途 | |--------------|------| -| `status` | `running` / `waiting_approval` / `completed` / `failed` / `cancelled` | +| `status` | `running` / `waiting_human` / `completed` / `failed` / `cancelled` | | `input` | 本轮用户输入原文 | | `partial_answer` / `partial_answer_role` | 流式 UI 草稿；**不参与** resume 与下次 brain | | `final_answer` / `error` / `failure_code` / `failure_raw_message` | 终态展示（短标题 + 原文详情） | -| `pending_approval` | 等待批量审批 | +| `pending_interaction` | 等待用户问题回答或批量审批；`full_access` 只自动通过审批，不自动回答问题 | | `artifacts` | run 级产物引用（Workbench 展示） | | `selectedReferences` / `attachments` | run 创建时 Composer 快照（与 log `user.context` 对应） | diff --git a/doc/ai-harness/tool-design.md b/doc/ai-harness/tool-design.md index 84bcda63..aafaef27 100644 --- a/doc/ai-harness/tool-design.md +++ b/doc/ai-harness/tool-design.md @@ -135,6 +135,7 @@ Terminal：`terminal_read` 输出上限与 inline 字符预算挂钩（见 termi | `learn_skill` | 由 `learning` 节点直接写 ToolMessage（loaded / duplicate / unknown / evicted 摘要），不走 `buildModelToolResultDelivery` | | `update_plan` | 由 `plan` 节点直接写 ToolMessage（计划项列表 `[status] title`），不走 `buildModelToolResultDelivery` | | `read_image` | 路径列表 + 尺寸；`images[]` 进 log 供多模态（human-harness 第 3.5 节） | +| `ask_user_question` | 只询问会改变执行结果的信息；每批 1 至 3 个问题，选项具体互斥，用户始终可选择“其他，请填写”；问题调用不得与副作用工具混用。答案通过 tool result 回到 brain。 | ### 6.2 写工具输出 diff --git a/doc/deploy/release.md b/doc/deploy/release.md index 5fbcc4f9..00f20bbe 100644 --- a/doc/deploy/release.md +++ b/doc/deploy/release.md @@ -75,11 +75,10 @@ prepared → ready → published git diff .. ``` -评估版本实际改动涉及的产品能力、数据升级、平台构建、安装更新和跨模块风险，然后从正式包 E2E 回归用例中确定范围： +评估版本实际改动涉及的产品能力、数据升级、平台构建、安装更新和跨模块风险，然后从正式包 E2E 回归用例中确定回归测试范围： - 影响边界可以可靠收敛时，只执行受影响能力对应的用例。 - 影响范围较广，或无法可靠判断是否遗漏时，执行全部发版回归用例。 -- 修改了 SQLite schema 时，至少包含 `sidebar-app-update`、`database-version-update-required`，并使用上一公开 schema 的可审查 fixture 验证原地升级。 - 评估结论必须记录版本、影响范围、实际执行的用例、结果和剩余风险。 - 同一次发布中，已通过且后续修复未波及的用例不要重跑；每次跑正式包 E2E 回归时，应仔细评估改动影响范围来跑对应用例，禁止偷懒直接全量重跑。 diff --git a/doc/prd/workbench.md b/doc/prd/workbench.md index 18b8e333..f2a27bea 100644 --- a/doc/prd/workbench.md +++ b/doc/prd/workbench.md @@ -473,13 +473,14 @@ ### 8.5 AI 运行中的阻塞与提示 -#### WB-BLOCKER-001 审批阻塞 +#### WB-BLOCKER-001 人机交互阻塞 - 系统行为： + - AI 提问与审批共用 `waiting_human` 和 composer 上方交互卡片。提问支持单选、多选及始终可用的“其他，请填写”；一批最多 3 个问题，回答在 tool result 中回到 AI。 - 审核权限下，AI 运行收到批量审批事件后，在输入框上方展示紧贴 composer 的审批卡片；卡片一次只展示当前一条审批项，用户可通过上下切换浏览整批，并且必须先完成整批审核后系统才会继续决策是否执行。 - 审批卡片不展示内部策略原因，命令文本与同意/拒绝图标按钮左右排列，指导输入框与提交按钮同排展示；审批不自动超时，必须等待用户明确操作或运行被取消/关闭。 - 等待审批期间，当前 assistant turn 的 Activity header 显示「等待用户审核中」；审核结束后 header 恢复为常规工具汇总短句。 - - 审完后，本批走过审批的工具行在 Activity 展开列表文案前仅加结果图标（同意绿勾、拒绝红叉、指导琥珀色对话气泡），不额外包灰底卡片；悬停图标时原工具文案隐藏并滑入揭示文案：同意/拒绝显示 `commandExplanation`（缺省回退 `command`），指导显示 `{commandExplanation} · 已反馈：{指导语}`。 + - Activity 展开后，在对应工具行下按时间顺序显示问题、选项文字、自定义答案和审批结果；不再使用审核图标前缀或 hover 替换工具文案。 - 不在过程流展示只读 blockers 提示，也不弹出「有 n 条命令需要审核 / 审核已通过」toast。决策结果写入 `chat_session_message_log` 的 `approval_batch_resolved` annotation，刷新后可恢复。 - 完全访问权限下，统一审批入口直接批准该 run 的所有审批项，不创建等待审批状态、不发送审批事件、不展示审批卡片与审批叙事。完全访问只跳过审批，不绕过命令安全限制、取消、工作地址内文件变更记录或其他运行时保护。 - 验收： diff --git a/doc/test-plan/ai-test.md b/doc/test-plan/ai-test.md index 3d1fd129..e5a3c23c 100644 --- a/doc/test-plan/ai-test.md +++ b/doc/test-plan/ai-test.md @@ -571,6 +571,12 @@ DOCX tool 回归以 7 个公开工具、story 内容树、稳定 `nodeId`、segm #### human hand 类别 +##### `human-ask-user-question` + +- 测试类型：`backend / mock-llm` +- What：提问工具校验 1 至 3 个问题，支持单选、多选和自定义答案，并在回答前阻止副作用工具。 +- Oracle：run 进入 `waiting_human`；回答前没有写操作；结构化 `{ answers: [...] }` tool result 回到下一轮 brain；同一 run 不能有两个 pending interaction；`full_access` 不会自动回答。 + ##### `hand-invalid-tool-input` - 测试类型：`backend` diff --git a/doc/test-plan/workbench-test.md b/doc/test-plan/workbench-test.md index 0408a861..657f2b5c 100644 --- a/doc/test-plan/workbench-test.md +++ b/doc/test-plan/workbench-test.md @@ -600,6 +600,12 @@ pnpm test:update:e2e - How：真实 AI 修改一个 Markdown 文件并打开审核模式；折叠面板后创建第二个伙伴，再切回原伙伴，等待异步记录刷新窗口；确认面板保持折叠，再手动展开并检查审核内容与文件 Tab。 - Oracle：切回后 shell 不存在且仅保留右缘展开按钮，对齐 `WB-PANEL-001-AC5`；手动展开后展示目标文件的最新审核内容且文件 Tab 保留，对齐 `WB-PANEL-001-AC1`。 +#### `human-interaction-question-card` + +- 测试类型：`e2e / live` +- What：模糊任务触发 AI 提问，用户完成 A/B/C/D、其他输入和多问题切换；回答前不显示写入结果，提交后 run 恢复。 +- Oracle：composer 上方出现统一交互卡片；刷新后 pending 问题可恢复；Activity 对应工具行下显示问题与答案，审批结果以文字显示且没有旧图标前缀或 hover 替换。 + ### 5.4 `scheduled-task` 定时任务测试单独归类，Oracle 引用 [`workbench.md`](../prd/workbench.md) 的 `ST-*` 编号。调度触发时点由真实程序运行与系统时间决定，case 通过「下一分钟」级别的短周期调度验证端到端触发，不伪造系统时钟。下列 AC 由 unit `tests/case/unit/scheduled-task-service.test.ts` 承担，不进入 Workbench E2E 发现范围：`ST-PARTNER-001-AC2`、`ST-RUN-001-AC3`、`ST-RUN-001-AC4`、`ST-RUN-003-AC1/AC2`、`ST-RUN-004-AC2/AC3`、`ST-RUN-005-AC1/AC2/AC3`。`ST-PARTNER-001-AC1`（删除专属伙伴后任务仍在）标为不测。 diff --git a/src/main/ipc/handlers.ts b/src/main/ipc/handlers.ts index 08e68882..0e6e998f 100644 --- a/src/main/ipc/handlers.ts +++ b/src/main/ipc/handlers.ts @@ -39,7 +39,9 @@ import type { ScheduledTaskRunNowRequest, ScheduledTaskDismissBusyNoticeRequest, WriteTextFileRequest, - WriteTextFileResponse + WriteTextFileResponse, + AiRunHumanInteractionRequest, + AiRunHumanInteractionResponse } from '../../shared/types' import { listReferenceItemsPage } from '../services/sourceScope/referenceList' import { WorkspaceSourceService } from '../services/workspace/WorkspaceSourceService' @@ -2932,6 +2934,12 @@ export function registerHandlers(services: { } ) + ipcMain.handle( + 'ai-run-human-interaction-submit', + async (_event, payload: AiRunHumanInteractionRequest): Promise => + await chatRunService.submitHumanInteraction(payload) + ) + ipcMain.handle( 'docx-apply-get', async (_event, payload: GetDocxApplyRecordRequest): Promise => { @@ -3032,7 +3040,7 @@ export function registerHandlers(services: { const [partners, activePartnerIds, unreadCompletedPartnerIds, latestUserInputs] = await Promise.all([ partnerStore.listPartners(), - aiRunStore.listPartnerIdsByRunStatuses(['running', 'waiting_approval']), + aiRunStore.listPartnerIdsByRunStatuses(['running', 'waiting_human', 'waiting_approval']), partnerStore.listUnreadCompletedPartnerIds(), aiRunStore.listLatestUserInputs() ]) diff --git a/src/main/services/ai/ChatRunService.ts b/src/main/services/ai/ChatRunService.ts index 15f36d1f..fb413a91 100644 --- a/src/main/services/ai/ChatRunService.ts +++ b/src/main/services/ai/ChatRunService.ts @@ -46,7 +46,13 @@ import type { ChatTurnRetractLatestRequest, ChatTurnRetractLatestResponse, ChatSubmitRequest, - ChatSubmitResponse + ChatSubmitResponse, + AskUserQuestionResult, + AiRunHumanInteractionRequest, + AiRunHumanInteractionResponse, + PendingHumanInteraction, + UserQuestion, + JsonObject } from '../../../shared/types' import { projectAiConversationMessages } from '../../../shared/workbench/conversationProjection' @@ -67,6 +73,14 @@ type ActiveApproval = { ) => void } +type ActiveQuestion = { + interactionId: string + taskId: string + toolCallId: string + questions: UserQuestion[] + resolve: (result: { accepted: true; result: AskUserQuestionResult } | { accepted: false; error: string }) => void +} + type ActiveRun = { run: AiRunRecord cancelled: boolean @@ -106,7 +120,7 @@ type ChatRunServiceDeps = { ) => void | Promise } -const ACTIVE_RUN_STATUSES: AiRunRecord['status'][] = ['running', 'waiting_approval'] +const ACTIVE_RUN_STATUSES: AiRunRecord['status'][] = ['running', 'waiting_human', 'waiting_approval'] const INTERRUPTED_RUN_MESSAGE = '运行已中断，请重新发送消息。' const PARTIAL_SAVE_DELAY_MS = 500 @@ -129,6 +143,7 @@ const appendArtifact = (artifacts: AiArtifactRef[] | undefined, artifact: AiArti export class ChatRunService { private readonly activeRuns = new Map() private readonly activeApprovals = new Map() + private readonly activeQuestions = new Map() private readonly pendingPartnerSubmissions = new Set() public constructor(private readonly deps: ChatRunServiceDeps) {} @@ -175,7 +190,7 @@ export class ChatRunService { } public async failInterruptedRuns(): Promise { - const interrupted = await this.deps.aiRunStore.listRunsByStatuses(ACTIVE_RUN_STATUSES) + const interrupted = await this.deps.aiRunStore.listRunsByStatuses(['running']) for (const run of interrupted) { const now = Date.now() const nextRun: AiRunRecord = { @@ -186,6 +201,7 @@ export class ChatRunService { partialAnswer: run.partialAnswer || INTERRUPTED_RUN_MESSAGE, finalAnswer: run.finalAnswer || INTERRUPTED_RUN_MESSAGE, pendingApproval: undefined, + pendingInteraction: undefined, updatedAt: now, finishedAt: now } @@ -398,6 +414,51 @@ export class ChatRunService { const approval = this.activeApprovals.get(normalizedRunId) approval?.resolve({ accepted: false, error: '已取消' }) this.activeApprovals.delete(normalizedRunId) + const question = this.activeQuestions.get(normalizedRunId) + question?.resolve({ accepted: false, error: '已取消' }) + this.activeQuestions.delete(normalizedRunId) + return { success: true } + } + + public async submitHumanInteraction( + payload: AiRunHumanInteractionRequest + ): Promise { + const runId = String(payload?.runId || '').trim() + const pending = this.activeQuestions.get(runId) + if (!pending) return { success: false, error: '当前运行没有待回答问题' } + if (pending.interactionId !== payload.interactionId) { + return { success: false, error: 'interaction not found' } + } + const answers = Array.isArray(payload.answers?.answers) ? payload.answers.answers : [] + if (answers.length !== pending.questions.length) return { success: false, error: '每个问题都必须回答' } + const questionIds = new Set(pending.questions.map((question) => question.id)) + const seenAnswerIds = new Set() + for (const answer of answers) { + if (!answer || !questionIds.has(answer.id) || seenAnswerIds.has(answer.id) || !Array.isArray(answer.selected)) { + return { success: false, error: '答案格式无效' } + } + seenAnswerIds.add(answer.id) + const question = pending.questions.find((item) => item.id === answer.id) + const optionIds = new Set(question?.options.map((option) => option.id) || []) + if (answer.selected.some((id) => !optionIds.has(id))) return { success: false, error: '存在无效选项' } + if (question && !question.multiSelect && answer.selected.length > 1) { + return { success: false, error: '单选问题只能选择一个选项' } + } + if (answer.selected.length === 0 && !String(answer.custom || '').trim()) { + return { success: false, error: '请选择选项或填写其他内容' } + } + } + await this.queueRunEvent(runId, { + type: 'human_interaction_resolved', + runId, + taskId: pending.taskId, + interactionId: pending.interactionId, + kind: 'question', + result: payload.answers as unknown as JsonObject, + resolvedAt: Date.now() + }) + this.activeQuestions.delete(runId) + pending.resolve({ accepted: true, result: payload.answers }) return { success: true } } @@ -460,7 +521,8 @@ export class ChatRunService { ...entry.run, ...(persisted.run || {}), status: 'running', - pendingApproval: undefined + pendingApproval: undefined, + pendingInteraction: undefined } } @@ -479,7 +541,8 @@ export class ChatRunService { return { run, messageLogRecords: records, - pendingApproval: run.pendingApproval + pendingApproval: run.pendingApproval, + pendingInteraction: run.pendingInteraction } } @@ -651,6 +714,7 @@ export class ChatRunService { sendEvent, recordLlmIo: this.deps.recordLlmIo, requestApprovalBatch: async (request) => await this.requestApprovalBatch(runId, request), + requestUserQuestion: async (request) => await this.requestUserQuestion(runId, request), browserUseService: this.deps.browserUseService, webCrawlerService: this.deps.webCrawlerService, humanRuntimeOverrides: this.deps.resolveHumanRuntimeOverrides?.(run) @@ -727,6 +791,9 @@ export class ChatRunService { const approval = this.activeApprovals.get(runId) approval?.resolve({ accepted: false, error: 'run closed' }) this.activeApprovals.delete(runId) + const question = this.activeQuestions.get(runId) + question?.resolve({ accepted: false, error: 'run closed' }) + this.activeQuestions.delete(runId) await this.flushRunEvents(runId) this.clearProjectionFlush(runId) this.activeRuns.delete(runId) @@ -757,8 +824,8 @@ export class ChatRunService { decisions: request.items.map((item) => ({ itemId: item.itemId, action: 'approve' })) } } - if (this.activeApprovals.has(runId)) { - return { accepted: false, error: 'approval batch already pending' } + if (this.activeApprovals.has(runId) || this.activeQuestions.has(runId)) { + return { accepted: false, error: '已有待处理的人机交互' } } await this.queueRunEvent(runId, { @@ -804,6 +871,56 @@ export class ChatRunService { }) } + private async requestUserQuestion( + runId: string, + request: { + taskId: string + interactionId: string + toolCallId: string + questions: UserQuestion[] + stage?: AiRunEvent['stage'] + } + ): Promise<{ accepted: true; result: AskUserQuestionResult } | { accepted: false; error: string }> { + if (this.activeQuestions.has(runId) || this.activeApprovals.has(runId)) { + return { accepted: false, error: '已有待处理的人机交互' } + } + const interaction: PendingHumanInteraction = { + kind: 'question', + interactionId: request.interactionId, + toolCallId: request.toolCallId, + questions: request.questions + } + const resultPromise = new Promise< + { accepted: true; result: AskUserQuestionResult } | { accepted: false; error: string } + >((resolve) => { + this.activeQuestions.set(runId, { + interactionId: request.interactionId, + taskId: request.taskId, + toolCallId: request.toolCallId, + questions: request.questions, + resolve: (result) => { + this.activeQuestions.delete(runId) + resolve(result) + } + }) + }) + try { + await this.queueRunEvent(runId, { + type: 'human_interaction_required', + runId, + taskId: request.taskId, + interactionId: request.interactionId, + interaction, + createdAt: Date.now(), + stage: request.stage + }) + } catch (error) { + const pending = this.activeQuestions.get(runId) + pending?.resolve({ accepted: false, error: error instanceof Error ? error.message : String(error) }) + } + return await resultPromise + } + private queueRunEvent(runId: string, event: AiRunEvent): Promise { const entry = this.activeRuns.get(runId) if (entry?.deleted) return Promise.resolve() @@ -903,20 +1020,37 @@ export class ChatRunService { } else if (event.type === 'approval_batch_required') { run.status = 'waiting_approval' run.pendingApproval = event + run.pendingInteraction = { + kind: 'approval', + interactionId: event.batchId, + batchId: event.batchId, + items: event.items + } run.updatedAt = now } else if (event.type === 'approval_batch_decision') { run.status = 'running' run.pendingApproval = undefined + run.pendingInteraction = undefined + run.updatedAt = now + } else if (event.type === 'human_interaction_required') { + run.status = 'waiting_human' + run.pendingInteraction = event.interaction + run.updatedAt = now + } else if (event.type === 'human_interaction_resolved') { + run.status = 'running' + run.pendingInteraction = undefined run.updatedAt = now } else if (event.type === 'run_completed') { run.status = run.status === 'failed' || run.status === 'cancelled' ? run.status : 'completed' run.pendingApproval = undefined + run.pendingInteraction = undefined run.updatedAt = now run.finishedAt = now } else if (event.type === 'run_failed') { run.status = 'failed' run.failureCode = event.code run.pendingApproval = undefined + run.pendingInteraction = undefined run.error = event.message run.failureRawMessage = event.rawMessage run.updatedAt = now @@ -930,6 +1064,7 @@ export class ChatRunService { run.status = 'cancelled' run.failureCode = 'user_cancelled' run.pendingApproval = undefined + run.pendingInteraction = undefined run.error = '已取消' run.updatedAt = now run.finishedAt = now diff --git a/src/main/services/ai/human/constants.ts b/src/main/services/ai/human/constants.ts index 2b9f5968..7252017a 100644 --- a/src/main/services/ai/human/constants.ts +++ b/src/main/services/ai/human/constants.ts @@ -8,6 +8,7 @@ export const HUMAN_STAGES = [ 'learning', 'plan', 'hand', + 'interaction', 'context_budget', 'context_compaction' ] as const diff --git a/src/main/services/ai/human/graph/humanGraph.ts b/src/main/services/ai/human/graph/humanGraph.ts index 9a347353..c0ae3bd8 100644 --- a/src/main/services/ai/human/graph/humanGraph.ts +++ b/src/main/services/ai/human/graph/humanGraph.ts @@ -9,6 +9,7 @@ import { createContextBudgetNode } from '../node/contextBudgetNode' import { createLearningNode } from '../node/learningNode' import { createPlanNode } from '../node/planNode' import { createHandNode } from '../node/handNode' +import { createHumanInteractionNode } from '../node/humanInteractionNode' import { createHumanRuntimeCache } from '../node/utils/runtimeCache' import { advanceDispatchState } from '../node/dispatch' import { createHumanRunStateSchema, createInitialHumanState } from '../state/schema' @@ -50,6 +51,7 @@ export const createHumanGraphRunner = (input: { ctx: AiRunContext }) => { const learningNode = createLearningNode({ ctx, cache }) const planNode = createPlanNode({ ctx, cache }) const handNode = createHandNode({ ctx, cache }) + const humanInteractionNode = createHumanInteractionNode({ ctx, cache }) const routeToolReturn = (state: HumanState): string => { if (state.execution.stage === 'context_budget') return 'human_context_budget_node' @@ -74,6 +76,9 @@ export const createHumanGraphRunner = (input: { ctx: AiRunContext }) => { ) .addNode('human_plan_node', (state) => runNode(state as HumanState, planNode)) .addNode('human_hand_node', (state) => runNode(state as HumanState, handNode)) + .addNode('human_interaction_node', (state) => + runNode(state as HumanState, humanInteractionNode) + ) .addEdge(START, 'human_context_budget_node') .addConditionalEdges('human_context_budget_node', (state) => { const humanState = state as HumanState @@ -94,11 +99,13 @@ export const createHumanGraphRunner = (input: { ctx: AiRunContext }) => { if (humanState.execution.stage === 'learning') return 'human_learning_node' if (humanState.execution.stage === 'plan') return 'human_plan_node' if (humanState.execution.stage === 'hand') return 'human_hand_node' + if (humanState.execution.stage === 'interaction') return 'human_interaction_node' return 'human_context_budget_node' }) .addConditionalEdges('human_learning_node', (state) => routeToolReturn(state as HumanState)) .addConditionalEdges('human_plan_node', (state) => routeToolReturn(state as HumanState)) .addConditionalEdges('human_hand_node', (state) => routeToolReturn(state as HumanState)) + .addEdge('human_interaction_node', 'human_brain_node') .compile() const baseInitialState = options.initialState || createInitialHumanState(taskInput) diff --git a/src/main/services/ai/human/node/dispatch/index.ts b/src/main/services/ai/human/node/dispatch/index.ts index c7cf0c8c..ac42f75b 100644 --- a/src/main/services/ai/human/node/dispatch/index.ts +++ b/src/main/services/ai/human/node/dispatch/index.ts @@ -7,6 +7,7 @@ const classifyToolCallTarget = (toolName: string): HumanDispatchTarget => { const normalized = String(toolName || '').trim() if (normalized === 'learn_skill') return 'learning' if (normalized === 'update_plan') return 'plan' + if (normalized === 'ask_user_question') return 'interaction' return 'hand' } @@ -16,8 +17,11 @@ type RawToolCall = { args?: Record } -export const buildDispatchQueue = (toolCalls: RawToolCall[]): HumanDispatchItem[] => - toolCalls +export const buildDispatchQueue = (toolCalls: RawToolCall[]): HumanDispatchItem[] => { + if (toolCalls.some((call) => call.name.trim() === 'ask_user_question') && toolCalls.length !== 1) { + throw new Error('ask_user_question must be the only tool call in a model response') + } + return toolCalls .filter((call) => call.name.trim().length > 0) .map((call) => ({ toolCallId: call.id, @@ -25,10 +29,19 @@ export const buildDispatchQueue = (toolCalls: RawToolCall[]): HumanDispatchItem[ args: (call.args || {}) as JsonObject, target: classifyToolCallTarget(call.name.trim()) })) +} const dispatchGroupToNextAction = (group: HumanDispatchItem[]): HumanNextAction => { const first = group[0]! switch (first.target) { + case 'interaction': + if (group.length !== 1) throw new Error('ask_user_question must be called alone') + return { + action: 'interaction', + toolCallId: first.toolCallId, + toolName: 'ask_user_question', + args: first.args + } case 'learning': { const skillIds = Array.isArray(first.args.skillIds) ? first.args.skillIds.filter((item): item is string => typeof item === 'string') diff --git a/src/main/services/ai/human/node/humanInteractionNode/index.ts b/src/main/services/ai/human/node/humanInteractionNode/index.ts new file mode 100644 index 00000000..e519e867 --- /dev/null +++ b/src/main/services/ai/human/node/humanInteractionNode/index.ts @@ -0,0 +1,54 @@ +import { randomUUID } from 'node:crypto' +import type { JsonObject, UserQuestion } from '../../../../../../shared/types' +import type { AiRunContext } from '../../../types' +import type { HumanRuntimeCache, HumanState, ToolCallResult } from '../../types' +import { appendLogEntriesToCache } from '../../memory/conversationLogMessages' +import { returnToBrain } from '../utils/stateTransitions' + +export const createHumanInteractionNode = (input: { + ctx: AiRunContext + cache: HumanRuntimeCache +}) => { + return async (state: HumanState): Promise => { + input.ctx.checkCancelled() + const action = state.execution.pendingRound?.nextAction + if (!action || action.action !== 'interaction') return returnToBrain(state) + const questions = Array.isArray(action.args.questions) ? action.args.questions : [] + if (questions.length < 1 || questions.length > 3) { + throw new Error('ask_user_question requires 1 to 3 questions') + } + const interactionId = randomUUID() + const response = await input.ctx.requestUserQuestion({ + taskId: state.task.taskId, + interactionId, + toolCallId: action.toolCallId, + questions: questions as unknown as UserQuestion[] + }) + if (!response.accepted) throw new Error(response.error) + const result: ToolCallResult = { + toolName: 'ask_user_question', + ok: true, + data: response.result as unknown as JsonObject + } + const content = JSON.stringify(response.result) + appendLogEntriesToCache(input.cache, [ + { kind: 'tool', toolCallId: action.toolCallId, toolName: 'ask_user_question', ok: true, content }, + { + kind: 'annotation', + eventType: 'human_interaction_resolved', + payload: { interactionId, kind: 'question', result: response.result as unknown as JsonObject } + } + ]) + await input.ctx.appendMessageLog([ + { kind: 'tool', toolCallId: action.toolCallId, toolName: 'ask_user_question', ok: true, content }, + { + kind: 'annotation', + eventType: 'human_interaction_resolved', + payload: { interactionId, kind: 'question', result: response.result as unknown as JsonObject } + } + ]) + return returnToBrain(state, { + execution: { pendingRound: undefined, lastToolResults: [result] } + }) + } +} diff --git a/src/main/services/ai/human/node/utils/stateTransitionState.ts b/src/main/services/ai/human/node/utils/stateTransitionState.ts index 1f851f05..c8e99a92 100644 --- a/src/main/services/ai/human/node/utils/stateTransitionState.ts +++ b/src/main/services/ai/human/node/utils/stateTransitionState.ts @@ -93,7 +93,13 @@ export const returnToBrainAfterTraceState = (input: { } const resolveBrainNextStage = (action: HumanNextAction['action']): HumanState['execution']['stage'] => - action === 'learning' ? 'learning' : action === 'hand' ? 'hand' : 'plan' + action === 'learning' + ? 'learning' + : action === 'hand' + ? 'hand' + : action === 'interaction' + ? 'interaction' + : 'plan' export const applyBrainDecisionState = (input: { state: HumanState diff --git a/src/main/services/ai/human/schema.ts b/src/main/services/ai/human/schema.ts index 1f75d134..7b2ab9aa 100644 --- a/src/main/services/ai/human/schema.ts +++ b/src/main/services/ai/human/schema.ts @@ -107,6 +107,12 @@ export const humanReferenceSchema = brainEvidenceReferenceSchema.extend({ }) export const humanNextActionSchema = z.discriminatedUnion('action', [ + z.object({ + action: z.literal('interaction'), + toolCallId: z.string().min(1), + toolName: z.literal('ask_user_question'), + args: jsonObjectSchema + }), z.object({ action: z.literal('learning'), wantLearnSkillIds: z.array(z.string().min(1)).min(1), @@ -162,7 +168,7 @@ export const humanDispatchItemSchema = z.object({ toolCallId: z.string().min(1), toolName: z.string().min(1), args: jsonObjectSchema, - target: z.enum(['learning', 'plan', 'hand']) + target: z.enum(['learning', 'plan', 'hand', 'interaction']) }) export const humanExecutionSchema = z.object({ diff --git a/src/main/services/ai/human/tools/agentControl/index.ts b/src/main/services/ai/human/tools/agentControl/index.ts index e2fa6098..231ec07a 100644 --- a/src/main/services/ai/human/tools/agentControl/index.ts +++ b/src/main/services/ai/human/tools/agentControl/index.ts @@ -1,6 +1,11 @@ import { tool } from '@langchain/core/tools' import type { ToolFamilyDefinition } from '../types' -import { LEARN_SKILL_TOOL_SPEC, READ_IMAGE_TOOL_SPEC, UPDATE_PLAN_TOOL_SPEC } from './toolSpecs' +import { + ASK_USER_QUESTION_TOOL_SPEC, + LEARN_SKILL_TOOL_SPEC, + READ_IMAGE_TOOL_SPEC, + UPDATE_PLAN_TOOL_SPEC +} from './toolSpecs' import { prepareImagesForMainModel } from './readImage' // agent-control 工具只用于让模型通过原生 tool call 表达控制意图（学习技能、更新计划）， @@ -35,6 +40,12 @@ export const agentControlToolFamily: ToolFamilyDefinition = { } ) - return [learnSkill, updatePlan, readImage] + const askUserQuestion = tool(async () => ({}), { + name: ASK_USER_QUESTION_TOOL_SPEC.name, + description: ASK_USER_QUESTION_TOOL_SPEC.promptSpec.description, + schema: ASK_USER_QUESTION_TOOL_SPEC.inputSchema + }) + + return [learnSkill, updatePlan, readImage, askUserQuestion] } } diff --git a/src/main/services/ai/human/tools/agentControl/toolSchemas.ts b/src/main/services/ai/human/tools/agentControl/toolSchemas.ts index eeee4d42..86805581 100644 --- a/src/main/services/ai/human/tools/agentControl/toolSchemas.ts +++ b/src/main/services/ai/human/tools/agentControl/toolSchemas.ts @@ -17,6 +17,33 @@ export const READ_IMAGE_SCHEMA = z.object({ .describe('本地或远程图片路径，1 至 10 个') }) +export const ASK_USER_QUESTION_SCHEMA = z.object({ + questions: z + .array( + z.object({ + id: z.string().min(1).describe('问题的稳定标识'), + question: z.string().min(1).describe('需要用户回答的问题'), + options: z + .array( + z.object({ + id: z.string().min(1).describe('选项的稳定标识'), + label: z.string().min(1).describe('用户看到的选项文字'), + description: z.string().min(1).optional().describe('选项影响说明'), + recommended: z.boolean().optional().describe('是否为推荐方案') + }) + ) + .min(1), + multiSelect: z.boolean().optional().describe('是否允许同时选择多个选项') + }) + ) + .min(1) + .max(3) + .describe('最多 3 个会改变执行结果的问题') +}) + +export type AskUserQuestionInput = z.infer +export type AskUserQuestionResult = { answers: Array<{ id: string; selected: string[]; custom?: string }> } + export type { LearnSkillOutput, ReadImageItem, diff --git a/src/main/services/ai/human/tools/agentControl/toolSpecs.ts b/src/main/services/ai/human/tools/agentControl/toolSpecs.ts index ace01f3c..31cd58f6 100644 --- a/src/main/services/ai/human/tools/agentControl/toolSpecs.ts +++ b/src/main/services/ai/human/tools/agentControl/toolSpecs.ts @@ -9,9 +9,30 @@ import { import { LEARN_SKILL_SCHEMA, READ_IMAGE_SCHEMA, - UPDATE_PLAN_SCHEMA + UPDATE_PLAN_SCHEMA, + ASK_USER_QUESTION_SCHEMA } from './toolSchemas' +const ASK_USER_QUESTION_DESCRIPTION = + '向用户询问会实际改变任务定义或执行结果的关键信息。能从资料确定的信息不要询问；每批最多 3 个相关问题，选项要具体、互斥、可执行，并在有合理默认方案时标记 recommended。用户始终可以选择“其他，请填写”。必须单独调用，不能与写文件、运行命令等工具调用混在同一批。' + +export const ASK_USER_QUESTION_TOOL_SPEC: HumanToolSpec< + z.infer, + 'ask_user_question', + 'agent-control' +> = { + name: 'ask_user_question', + familyId: 'agent-control', + skillIds: ['agent-control'], + inputSchema: ASK_USER_QUESTION_SCHEMA, + promptSpec: { name: 'ask_user_question', description: ASK_USER_QUESTION_DESCRIPTION }, + resolveExecutionMeta: createStaticExecutionMetaResolver({ + mutability: 'read_only', + defaultResourceKey: 'agent-control' + }), + modelResult: { renderSuccess: (result) => JSON.stringify(result) } +} + export const LEARN_SKILL_TOOL_SPEC: HumanToolSpec< z.infer, 'learn_skill', @@ -75,5 +96,6 @@ export const READ_IMAGE_TOOL_SPEC: HumanToolSpec< export const AGENT_CONTROL_TOOL_SPECS = [ LEARN_SKILL_TOOL_SPEC, UPDATE_PLAN_TOOL_SPEC, - READ_IMAGE_TOOL_SPEC + READ_IMAGE_TOOL_SPEC, + ASK_USER_QUESTION_TOOL_SPEC ] as const diff --git a/src/main/services/ai/runtime.ts b/src/main/services/ai/runtime.ts index 739c0363..cb459f1a 100644 --- a/src/main/services/ai/runtime.ts +++ b/src/main/services/ai/runtime.ts @@ -119,6 +119,10 @@ export const createAiRunContext = (payload: AiRunStartPayload, deps: AiRunDeps): sendEvent: deps.sendEvent, recordLlmIo: deps.recordLlmIo, requestApprovalBatch: deps.requestApprovalBatch, + requestUserQuestion: deps.requestUserQuestion || (async () => ({ + accepted: false as const, + error: '当前运行不支持用户提问' + })), getLlmConfig: loadRequestedConfig, browserUseService: deps.browserUseService, webCrawlerService: deps.webCrawlerService, diff --git a/src/main/services/ai/types.ts b/src/main/services/ai/types.ts index 2fa3923b..7e861130 100644 --- a/src/main/services/ai/types.ts +++ b/src/main/services/ai/types.ts @@ -17,6 +17,8 @@ import type { z } from 'zod/v4' import type { AiApprovalBatchDecision, AiApprovalBatchItem, + AskUserQuestionInput, + AskUserQuestionResult, AiLlmIoRecord, AiLlmStreamingMode, AiLlmUsage, @@ -308,6 +310,13 @@ export type AiRunDeps = { sendEvent: (event: AiRunEvent) => void recordLlmIo?: (record: AiLlmIoRecord) => void requestApprovalBatch: (request: AiApprovalBatchRequest) => Promise + requestUserQuestion?: (request: { + taskId: string + interactionId: string + toolCallId: string + questions: AskUserQuestionInput['questions'] + stage?: AiRunEventStage + }) => Promise<{ accepted: true; result: AskUserQuestionResult } | { accepted: false; error: string }> llmDriver?: AiLlmDriver chatToolsWrapper?: (real: AiRunContext['chatTools']) => AiRunContext['chatTools'] browserUseService?: BrowserUseService @@ -367,6 +376,13 @@ export type AiRunContext = { sendEvent: (event: AiRunEvent) => void recordLlmIo?: (record: AiLlmIoRecord) => void requestApprovalBatch: (request: AiApprovalBatchRequest) => Promise + requestUserQuestion: (request: { + taskId: string + interactionId: string + toolCallId: string + questions: AskUserQuestionInput['questions'] + stage?: AiRunEventStage + }) => Promise<{ accepted: true; result: AskUserQuestionResult } | { accepted: false; error: string }> getLlmConfig: () => Promise browserUseService?: BrowserUseService webCrawlerService?: WebCrawlerService diff --git a/src/main/services/storage/domains/AiRunStore.ts b/src/main/services/storage/domains/AiRunStore.ts index f878cc2c..345f2bdf 100644 --- a/src/main/services/storage/domains/AiRunStore.ts +++ b/src/main/services/storage/domains/AiRunStore.ts @@ -38,6 +38,7 @@ type AiRunRow = { failure_code?: unknown failure_raw_message?: unknown pending_approval_json?: unknown + pending_interaction_json?: unknown artifacts_json?: unknown created_at?: unknown updated_at?: unknown @@ -48,6 +49,7 @@ const normalizeRunStatus = (value: unknown): AiRunStatus => { const status = String(value || '') if ( status === 'running' || + status === 'waiting_human' || status === 'waiting_approval' || status === 'completed' || status === 'failed' || @@ -82,6 +84,9 @@ const toRunRecord = (row: AiRunRow): AiRunRecord => ({ : undefined, failureRawMessage: row.failure_raw_message ? String(row.failure_raw_message) : undefined, pendingApproval: parseJsonObject(row.pending_approval_json), + pendingInteraction: parseJsonObject( + row.pending_interaction_json || row.pending_approval_json + ), artifacts: parseJsonArray(row.artifacts_json), createdAt: toRequiredNumber(row.created_at), updatedAt: toRequiredNumber(row.updated_at), @@ -100,13 +105,13 @@ export class AiRunStore { selected_references_json, attachments_json, model_name, main_model_json, execution_permission, partial_answer, partial_answer_role, final_answer, error, failure_code, failure_raw_message, - pending_approval_json, artifacts_json, created_at, updated_at, finished_at + pending_approval_json, pending_interaction_json, artifacts_json, created_at, updated_at, finished_at ) VALUES ( @id, @partner_id, @user_message_id, @assistant_message_id, @status, @input, @selected_references_json, @attachments_json, @model_name, @main_model_json, @execution_permission, @partial_answer, @partial_answer_role, @final_answer, @error, @failure_code, @failure_raw_message, - @pending_approval_json, @artifacts_json, @created_at, @updated_at, @finished_at + @pending_approval_json, @pending_interaction_json, @artifacts_json, @created_at, @updated_at, @finished_at ) ON CONFLICT(id) DO UPDATE SET partner_id = excluded.partner_id, @@ -126,6 +131,7 @@ export class AiRunStore { failure_code = excluded.failure_code, failure_raw_message = excluded.failure_raw_message, pending_approval_json = excluded.pending_approval_json, + pending_interaction_json = excluded.pending_interaction_json, artifacts_json = excluded.artifacts_json, updated_at = excluded.updated_at, finished_at = excluded.finished_at @@ -149,7 +155,8 @@ export class AiRunStore { error: record.error || null, failure_code: record.failureCode || null, failure_raw_message: record.failureRawMessage || null, - pending_approval_json: jsonStringify(record.pendingApproval), + pending_approval_json: jsonStringify(record.pendingInteraction || record.pendingApproval), + pending_interaction_json: jsonStringify(record.pendingInteraction), artifacts_json: jsonStringify(record.artifacts), created_at: record.createdAt, updated_at: record.updatedAt, @@ -281,7 +288,10 @@ export class AiRunStore { | undefined if (!runRow) return { ok: false, error: 'run not found' } const run = toRunRecord(runRow) - if (run.status !== 'waiting_approval' || !run.pendingApproval) { + if ( + (run.status !== 'waiting_human' && run.status !== 'waiting_approval') || + !run.pendingApproval + ) { return { ok: false, error: 'approval batch not found' } } if (run.pendingApproval.batchId !== input.request.batchId) { @@ -295,6 +305,7 @@ export class AiRunStore { UPDATE ai_run SET status = 'running', pending_approval_json = NULL, + pending_interaction_json = NULL, updated_at = @updated_at WHERE id = @id ` diff --git a/src/main/services/storage/sqlite/SqliteCore.ts b/src/main/services/storage/sqlite/SqliteCore.ts index 0727c9de..d6d0ddf8 100644 --- a/src/main/services/storage/sqlite/SqliteCore.ts +++ b/src/main/services/storage/sqlite/SqliteCore.ts @@ -15,6 +15,7 @@ import { repairAiRuntimeSystemNotificationsColumnMigration } from './migrations/ import { upgradeFromV9Migration } from './migrations/010_upgrade_from_v9' import { dropTerminalFileChangeRecordsMigration } from './migrations/011_drop_terminal_file_change_records' import { appSettingsMigration } from './migrations/012_app_settings' +import { aiRunPendingInteractionMigration } from './migrations/013_ai_run_pending_interaction' export type SqliteDatabase = Database.Database export type SqliteStatement = Database.Statement @@ -31,10 +32,11 @@ const MIGRATIONS: readonly Migration[] = [ repairAiRuntimeSystemNotificationsColumnMigration, upgradeFromV9Migration, dropTerminalFileChangeRecordsMigration, - appSettingsMigration + appSettingsMigration, + aiRunPendingInteractionMigration ] -export const CURRENT_SCHEMA_VERSION = 12 +export const CURRENT_SCHEMA_VERSION = 13 const MIGRATION_NAME_ALIASES = new Map>([ [2, new Set(['workspace_change_sets'])] diff --git a/src/main/services/storage/sqlite/migrations/013_ai_run_pending_interaction.ts b/src/main/services/storage/sqlite/migrations/013_ai_run_pending_interaction.ts new file mode 100644 index 00000000..82939971 --- /dev/null +++ b/src/main/services/storage/sqlite/migrations/013_ai_run_pending_interaction.ts @@ -0,0 +1,7 @@ +import type { Migration } from './types' + +export const aiRunPendingInteractionMigration: Migration = { + version: 13, + name: 'ai_run_pending_interaction', + sql: `ALTER TABLE ai_run ADD COLUMN pending_interaction_json TEXT;` +} diff --git a/src/main/services/storage/sqlite/migrations/released-migrations.json b/src/main/services/storage/sqlite/migrations/released-migrations.json index f864c737..39883071 100644 --- a/src/main/services/storage/sqlite/migrations/released-migrations.json +++ b/src/main/services/storage/sqlite/migrations/released-migrations.json @@ -1,5 +1,5 @@ { - "publishedThrough": 12, + "publishedThrough": 13, "migrations": [ { "version": 1, @@ -72,6 +72,12 @@ "name": "app_settings", "file": "012_app_settings.ts", "sha256": "0fd373bf26854cb7c6816cc60621242691b2fb9fd93200c40d33c8651e584230" + }, + { + "version": 13, + "name": "ai_run_pending_interaction", + "file": "013_ai_run_pending_interaction.ts", + "sha256": "e214a7f29b3354dd824eb94d581811dc8b8155b199348c7c0490d7be6500ba22" } ] } diff --git a/src/preload/api.ts b/src/preload/api.ts index a67f5186..076848bd 100644 --- a/src/preload/api.ts +++ b/src/preload/api.ts @@ -23,6 +23,8 @@ import type { BrowserTabState, AiRunApprovalBatchDecisionRequest, AiRunApprovalBatchDecisionResponse, + AiRunHumanInteractionRequest, + AiRunHumanInteractionResponse, AiRunCancelResponse, AiContextUsageRequest, AiContextUsageResponse, @@ -287,6 +289,9 @@ export type DesktopApi = { aiRunApprovalBatchDecide: ( payload: AiRunApprovalBatchDecisionRequest ) => Promise + aiRunHumanInteractionSubmit: ( + payload: AiRunHumanInteractionRequest + ) => Promise aiContextUsageGet: (payload: AiContextUsageRequest) => Promise browserPanelSetBounds: ( payload: BrowserPanelBoundsRequest diff --git a/src/preload/index.ts b/src/preload/index.ts index 67b9582b..68ec50dc 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -392,6 +392,10 @@ const api: DesktopApi = { emitE2EIpcCall('aiRunApprovalBatchDecide') return ipcRenderer.invoke('ai-run-approval-batch-decide', payload) }, + aiRunHumanInteractionSubmit: (payload) => { + emitE2EIpcCall('aiRunHumanInteractionSubmit') + return ipcRenderer.invoke('ai-run-human-interaction-submit', payload) + }, aiContextUsageGet: (payload) => ipcRenderer.invoke('ai-context-usage-get', payload), browserPanelSetBounds: (payload) => ipcRenderer.invoke('browser-panel-set-bounds', payload), browserPanelSetFollow: (payload) => ipcRenderer.send('browser-panel-set-follow', payload), diff --git a/src/renderer/src/components/workbench/AssistantTurnCard.tsx b/src/renderer/src/components/workbench/AssistantTurnCard.tsx index d17c46d6..8e9a40a9 100644 --- a/src/renderer/src/components/workbench/AssistantTurnCard.tsx +++ b/src/renderer/src/components/workbench/AssistantTurnCard.tsx @@ -1,7 +1,4 @@ import React, { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react' -import { FiCheckCircle } from 'react-icons/fi' -import { MdOutlineSmsFailed } from 'react-icons/md' -import { RiForbidLine } from 'react-icons/ri' import { FaSpider } from 'react-icons/fa' import { PiBrowsers, PiEyes } from 'react-icons/pi' import { @@ -35,7 +32,6 @@ import type { WorkspaceSource } from '../../../../shared/types' import type { OpenableMarkdownReference } from '../markdownReference' import { findApprovalReviewForProcessItem, - formatApprovalReviewRevealText, type AssistantApprovalReview } from '../../../../shared/workbench/approvalReview' import { workbenchSelectors } from '../../testing/selectors' @@ -377,29 +373,15 @@ const buildAssistantTimeline = (input: { return timeline } -const WAITING_APPROVAL_SUMMARY: AssistantRichTextSegment[] = [{ text: '等待用户审核中' }] - -const ApprovalReviewIcon: React.FC<{ action: AssistantApprovalReview['action'] }> = ({ - action -}) => { - if (action === 'approve') { - return - } - if (action === 'reject') { - return - } - return -} +const WAITING_APPROVAL_SUMMARY: AssistantRichTextSegment[] = [{ text: '等待用户处理' }] const AssistantActivityItemView: React.FC<{ item: AssistantProcessGroup['items'][number] review?: AssistantApprovalReview onPreviewImage?: (image: ImageLightboxImage) => void }> = ({ item, review, onPreviewImage }) => { - const [revealing, setRevealing] = useState(false) const failed = item.status === 'failed' || item.status === 'partial_failed' const toolText = getItemText(item) - const revealText = review ? formatApprovalReviewRevealText(review) : '' const readImagePaths = item.toolName === 'read_image' ? readImagePathsFromInput(item.input) : [] const thumbs = @@ -411,6 +393,7 @@ const AssistantActivityItemView: React.FC<{ return (
{renderSegments(toolText)} + {item.interaction ? : null} {thumbs}
) @@ -421,36 +404,24 @@ const AssistantActivityItemView: React.FC<{ data-testid={workbenchSelectors.assistantApprovalReviewedItem} className={failed ? 'text-[#B3261E]' : undefined} > -
- setRevealing(true)} - onMouseLeave={() => setRevealing(false)} - > - - - - - {renderSegments(toolText)} - - - {revealText} - - -
+
{renderSegments(toolText)}
+
审核结果：{review.action === 'approve' ? '已允许' : review.action === 'reject' ? '已拒绝' : `已提供指导${review.message ? `：${review.message}` : ''}`}
{thumbs} ) } +const InteractionResultView: React.FC<{ interaction: NonNullable }> = ({ interaction }) => ( +
+ {interaction.questions.map((question) => { + const answer = interaction.answers?.find((item) => item.id === question.id) + const labels = question.options.filter((option) => answer?.selected.includes(option.id)).map((option) => option.label) + if (answer?.custom) labels.push(answer.custom) + return
{question.question}
{labels.join('、') || '未填写'}
+ })} +
+) + const AssistantActivityGroupView: React.FC<{ groups: AssistantProcessGroup[] id: string diff --git a/src/renderer/src/components/workbench/HumanInteractionCard.tsx b/src/renderer/src/components/workbench/HumanInteractionCard.tsx new file mode 100644 index 00000000..b44472f7 --- /dev/null +++ b/src/renderer/src/components/workbench/HumanInteractionCard.tsx @@ -0,0 +1,75 @@ +import React, { useMemo, useState } from 'react' +import { Button, Input } from 'antd' +import { HiArrowUp, HiChevronLeft, HiChevronRight } from 'react-icons/hi2' +import type { PendingHumanInteraction, AskUserQuestionAnswer } from '../../../../shared/types' + +type Props = { + interaction: Extract + onSubmit: (answers: { answers: AskUserQuestionAnswer[] }) => Promise + onCancel: () => void +} + +const OTHER_ID = '__custom__' + +const HumanInteractionCard: React.FC = ({ interaction, onSubmit, onCancel }) => { + const [index, setIndex] = useState(0) + const [selected, setSelected] = useState>({}) + const [custom, setCustom] = useState>({}) + const [submitting, setSubmitting] = useState(false) + const question = interaction.questions[index] + const currentSelected = selected[question.id] || [] + const isCustom = currentSelected.includes(OTHER_ID) + const canSubmit = useMemo( + () => interaction.questions.every((item) => { + const values = selected[item.id] || [] + return values.length > 0 && (!values.includes(OTHER_ID) || Boolean(custom[item.id]?.trim())) + }), + [custom, interaction.questions, selected] + ) + const toggle = (id: string): void => { + setSelected((previous) => { + const values = previous[question.id] || [] + if (id === OTHER_ID) return { ...previous, [question.id]: values.includes(id) ? values.filter((value) => value !== id) : question.multiSelect ? [...values, id] : [id] } + if (!question.multiSelect) return { ...previous, [question.id]: [id] } + return { ...previous, [question.id]: values.includes(id) ? values.filter((value) => value !== id) : [...values.filter((value) => value !== OTHER_ID), id] } + }) + } + const submit = async (): Promise => { + if (!canSubmit || submitting) return + setSubmitting(true) + try { + await onSubmit({ + answers: interaction.questions.map((item) => { + const values = selected[item.id] || [] + return { id: item.id, selected: values.filter((value) => value !== OTHER_ID), custom: custom[item.id]?.trim() || undefined } + }) + }) + } finally { + setSubmitting(false) + } + } + return ( +
+
+
需要补充一点信息
+
+ + {index + 1} / {interaction.questions.length} + +
+
+
{question.question}
+
+ {question.options.map((option, optionIndex) => { + const active = currentSelected.includes(option.id) + return + })} + +
+ {isCustom ?
setCustom((previous) => ({ ...previous, [question.id]: event.target.value }))} placeholder="请填写你的要求" />
: null} +
+
+ ) +} + +export default HumanInteractionCard diff --git a/src/renderer/src/components/workbench/messageLogProjection.ts b/src/renderer/src/components/workbench/messageLogProjection.ts index 50db0476..49728ddf 100644 --- a/src/renderer/src/components/workbench/messageLogProjection.ts +++ b/src/renderer/src/components/workbench/messageLogProjection.ts @@ -101,6 +101,17 @@ export const projectProcessGroups = (input: { createdAt, sourceEventType: 'assistant_process_item_started' }) + const interaction = entry.toolName === 'ask_user_question' + ? (() => { + try { + const parsed = JSON.parse(entry.content) as { answers?: Array<{ id: string; selected: string[]; custom?: string }> } + const questions = Array.isArray((toolArgsById.get(entry.toolCallId) as { questions?: unknown })?.questions) + ? ((toolArgsById.get(entry.toolCallId) as { questions: Array<{ id: string; question: string; options: Array<{ id: string; label: string }> }> }).questions) + : [] + return { questions, answers: parsed.answers } + } catch { return undefined } + })() + : undefined const group = createDisplayProcessGroup({ groupId: `${input.runId}:tool:${entry.toolCallId}`, kind: 'skill', @@ -109,7 +120,7 @@ export const projectProcessGroups = (input: { createdAt, sourceEventType: 'assistant_process_group_started' }) - upsertGroup({ ...group, items: [item] }) + upsertGroup({ ...group, items: [interaction ? { ...item, interaction } : item] }) } } diff --git a/src/renderer/src/components/workbench/types.ts b/src/renderer/src/components/workbench/types.ts index 0169c6f7..7c7ccceb 100644 --- a/src/renderer/src/components/workbench/types.ts +++ b/src/renderer/src/components/workbench/types.ts @@ -127,6 +127,10 @@ export type AssistantProcessGroupItem = { toolName?: AssistantToolName targetLabel?: string input?: IpcJsonObject + interaction?: { + questions: Array<{ id: string; question: string; options: Array<{ id: string; label: string }> }> + answers?: Array<{ id: string; selected: string[]; custom?: string }> + } } export type AssistantProcessGroup = { diff --git a/src/renderer/src/components/workbench/utils.ts b/src/renderer/src/components/workbench/utils.ts index 812ef3c4..7d4123ff 100644 --- a/src/renderer/src/components/workbench/utils.ts +++ b/src/renderer/src/components/workbench/utils.ts @@ -375,6 +375,7 @@ const TOOL_ITEM_RENDERERS = { return toolCopy('操作浏览器', sentence(text('操作浏览器 '), param(instructionLabel))) }, read_image: () => toolCopy('读取图片', sentence(text('读取图片'))), + ask_user_question: () => toolCopy('询问用户', sentence(text('等待用户确认关键信息'))), image_generate: (input) => toolCopy('生成图片', sentence(text('生成图片 '), param(readToolFileLabel(input)))), learn_skill: (input, status) => { diff --git a/src/renderer/src/pages/ChatPage.tsx b/src/renderer/src/pages/ChatPage.tsx index 864e05f7..ae0a4a46 100644 --- a/src/renderer/src/pages/ChatPage.tsx +++ b/src/renderer/src/pages/ChatPage.tsx @@ -38,7 +38,10 @@ import type { ScheduledTask, SelectedReference, WorkbenchModelSelection, - WorkspaceSource + WorkspaceSource, + PendingHumanInteraction, + AskUserQuestionAnswer, + AiRunHumanInteractionRequest } from '../../../shared/types' import { PARTNER_AVATAR_SHAPES, PARTNER_AVATAR_COLORS } from '../../../shared/types' import { @@ -59,6 +62,7 @@ import { HiOutlineCog6Tooth } from 'react-icons/hi2' import AssistantTurnCard from '../components/workbench/AssistantTurnCard' import { FullAccessConfirmModal } from '../components/workbench/FullAccessConfirmModal' import WorkbenchComposer from '../components/workbench/WorkbenchComposer' +import HumanInteractionCard from '../components/workbench/HumanInteractionCard' import { buildApprovalGuidePlaceholder } from '../components/workbench/composer/constants' import { MessageTimeline } from '../components/workbench/MessageTimeline' import { @@ -457,6 +461,14 @@ const reduceAssistantTurnByRunEvent = ( } } + if (event.type === 'human_interaction_required' && event.interaction.kind === 'question') { + next = { ...next, isBrainThinking: false, waitingApproval: true } + } + + if (event.type === 'human_interaction_resolved' && event.kind === 'question') { + next = { ...next, waitingApproval: false } + } + if (event.type === 'assistant_brain_stream_started') { const retryRunning = next.processGroups.some( (group) => group.kind === 'retry' && group.status === 'running' @@ -642,7 +654,7 @@ const buildAssistantTurnFromRunSnapshot = ( processThoughts: projected.processThoughts, processGroups: projected.processGroups, isBrainThinking: projected.isBrainThinking, - waitingApproval: Boolean(snapshot.run.pendingApproval), + waitingApproval: Boolean(snapshot.run.pendingApproval || snapshot.run.pendingInteraction), pendingApprovalItems: snapshot.run.pendingApproval?.items || [], approvalReviews } @@ -877,6 +889,10 @@ const ChatPage: React.FC = () => { Record >({}) const [approvalSubmitting, setApprovalSubmitting] = useState(false) + const [pendingQuestion, setPendingQuestion] = useState<{ + runId: string + interaction: Extract + } | null>(null) const activeRunMessageIdsRef = useRef>({}) const activeRunPartnerIdsRef = useRef>({}) @@ -2002,6 +2018,12 @@ const ChatPage: React.FC = () => { activeRuns.find((item) => item.pendingApproval && !item.pendingApproval.internal) ?.pendingApproval || null setPendingApprovalBatch(approval) + const questionRun = activeRuns.find((item) => item.pendingInteraction?.kind === 'question') + setPendingQuestion( + questionRun?.pendingInteraction?.kind === 'question' + ? { runId: questionRun.run.id, interaction: questionRun.pendingInteraction } + : null + ) setApprovalCurrentIndex(0) setApprovalDraftDecisions({}) setApprovalSubmitting(false) @@ -3330,6 +3352,22 @@ const ChatPage: React.FC = () => { return } + if (event.type === 'human_interaction_required') { + if (event.interaction.kind === 'question') { + setLoading(false) + setPendingQuestion({ runId: event.runId, interaction: event.interaction }) + } + return + } + + if (event.type === 'human_interaction_resolved') { + if (event.kind === 'question') { + setPendingQuestion(null) + setLoading(true) + } + return + } + if (event.type === 'approval_batch_decision') { if (event.internal) return setApprovalSubmitting(false) @@ -3370,6 +3408,7 @@ const ChatPage: React.FC = () => { setAiRunId(null) setCancellingRun(false) setPendingApprovalBatch(null) + setPendingQuestion(null) setApprovalSubmitting(false) setCurrentMessages((previous) => previous.map((item) => @@ -3396,6 +3435,7 @@ const ChatPage: React.FC = () => { setAiRunId(null) setCancellingRun(false) setPendingApprovalBatch(null) + setPendingQuestion(null) setApprovalSubmitting(false) setCurrentMessages((previous) => previous.map((item) => @@ -3420,6 +3460,7 @@ const ChatPage: React.FC = () => { setAiRunId(null) setCancellingRun(false) setPendingApprovalBatch(null) + setPendingQuestion(null) setApprovalSubmitting(false) if (eventPartnerId && eventPartnerId === currentPartnerIdRef.current) { void loadFileChangesForPartnerRef.current(eventPartnerId, event.runId, { @@ -3576,6 +3617,7 @@ const ChatPage: React.FC = () => { scrollToBottom(true) setCancellingRun(false) setPendingApprovalBatch(null) + setPendingQuestion(null) setApprovalSubmitting(false) lastSubmittedInputRef.current = userQuery setPartners((items) => @@ -3832,6 +3874,21 @@ const ChatPage: React.FC = () => { setApprovalDraftDecisions({}) } + const submitHumanQuestion = async (answers: { answers: AskUserQuestionAnswer[] }): Promise => { + if (!pendingQuestion) return + const response = await window.api.aiRunHumanInteractionSubmit({ + runId: pendingQuestion.runId, + interactionId: pendingQuestion.interaction.interactionId, + answers + } satisfies AiRunHumanInteractionRequest) + if (!response.success) { + emitErrorMessage(response.error || '提交问题回答失败') + return + } + setPendingQuestion(null) + setLoading(true) + } + const flushApprovalBatch = async ( draftDecisions: Record ): Promise => { @@ -4714,6 +4771,12 @@ const ChatPage: React.FC = () => { ? '还没有伙伴。从左侧创建一个开始对话。' : '请选择一个伙伴开始对话。'} + ) : pendingQuestion ? ( + void handleCancel()} + /> ) : pendingApprovalBatch ? (
export const getAssistantToolUiKind = ( diff --git a/src/shared/runtimeCapabilities.ts b/src/shared/runtimeCapabilities.ts index bdd87ba6..c18b7657 100644 --- a/src/shared/runtimeCapabilities.ts +++ b/src/shared/runtimeCapabilities.ts @@ -48,6 +48,7 @@ export const RUNTIME_TOOL_CATALOG = { learn_skill: { familyId: 'agent-control', skillId: 'agent-control' }, update_plan: { familyId: 'agent-control', skillId: 'agent-control' }, read_image: { familyId: 'agent-control', skillId: 'agent-control' } + ,ask_user_question: { familyId: 'agent-control', skillId: 'agent-control' } } as const satisfies Record export type RuntimeToolName = keyof typeof RUNTIME_TOOL_CATALOG diff --git a/src/shared/types.ts b/src/shared/types.ts index 1b0eb1a6..fe80654f 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -1048,7 +1048,7 @@ export interface OpenFileWithApplicationResponse { error?: string } -export type AiRunStatus = 'running' | 'waiting_approval' | 'completed' | 'failed' | 'cancelled' +export type AiRunStatus = 'running' | 'waiting_human' | 'waiting_approval' | 'completed' | 'failed' | 'cancelled' export type AiRunWarningCode = | 'workspace_binding_missing' | 'workspace_missing' @@ -1203,6 +1203,52 @@ export type AiArtifactRef = AiFileArtifactRef export type AiPreference = 'default' | 'local_sources_relaxed' | 'local_sources_strict' export type AiExecutionPermission = 'review' | 'full_access' +export type UserQuestionOption = { + id: string + label: string + description?: string + recommended?: boolean +} + +export type UserQuestion = { + id: string + question: string + options: UserQuestionOption[] + multiSelect?: boolean +} + +export type AskUserQuestionInput = { questions: UserQuestion[] } + +export type AskUserQuestionAnswer = { + id: string + selected: string[] + custom?: string +} + +export type AskUserQuestionResult = { answers: AskUserQuestionAnswer[] } + +export type PendingHumanInteraction = + | { + kind: 'question' + interactionId: string + toolCallId: string + questions: UserQuestion[] + } + | { + kind: 'approval' + interactionId: string + items: AiApprovalBatchItem[] + batchId?: string + } + +export type AiRunHumanInteractionRequest = { + runId: string + interactionId: string + answers: AskUserQuestionResult +} + +export type AiRunHumanInteractionResponse = { success: boolean; error?: string } + export type ChatSubmitRequest = { preference?: AiPreference executionPermission?: AiExecutionPermission @@ -1286,6 +1332,7 @@ export type AiRunRecord = { failureCode?: AiRunFailureCode failureRawMessage?: string pendingApproval?: Extract + pendingInteraction?: PendingHumanInteraction artifacts?: AiArtifactRef[] createdAt: number updatedAt: number @@ -1296,6 +1343,7 @@ export type PartnerRunSnapshot = { run: AiRunRecord messageLogRecords?: ChatMessageLogRecord[] pendingApproval?: Extract + pendingInteraction?: PendingHumanInteraction } export type PartnerSnapshotRequest = { @@ -1860,6 +1908,25 @@ type AiRunEventPayload = type: 'run_cancelled' runId: string } + | { + type: 'human_interaction_required' + runId: string + taskId: string + interactionId: string + interaction: PendingHumanInteraction + createdAt: number + stage?: AiRunEventStage + } + | { + type: 'human_interaction_resolved' + runId: string + taskId: string + interactionId: string + kind: 'question' | 'approval' + result: JsonObject + resolvedAt: number + stage?: AiRunEventStage + } export type AiRunEvent = AiRunEventPayload & { internal?: boolean @@ -1936,6 +2003,15 @@ export type ChatMessageLogAnnotationEntry = }> } } + | { + kind: 'annotation' + eventType: 'human_interaction_resolved' + payload: { + interactionId: string + kind: 'question' | 'approval' + result: JsonObject + } + } export type ChatMessageLogEntry = | { diff --git a/src/shared/workbench/conversationProjection.ts b/src/shared/workbench/conversationProjection.ts index b67c0152..353297ed 100644 --- a/src/shared/workbench/conversationProjection.ts +++ b/src/shared/workbench/conversationProjection.ts @@ -7,7 +7,7 @@ import type { SelectedReference } from '../types' -const ACTIVE_RUN_STATUSES = new Set(['running', 'waiting_approval']) +const ACTIVE_RUN_STATUSES = new Set(['running', 'waiting_human', 'waiting_approval']) const isDeliveryAssistant = (record: ChatMessageLogRecord): boolean => record.entry.kind === 'assistant' && record.entry.toolCalls.length === 0 diff --git a/tests/case/unit/ai/ask-user-question-tool-contract.test.ts b/tests/case/unit/ai/ask-user-question-tool-contract.test.ts new file mode 100644 index 00000000..9f108769 --- /dev/null +++ b/tests/case/unit/ai/ask-user-question-tool-contract.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest' +import { ASK_USER_QUESTION_SCHEMA } from '../../../../src/main/services/ai/human/tools/agentControl/toolSchemas' +import { buildDispatchQueue } from '../../../../src/main/services/ai/human/node/dispatch' + +describe('ask_user_question contract', () => { + it('accepts minimal questions and rejects more than three', () => { + const question = { id: 'goal', question: '选择目标', options: [{ id: 'a', label: '方案 A' }] } + expect(ASK_USER_QUESTION_SCHEMA.safeParse({ questions: [question] }).success).toBe(true) + expect(ASK_USER_QUESTION_SCHEMA.safeParse({ questions: [question, question, question, question] }).success).toBe(false) + }) + + it('must be the only tool call in a model response', () => { + expect(() => buildDispatchQueue([{ id: 'q', name: 'ask_user_question', args: {} }])).not.toThrow() + expect(() => buildDispatchQueue([ + { id: 'q', name: 'ask_user_question', args: {} }, + { id: 'w', name: 'text_file_edit', args: {} } + ])).toThrow(/must be the only tool call/) + }) +})