From bb3a374b611ea63bf9ca01a617d0c19c4e971adb Mon Sep 17 00:00:00 2001 From: hy Date: Wed, 18 Mar 2026 16:26:54 +0800 Subject: [PATCH] Advance human live tests and simplify report evidence --- AGENTS.md | 2 +- doc/README.md | 2 +- doc/ai-workbench-test/todo-stage-2.md | 19 +- doc/test.md | 10 +- src/main/services/ai/node/human/brainNode.ts | 256 ++++++++++++++++++ .../services/ai/node/human/responseNode.ts | 12 +- src/main/services/ai/prompts/human/brain.ts | 6 +- tests/ai/backend/human/helpers.ts | 23 +- .../live/human-live-direct-kb-answer.test.ts | 167 ++++++++++++ ...ve-evidence-insufficient-safe-stop.test.ts | 93 +++++++ .../live/human-live-kb-docx-bridge.test.ts | 233 ++++++++++++++++ tests/report/ai/report.md | 122 +++++++-- 12 files changed, 890 insertions(+), 55 deletions(-) create mode 100644 tests/ai/backend/human/live/human-live-direct-kb-answer.test.ts create mode 100644 tests/ai/backend/human/live/human-live-evidence-insufficient-safe-stop.test.ts create mode 100644 tests/ai/backend/human/live/human-live-kb-docx-bridge.test.ts diff --git a/AGENTS.md b/AGENTS.md index 5b5ed182..8df51109 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,7 @@ - 当涉及数据库、LanceDB、表职责、存储分层或检索存储实现时,请阅读 `doc/database.md` - 当涉及目录规范、文件归位、模块边界或工程结构时,请阅读 `doc/project-structure.md` - 当涉及 UI 风格、布局规范或组件交互约束时,请阅读 `doc/ui-style.md` -- 当涉及测试相关内容,包括测试目录、raw log、report 规则或历史总数据 / 最新总数据口径时,请阅读 `doc/test.md`; +- 当涉及测试相关内容,包括测试目录、raw log、report 规则、证据展示方式或历史总数据 / 最新总数据口径时,请阅读 `doc/test.md`; ## 2. 注意事项 - 禁止使用 `any` 与 `unknown` diff --git a/doc/README.md b/doc/README.md index badaeed0..a819692a 100644 --- a/doc/README.md +++ b/doc/README.md @@ -18,4 +18,4 @@ - [ui-style.md](/Users/test/work-space/NCurator-Desktop/doc/ui-style.md) - 说明界面设计风格、布局规则和组件使用规范。 - [test.md](/Users/test/work-space/NCurator-Desktop/doc/test.md) - - 说明测试目录、测试用例设计准则,以及 report 中历史总数据 / 最新总数据的统计口径。 + - 说明测试目录、测试用例设计准则,以及 report 中证据展示和历史总数据 / 最新总数据的统计口径。 diff --git a/doc/ai-workbench-test/todo-stage-2.md b/doc/ai-workbench-test/todo-stage-2.md index 054b7cb3..ba5f7430 100644 --- a/doc/ai-workbench-test/todo-stage-2.md +++ b/doc/ai-workbench-test/todo-stage-2.md @@ -24,20 +24,24 @@ - [x] 落地 `human-long-task-with-corrections` - [x] 落地 `human-long-task-approval-branch` - [x] 落地 `human-live-direct-docx-edit` +- [x] 落地 `human-live-direct-kb-answer` +- [x] 落地 `human-live-evidence-insufficient-safe-stop` - [x] 落地 `human-live-terminal-docx-handoff` - [x] 落地 `human-live-long-task-project-brief` +- [x] 跑完 Stage 2 `backend/human` non-live 测试并刷新 `tests/logs/ai/backend/human/{basic,boundary,scenario}/**` ## In Progress -- [ ] 落地 `human-live-direct-kb-answer` - [ ] 落地 `human-live-kb-docx-bridge` -- [ ] 落地 `human-live-evidence-insufficient-safe-stop` -- [ ] 跑完 Stage 2 backend/human 测试并刷新 `tests/logs/ai/**` -- [ ] 基于新的 human logs 更新 `tests/report/ai/report.md` +- [x] 基于新的 human logs 更新 `tests/report/ai/report.md` ## Blocked -- 当前无新增外部阻塞 +- `human-live-kb-docx-bridge` 在 2026-03-18 的 primary `gpt-5` live 运行里仍未形成稳定交付: + - `20260318-150310-015`:`invalid_input`,summary=`524 status code (no body)`,只留下两次 `kb_hybrid_search_chunks` +- 为区分模型/服务问题与编排问题,已在 2026-03-18 补跑 stronger `gpt-5.2`: + - `20260318-150906-789`:在 `kb_hybrid_search_chunks -> kb_list_documents` 后判断剩余步数不足,未进入 docx handoff + - `20260318-151207-536`:`bounded_loop_error`,错误转入多次 `terminal_run_command`,仍未产出 preview artifact - `human` 阶段 live 结论仍不完整,Stage 2 尚不能仅凭 non-live case 视为完成 ## Evidence @@ -45,6 +49,11 @@ - `doc/ai-workbench-test/plan.md` - `tests/ai/backend/human/{basic,boundary,scenario,live}/*.test.ts` - `tests/logs/ai/backend/human/**` +- `tests/logs/ai/backend/human/live/human-live-direct-kb-answer/20260318-145113-697/*` +- `tests/logs/ai/backend/human/live/human-live-evidence-insufficient-safe-stop/20260318-150735-678/*` +- `tests/logs/ai/backend/human/live/human-live-kb-docx-bridge/20260318-150310-015/*` +- `tests/logs/ai/backend/human/live/human-live-kb-docx-bridge/20260318-150906-789/*` +- `tests/logs/ai/backend/human/live/human-live-kb-docx-bridge/20260318-151207-536/*` ## Exit Criteria diff --git a/doc/test.md b/doc/test.md index ecf21330..b6974197 100644 --- a/doc/test.md +++ b/doc/test.md @@ -423,7 +423,9 @@ tests/report// - `失败分析` - `当前评估` - `下一步建议` -- `证据` 必须列出对应 run 的 `meta.json`、`timeline.jsonl`、`artifacts.json` 路径;如果有关键附件,也应一并列出。 +- `证据` 默认只列出能定位该条证据的最小信息,例如 `caseId`、`runId`、`status`。 +- 不要在 report 正文里机械展开 `meta.json`、`timeline.jsonl`、`artifacts.json` 这些文件名。 +- 只有当失败分析或结论明确依赖某个关键附件时,才额外补充附件名称或用途说明。 - `通过率` 只写当前节点覆盖范围内的统计,不混入其他节点数据。 - `通过率` 固定拆成两组: - `历史总数据`:`totalRuns`、`passedRuns`、`failedRuns`、`historicalSuccessRate` @@ -464,7 +466,7 @@ tests/report// ##### 证据 -- `20260315-101530-112` | passed | `tests/logs/ai/backend/tools/terminal/terminal-basic-command/20260315-101530-112/meta.json` | `tests/logs/ai/backend/tools/terminal/terminal-basic-command/20260315-101530-112/timeline.jsonl` | `tests/logs/ai/backend/tools/terminal/terminal-basic-command/20260315-101530-112/artifacts.json` +- `terminal-basic-command` | run=`20260315-101530-112` | status=`passed` ##### 通过率 @@ -499,8 +501,8 @@ tests/report// ##### 证据 -- `20260315-103002-008` | passed | `tests/logs/ai/e2e/workbench/workbench-plan-artifact-smoke/20260315-103002-008/meta.json` | `tests/logs/ai/e2e/workbench/workbench-plan-artifact-smoke/20260315-103002-008/timeline.jsonl` | `tests/logs/ai/e2e/workbench/workbench-plan-artifact-smoke/20260315-103002-008/artifacts.json` -- artifact references: `tests/logs/ai/e2e/workbench/workbench-plan-artifact-smoke/20260315-103002-008/artifacts/trace/playwright-trace.zip` +- `workbench-plan-artifact-smoke` | run=`20260315-103002-008` | status=`passed` +- 关键附件: playwright trace ##### 通过率 diff --git a/src/main/services/ai/node/human/brainNode.ts b/src/main/services/ai/node/human/brainNode.ts index be2e6114..0280cd4c 100644 --- a/src/main/services/ai/node/human/brainNode.ts +++ b/src/main/services/ai/node/human/brainNode.ts @@ -171,6 +171,261 @@ const toResultData = (value: JsonValue | undefined, key: string): JsonObject | u return { [key]: value } } +type DecisionDocumentRef = { + documentName?: string + documentId?: string +} + +const clipText = (value: string, maxChars = 120): string => { + const text = value.trim() + if (text.length <= maxChars) return text + return `${text.slice(0, maxChars - 3)}...` +} + +const parseJsonValueText = (value: string): JsonValue | undefined => { + const trimmed = value.trim() + if (!trimmed) return undefined + try { + return JSON.parse(trimmed) as JsonValue + } catch { + const arrayStart = trimmed.indexOf('[') + const arrayEnd = trimmed.lastIndexOf(']') + if (arrayStart >= 0 && arrayEnd > arrayStart) { + try { + return JSON.parse(trimmed.slice(arrayStart, arrayEnd + 1)) as JsonValue + } catch { + return undefined + } + } + const objectStart = trimmed.indexOf('{') + const objectEnd = trimmed.lastIndexOf('}') + if (objectStart >= 0 && objectEnd > objectStart) { + try { + return JSON.parse(trimmed.slice(objectStart, objectEnd + 1)) as JsonValue + } catch { + return undefined + } + } + } + return undefined +} + +const extractDocumentRefFromRecord = (value: JsonObject): DecisionDocumentRef | null => { + const documentName = toOptionalText(value.document_name) || toOptionalText(value.name) + const documentId = toOptionalText(value.document_id) || toOptionalText(value.id) + if (!documentName && !documentId) return null + return { + documentName, + documentId + } +} + +const collectDocumentRefsFromValue = (value: JsonValue | undefined): DecisionDocumentRef[] => { + if (Array.isArray(value)) { + return value.flatMap((item) => collectDocumentRefsFromValue(item)) + } + + if (!isJsonObject(value)) return [] + + const refs: DecisionDocumentRef[] = [] + const directRef = extractDocumentRefFromRecord(value) + if (directRef) refs.push(directRef) + + const raw = toOptionalText(value.raw) + if (raw) { + refs.push(...collectDocumentRefsFromValue(parseJsonValueText(raw))) + } + + if (Array.isArray(value.items)) { + refs.push(...collectDocumentRefsFromValue(value.items)) + } + + return refs +} + +const dedupeDocumentRefs = (refs: DecisionDocumentRef[]): DecisionDocumentRef[] => { + const seen = new Set() + const unique: DecisionDocumentRef[] = [] + + for (const ref of refs) { + const key = `${ref.documentId || ''}::${ref.documentName || ''}` + if (!ref.documentId && !ref.documentName) continue + if (seen.has(key)) continue + seen.add(key) + unique.push(ref) + } + + return unique +} + +const formatDocumentRef = (ref: DecisionDocumentRef): string | undefined => { + if (ref.documentName && ref.documentId) return `${ref.documentName} (${ref.documentId})` + return ref.documentName || ref.documentId +} + +const countValueItems = (value: JsonValue | undefined): number => { + if (Array.isArray(value)) return value.length + if (isJsonObject(value) && Array.isArray(value.items)) return value.items.length + return value === undefined ? 0 : 1 +} + +const extractPrimaryEvidenceText = (value: JsonValue | undefined): string | undefined => { + if (Array.isArray(value)) { + for (const item of value) { + const nested = extractPrimaryEvidenceText(item) + if (nested) return nested + } + return undefined + } + + if (typeof value === 'string') { + const text = value.trim() + return text ? clipText(text) : undefined + } + + if (!isJsonObject(value)) return undefined + + const directText = + toOptionalText(value.text) || toOptionalText(value.summary) || toOptionalText(value.message) + if (directText) return clipText(directText) + + const raw = toOptionalText(value.raw) + if (raw) { + const nested = extractPrimaryEvidenceText(parseJsonValueText(raw)) + if (nested) return nested + } + + if (Array.isArray(value.items)) { + return extractPrimaryEvidenceText(value.items) + } + + return undefined +} + +const resolveToolFamily = (toolName: string | undefined): string | undefined => { + const normalized = String(toolName || '').trim() + if (!normalized) return undefined + const separatorIndex = normalized.indexOf('_') + return separatorIndex > 0 ? normalized.slice(0, separatorIndex) : normalized +} + +const countTrailingSameToolName = ( + rows: Array<{ toolName: string; toolFamily: string; summary: string }> +): number => { + if (rows.length === 0) return 0 + let count = 1 + for (let index = rows.length - 1; index > 0; index -= 1) { + if (rows[index]?.toolName !== rows[index - 1]?.toolName) break + count += 1 + } + return count +} + +const countTrailingSameToolFamily = ( + rows: Array<{ toolName: string; toolFamily: string; summary: string }> +): number => { + if (rows.length === 0) return 0 + let count = 1 + for (let index = rows.length - 1; index > 0; index -= 1) { + if (rows[index]?.toolFamily !== rows[index - 1]?.toolFamily) break + count += 1 + } + return count +} + +const countTrailingSameSummary = ( + rows: Array<{ toolName: string; toolFamily: string; summary: string }> +): number => { + if (rows.length === 0) return 0 + let count = 1 + for (let index = rows.length - 1; index > 0; index -= 1) { + if (rows[index]?.summary !== rows[index - 1]?.summary) break + count += 1 + } + return count +} + +const buildDecisionStateText = (input: { state: HumanRunState; ctx: AiRunContext }): string => { + const { state, ctx } = input + const remainingSteps = Math.max(0, state.maxSteps - state.steps) + const objective = String(state.input.objective || '') + const recentToolRows = state.stepHistory + .filter( + (step) => + step.action === 'invoke_tool' && typeof step.toolName === 'string' && step.toolName.length > 0 + ) + .slice(-4) + .map((step) => ({ + toolName: String(step.toolName || ''), + toolFamily: resolveToolFamily(String(step.toolName || '')) || 'unknown', + resultStatus: step.resultStatus || 'unknown', + summary: clipText(String(step.summary || '')) + })) + + let lastObservationSignals: JsonValue = { + kind: 'none' + } + + if (state.lastObservation?.kind === 'skill_loaded') { + lastObservationSignals = { + kind: 'skill_loaded', + skillId: state.lastObservation.skillId, + activeSkillIds: state.lastObservation.activeSkillIds, + activeToolNames: state.lastObservation.activeToolNames + } + } else if (state.lastObservation?.kind === 'terminal') { + lastObservationSignals = { + kind: 'terminal', + status: state.lastObservation.status, + code: state.lastObservation.code, + summary: clipText(state.lastObservation.summary), + hasData: state.lastObservation.data !== undefined + } + } else if (state.lastObservation?.kind === 'tool_result') { + const uniqueRefs = dedupeDocumentRefs(collectDocumentRefsFromValue(state.lastObservation.data)) + const primarySourceRef = uniqueRefs.map((ref) => formatDocumentRef(ref)).find(Boolean) + lastObservationSignals = { + kind: 'tool_result', + toolName: state.lastObservation.toolName, + toolFamily: resolveToolFamily(state.lastObservation.toolName) || 'unknown', + ok: state.lastObservation.ok, + code: state.lastObservation.code, + itemCount: countValueItems(state.lastObservation.data), + uniqueSourceRefCount: uniqueRefs.length, + primarySourceRef: primarySourceRef || 'none', + primaryEvidenceText: extractPrimaryEvidenceText(state.lastObservation.data) || 'none', + summary: clipText(state.lastObservation.summary) + } + } + + return toJsonText({ + stepBudget: { + currentStep: state.steps + 1, + maxSteps: state.maxSteps, + remainingSteps, + lowBudget: remainingSteps <= 2 + }, + taskSignals: { + deliveryKind: state.deliveryKind, + workspaceBound: Boolean(ctx.workspace), + selectedDocumentCount: ctx.selectedDocumentIds?.length || 0, + mentionsDocx: /\.docx\b|docx/i.test(objective), + mentionsPreview: /preview|预览/i.test(objective), + mentionsSave: /save|保存|另存/i.test(objective) + }, + progressSignals: { + activeSkillCount: state.activeSkillIds.length, + activeToolCount: state.activeTools.length, + recentToolCalls: recentToolRows, + trailingSameToolCount: countTrailingSameToolName(recentToolRows), + trailingSameToolFamilyCount: countTrailingSameToolFamily(recentToolRows), + trailingSameSummaryCount: countTrailingSameSummary(recentToolRows), + recentErrorCount: recentToolRows.filter((row) => row.resultStatus === 'error').length + }, + lastObservationSignals + }) +} + const buildDecisionPrompt = (input: { state: HumanRunState; ctx: AiRunContext }) => { const { state } = input const prompt = buildHumanBrainPrompt({ @@ -181,6 +436,7 @@ const buildDecisionPrompt = (input: { state: HumanRunState; ctx: AiRunContext }) availableSkillCardsText: toJsonText(state.availableSkillCards), loadedSkillsText: serializeLoadedSkills(state), activeToolsText: serializeActiveTools(state), + decisionStateText: buildDecisionStateText(input), lastObservationText: state.lastObservation ? toJsonText(state.lastObservation) : 'none', stepHistoryText: state.stepHistory.length > 0 diff --git a/src/main/services/ai/node/human/responseNode.ts b/src/main/services/ai/node/human/responseNode.ts index 59c59dcb..d071f10e 100644 --- a/src/main/services/ai/node/human/responseNode.ts +++ b/src/main/services/ai/node/human/responseNode.ts @@ -98,10 +98,14 @@ const deriveLocalSourceStatus = (state: HumanRunState): { summary: string } => { const joined = [state.summary, state.deliveryMarkdown].join('\n') - const mentionsInsufficient = - joined.includes('本地资料不足') || - joined.includes('是否允许继续使用非本地信息') || - joined.includes('非本地信息') + const insufficientMarkers = [ + '本地资料不足', + '本地证据不足', + '当前本地资料不足', + '当前本地证据不足', + '本地范围不足' + ] + const mentionsInsufficient = insufficientMarkers.some((marker) => joined.includes(marker)) if (mentionsInsufficient) { return { status: 'insufficient', diff --git a/src/main/services/ai/prompts/human/brain.ts b/src/main/services/ai/prompts/human/brain.ts index 50ff6748..6a999836 100644 --- a/src/main/services/ai/prompts/human/brain.ts +++ b/src/main/services/ai/prompts/human/brain.ts @@ -9,6 +9,7 @@ export type HumanBrainPromptInput = { availableSkillCardsText: string loadedSkillsText: string activeToolsText: string + decisionStateText: string lastObservationText: string stepHistoryText: string } @@ -19,7 +20,9 @@ export const buildHumanBrainPrompt = (input: HumanBrainPromptInput) => { '【工具边界】只有 active tools 中列出的工具可以 invoke_tool。', '【完成标准】仅当任务目标已满足且证据充分时才 complete_task;summary 必须写清结论与关键证据。', '【结果收敛】若任务目标已明确要求 resultData 字段,且最近 observation 已经提供这些字段所需证据,下一步必须 complete_task 并写入 resultData,不要继续为同一证据反复调用 KB 或 shell。', - '【KB 收敛】若 kb_hybrid_search_chunks 的 observation 已经出现唯一的 document_name/document_id,或首条文本已经直接给出答案,不要再调用 kb_list_documents / 再次搜索来“确认同一证据”;应直接 complete_task。', + '【证据收敛】若最近 observation 已经足以唯一支撑当前结论或下一交付,优先 complete_task;不要为了“再次确认同一证据”继续调用等价工具。', + '【无新增信息】若最近连续几步没有带来新信息,优先切换能力、缩小目标或明确资料不足;不要只是在同类工具之间来回试探。', + '【最小路径】优先完成满足 objective 与 delivery kind 的最小交付路径;若任务只要求 answer、handoff 或 preview,不要扩展成非必需的保存、重复检索或额外产物。', '【阻塞标准】workspace_required/approval_denied 等不可恢复条件必须 block_task。', '【反循环】最近一步没有带来新信息时,禁止重复相同 toolName + args;应切换 skill、调整参数或收敛。', '【禁止过度复核】同一文件名、同一命令输出或同一 KB 证据已经足够支撑结论时,不要再做“再次确认”式工具调用。', @@ -45,6 +48,7 @@ export const buildHumanBrainPrompt = (input: HumanBrainPromptInput) => { { label: 'Available Skill Cards', value: input.availableSkillCardsText }, { label: 'Loaded Skills', value: input.loadedSkillsText }, { label: 'Active Tools', value: input.activeToolsText }, + { label: 'Decision State', value: input.decisionStateText }, { label: 'Last Observation', value: input.lastObservationText }, { label: 'Recent Steps', value: input.stepHistoryText } ]) diff --git a/tests/ai/backend/human/helpers.ts b/tests/ai/backend/human/helpers.ts index 233297af..41e99f66 100644 --- a/tests/ai/backend/human/helpers.ts +++ b/tests/ai/backend/human/helpers.ts @@ -4,6 +4,7 @@ import { hasLiveLlmConfigInEnv, loadLiveLlmConfigFromEnv } from '../../../harnes import type { BackendTestRuntime, TestCaseDefinition } from '../../../harness/backend/test-types' type AiHumanCaseDefinition = Omit, 'domain' | 'layer' | 'suite'> +const HUMAN_LIVE_TEST_TIMEOUT_MS = 300_000 export const defineAiHumanCase = (definition: AiHumanCaseDefinition): void => { it(definition.caseId, async () => { @@ -25,15 +26,19 @@ export const defineAiHumanLiveCase = (definition: AiHumanCaseDefinition const runtimeModelName = loadLiveLlmConfigFromEnv('primary').model_name - it(definition.caseId, async () => { - await runBackendCase({ - ...definition, - domain: 'ai', - layer: 'backend', - suite: 'human', - runtimeModelName - }) - }) + it( + definition.caseId, + async () => { + await runBackendCase({ + ...definition, + domain: 'ai', + layer: 'backend', + suite: 'human', + runtimeModelName + }) + }, + HUMAN_LIVE_TEST_TIMEOUT_MS + ) } export const loadPrimaryLiveHumanConfig = () => loadLiveLlmConfigFromEnv('primary') diff --git a/tests/ai/backend/human/live/human-live-direct-kb-answer.test.ts b/tests/ai/backend/human/live/human-live-direct-kb-answer.test.ts new file mode 100644 index 00000000..07eeb859 --- /dev/null +++ b/tests/ai/backend/human/live/human-live-direct-kb-answer.test.ts @@ -0,0 +1,167 @@ +import type { DocumentRecord, JsonValue, SearchResult } from '../../../../../src/shared/types' +import { createHumanTestRunner } from '../../../../harness/backend/humanHarness' +import { + defineAiHumanLiveCase, + getEventsByType, + loadPrimaryLiveHumanConfig +} from '../helpers' + +const releaseChecklistDocument: DocumentRecord = { + id: 'doc-beta', + name: 'release-checklist.md', + source_type: 'file', + created_at: 1710000000000, + import_status: 2 +} + +const getFirstPreviewRecord = ( + value: JsonValue | undefined +): Record | null => { + if (!value) return null + if (Array.isArray(value)) { + const first = value[0] + if (!first || Array.isArray(first) || typeof first !== 'object') return null + return first as Record + } + if (typeof value !== 'object') return null + return value as Record +} + +defineAiHumanLiveCase({ + caseId: 'human-live-direct-kb-answer', + caseType: 'live', + groups: ['live'], + description: 'live 模型直接使用本地 KB 回答问题,并在结果里保留可回看的文档引用。', + fixtures: [], + runMode: 'single', + execute: async (runtime) => { + const runner = createHumanTestRunner( + runtime, + { + bindWorkspace: false, + selectedDocumentIds: ['doc-beta'], + liveModelConfig: loadPrimaryLiveHumanConfig(), + documentsStoreOverrides: { + hybridSearch: async () => { + const rows: SearchResult[] = [ + { + id: 'chunk-beta-1', + text: 'Approval flow still needs regression coverage before release.', + document_id: 'doc-beta', + document_name: 'release-checklist.md', + source_type: 'file', + _score: 0.91 + } + ] + return rows + }, + vectorSearch: async () => { + const rows: SearchResult[] = [ + { + id: 'chunk-beta-1', + text: 'Approval flow still needs regression coverage before release.', + document_id: 'doc-beta', + document_name: 'release-checklist.md', + source_type: 'file', + _score: 0.91 + } + ] + return rows + }, + ftsSearch: async () => { + const rows: SearchResult[] = [ + { + id: 'chunk-beta-1', + text: 'Approval flow still needs regression coverage before release.', + document_id: 'doc-beta', + document_name: 'release-checklist.md', + source_type: 'file', + _score: 0.91 + } + ] + return rows + }, + listDocuments: async () => ({ + items: [releaseChecklistDocument], + total: 1 + }), + getDocumentById: async (id: string) => (id === 'doc-beta' ? releaseChecklistDocument : null) + } + }, + { maxSteps: 6 } + ) + + const output = await runner.run( + { + taskId: runtime.taskId, + objective: + '请仅使用当前本地资料范围回答 approval flow regression coverage 是否已经完成。先做本地 KB 检索,一旦拿到直接证据就停止,不要重复确认。最终答复要明确说明是否完成,并引用依据文档;答复里保留短语 "regression coverage"。' + }, + { + deliveryKind: 'final' + } + ) + + const toolResults = getEventsByType(runtime, 'tool_call_result') + const firstKbResult = toolResults.find((event) => event.toolName.startsWith('kb_')) + const kbPreview = firstKbResult ? getFirstPreviewRecord(firstKbResult.outputPreview) : null + + return { + output, + kbPreview, + toolCalls: getEventsByType(runtime, 'tool_call_started'), + deliveries: getEventsByType(runtime, 'task_delivery_ready') + } + }, + assert: async (result, runtime) => { + const joinedOutputText = [ + result.output.summary, + result.output.deliveryMarkdown, + JSON.stringify(result.output.resultData || {}) + ].join('\n') + + runtime.assertThat( + result.output.status === 'completed' && + result.output.resultCode === 'ok' && + result.output.localSourceStatus === 'used', + { + name: 'human-live-kb-answer-completed', + message: 'live human 应能基于本地 KB 形成完成态问答交付', + data: { + status: result.output.status, + resultCode: result.output.resultCode, + localSourceStatus: result.output.localSourceStatus, + summary: result.output.summary, + resultData: result.output.resultData, + toolNames: result.toolCalls.map((event) => event.toolName) + } + } + ) + runtime.assertThat( + Boolean(result.kbPreview) && + String(result.kbPreview?.document_name || '').includes('release-checklist.md') && + String(result.kbPreview?.document_id || '').includes('doc-beta'), + { + name: 'human-live-kb-answer-evidence', + message: 'live human 应先拿到可回看的 KB 文档证据' + } + ) + runtime.assertThat( + joinedOutputText.includes('regression coverage') && + (joinedOutputText.includes('release-checklist.md') || joinedOutputText.includes('doc-beta')), + { + name: 'human-live-kb-answer-result-data', + message: 'live human 应在交付中保留答案关键短语和文档引用' + } + ) + runtime.assertThat( + result.toolCalls.some((event) => event.toolName.startsWith('kb_')) && + result.toolCalls.every((event) => event.toolName.startsWith('kb_')) && + result.deliveries.length === 1, + { + name: 'human-live-kb-answer-events', + message: 'live human 直接问答应只留下 KB 检索和最终交付事件' + } + ) + } +}) diff --git a/tests/ai/backend/human/live/human-live-evidence-insufficient-safe-stop.test.ts b/tests/ai/backend/human/live/human-live-evidence-insufficient-safe-stop.test.ts new file mode 100644 index 00000000..479e2bda --- /dev/null +++ b/tests/ai/backend/human/live/human-live-evidence-insufficient-safe-stop.test.ts @@ -0,0 +1,93 @@ +import { createHumanTestRunner } from '../../../../harness/backend/humanHarness' +import { + defineAiHumanLiveCase, + getEventsByType, + loadPrimaryLiveHumanConfig +} from '../helpers' + +defineAiHumanLiveCase({ + caseId: 'human-live-evidence-insufficient-safe-stop', + caseType: 'live', + groups: ['live'], + description: 'live 模型在本地证据不足时应安全收敛,而不是编造结论。', + fixtures: [], + runMode: 'single', + execute: async (runtime) => { + const runner = createHumanTestRunner( + runtime, + { + bindWorkspace: false, + selectedDocumentIds: ['doc-gap'], + liveModelConfig: loadPrimaryLiveHumanConfig(), + documentsStoreOverrides: { + hybridSearch: async () => [], + vectorSearch: async () => [], + ftsSearch: async () => [], + listDocuments: async () => ({ + items: [], + total: 0 + }), + getDocumentById: async () => null + } + }, + { maxSteps: 6 } + ) + + const output = await runner.run( + { + taskId: runtime.taskId, + objective: + '请仅使用当前本地资料范围确认 final owner 与 timeline。如果当前本地资料不足,不要猜测,也不要继续重复检索;请直接给出最终答复,答复中必须包含“本地资料不足”和“是否允许继续使用非本地信息”,并在 resultData 中保留 gap_summary 与 next_step。' + }, + { + deliveryKind: 'final' + } + ) + + return { + output, + toolCalls: getEventsByType(runtime, 'tool_call_started'), + toolResults: getEventsByType(runtime, 'tool_call_result'), + deliveries: getEventsByType(runtime, 'task_delivery_ready') + } + }, + assert: async (result, runtime) => { + const kbToolResults = result.toolResults.filter((event) => event.toolName.startsWith('kb_')) + const emptyKbResults = kbToolResults.every( + (event) => Array.isArray(event.outputPreview) && event.outputPreview.length === 0 + ) + + runtime.assertThat( + result.output.localSourceStatus === 'insufficient' && + result.output.deliveryMarkdown.includes('本地资料不足') && + result.output.deliveryMarkdown.includes('是否允许继续使用非本地信息'), + { + name: 'human-live-safe-stop-status', + message: 'live human 证据不足时应明确说明本地不足,并询问是否允许使用非本地信息' + } + ) + runtime.assertThat( + typeof result.output.resultData?.gap_summary === 'string' && + result.output.resultData.gap_summary.length > 0 && + typeof result.output.resultData?.next_step === 'string' && + result.output.resultData.next_step.length > 0, + { + name: 'human-live-safe-stop-result-data', + message: 'live human 安全收敛时应保留结构化缺口与下一步说明' + } + ) + runtime.assertThat( + kbToolResults.length >= 1 && + emptyKbResults && + result.toolCalls.every((event) => event.toolName.startsWith('kb_')), + { + name: 'human-live-safe-stop-no-hallucination', + message: 'live human 应只基于本地 KB 空结果收敛,不应伪造其他工具证据' + } + ) + runtime.assertThat(result.toolCalls.length <= 2 && result.deliveries.length === 1, { + name: 'human-live-safe-stop-no-loop', + message: 'live human 证据不足时不应陷入重复检索循环' + }) + } +}) diff --git a/tests/ai/backend/human/live/human-live-kb-docx-bridge.test.ts b/tests/ai/backend/human/live/human-live-kb-docx-bridge.test.ts new file mode 100644 index 00000000..ceebf6c4 --- /dev/null +++ b/tests/ai/backend/human/live/human-live-kb-docx-bridge.test.ts @@ -0,0 +1,233 @@ +import fs from 'node:fs/promises' +import type { DocumentRecord, JsonValue, SearchResult } from '../../../../../src/shared/types' +import { createDocxToolHarness, writeTestDocxFile } from '../../../../harness/backend/aiToolHarness' +import { createHumanTestRunner } from '../../../../harness/backend/humanHarness' +import { + defineAiHumanLiveCase, + getEventsByType, + loadPrimaryLiveHumanConfig +} from '../helpers' + +const releasePolicyDocument: DocumentRecord = { + id: 'doc-policy', + name: 'release-policy.md', + source_type: 'file', + created_at: 1710000000000, + import_status: 2 +} + +const fileExists = async (filePath: string): Promise => { + try { + await fs.stat(filePath) + return true + } catch { + return false + } +} + +const getFirstPreviewRecord = ( + value: JsonValue | undefined +): Record | null => { + if (!value) return null + if (Array.isArray(value)) { + const first = value[0] + if (!first || Array.isArray(first) || typeof first !== 'object') return null + return first as Record + } + if (typeof value !== 'object') return null + return value as Record +} + +defineAiHumanLiveCase({ + caseId: 'human-live-kb-docx-bridge', + caseType: 'live', + groups: ['live'], + description: 'live 模型先检索本地 KB,再把规范要求落到 docx preview 并交付。', + fixtures: [], + runMode: 'single', + execute: async (runtime) => { + await writeTestDocxFile(runtime.workspaceRoot, 'release-check.docx', [ + { type: 'heading', text: 'Release Checklist', level: 1 }, + { type: 'paragraph', text: 'Checklist summary: pending policy sync.' }, + { type: 'paragraph', text: 'Risk note: TBD.' } + ]) + + const runner = createHumanTestRunner( + runtime, + { + selectedDocumentIds: ['doc-policy'], + liveModelConfig: loadPrimaryLiveHumanConfig(), + documentsStoreOverrides: { + hybridSearch: async () => { + const rows: SearchResult[] = [ + { + id: 'chunk-policy-1', + text: 'Release reports must mention approval-denied coverage and terminal artifact assertion before sign-off.', + document_id: 'doc-policy', + document_name: 'release-policy.md', + source_type: 'file', + _score: 0.93 + } + ] + return rows + }, + vectorSearch: async () => { + const rows: SearchResult[] = [ + { + id: 'chunk-policy-1', + text: 'Release reports must mention approval-denied coverage and terminal artifact assertion before sign-off.', + document_id: 'doc-policy', + document_name: 'release-policy.md', + source_type: 'file', + _score: 0.93 + } + ] + return rows + }, + ftsSearch: async () => { + const rows: SearchResult[] = [ + { + id: 'chunk-policy-1', + text: 'Release reports must mention approval-denied coverage and terminal artifact assertion before sign-off.', + document_id: 'doc-policy', + document_name: 'release-policy.md', + source_type: 'file', + _score: 0.93 + } + ] + return rows + }, + listDocuments: async () => ({ + items: [releasePolicyDocument], + total: 1 + }), + getDocumentById: async (id: string) => (id === 'doc-policy' ? releasePolicyDocument : null) + } + }, + { maxSteps: 8 } + ) + + const output = await runner.run( + { + taskId: runtime.taskId, + objective: + '请先做一次当前本地发布规范检索,从 release-policy.md 提炼要求后直接修改 release-check.docx。拿到条款后不要再列文档或重复检索,也不需要 save;只需把摘要段改成包含 “approval-denied coverage and terminal artifact assertion before sign-off”,生成可审阅 preview,并以 handoff 形式交付,在交付里说明来源文档。' + }, + { + deliveryKind: 'handoff' + } + ) + + const toolCalls = getEventsByType(runtime, 'tool_call_started') + const toolResults = getEventsByType(runtime, 'tool_call_result') + const fileArtifacts = getEventsByType(runtime, 'file_artifact') + const previewArtifact = fileArtifacts.find((event) => event.artifactRole === 'preview') + const previewPath = previewArtifact?.path || '' + const previewExists = previewPath ? await fileExists(previewPath) : false + const firstKbResult = toolResults.find((event) => event.toolName.startsWith('kb_')) + const kbPreview = firstKbResult ? getFirstPreviewRecord(firstKbResult.outputPreview) : null + + if (!previewExists) { + return { + output, + kbPreview, + previewExists, + previewPath, + summarySearchOk: false, + summaryMatchCount: 0, + toolCalls, + fileArtifacts + } + } + + const inspectionHarness = createDocxToolHarness(runtime) + + try { + const openPreview = await inspectionHarness.invoke<{ + ok: boolean + docId: string + }>('docx_open', { + sourcePath: previewPath + }) + const summarySearch = await inspectionHarness.invoke<{ + ok: boolean + matches: Array<{ blockId: string; text: string }> + }>('docx_search', { + docId: openPreview.docId, + query: 'approval-denied coverage and terminal artifact assertion before sign-off', + mode: 'literal' + }) + + return { + output, + kbPreview, + previewExists, + previewPath, + summarySearchOk: summarySearch.ok, + summaryMatchCount: summarySearch.matches.length, + toolCalls, + fileArtifacts + } + } finally { + await inspectionHarness.release() + } + }, + assert: async (result, runtime) => { + const kbCallIndex = result.toolCalls.findIndex((event) => event.toolName.startsWith('kb_')) + const docxApplyIndex = result.toolCalls.findIndex((event) => event.toolName === 'docx_apply_ops') + const joinedOutputText = [ + result.output.summary, + result.output.deliveryMarkdown, + JSON.stringify(result.output.resultData || {}) + ].join('\n') + + runtime.assertThat( + result.output.status === 'completed' && + result.output.resultCode === 'ok' && + result.output.localSourceStatus === 'used', + { + name: 'human-live-kb-docx-bridge-completed', + message: 'live human 的 KB -> DOCX bridge 应形成完成态交付', + data: { + status: result.output.status, + resultCode: result.output.resultCode, + localSourceStatus: result.output.localSourceStatus, + summary: result.output.summary, + resultData: result.output.resultData, + previewExists: result.previewExists, + previewPath: result.previewPath, + toolNames: result.toolCalls.map((event) => event.toolName) + } + } + ) + runtime.assertThat( + Boolean(result.kbPreview) && + String(result.kbPreview?.document_name || '').includes('release-policy.md') && + String(result.kbPreview?.document_id || '').includes('doc-policy'), + { + name: 'human-live-kb-docx-bridge-kb-evidence', + message: 'live human 应先拿到本地发布规范证据' + } + ) + runtime.assertThat( + result.previewExists && + result.summarySearchOk && + result.summaryMatchCount >= 1 && + joinedOutputText.includes('release-policy.md'), + { + name: 'human-live-kb-docx-bridge-preview', + message: 'live human 应把 KB 规范写入 preview,并在交付中保留来源文档' + } + ) + runtime.assertThat( + kbCallIndex >= 0 && + docxApplyIndex > kbCallIndex && + result.toolCalls.some((event) => event.toolName === 'docx_open') && + result.fileArtifacts.some((event) => event.artifactRole === 'preview'), + { + name: 'human-live-kb-docx-bridge-events', + message: 'live human 应先检索 KB,再进入 docx 编辑和 preview 交付链路' + } + ) + } +}) diff --git a/tests/report/ai/report.md b/tests/report/ai/report.md index 6c69c3a7..dc5d28dc 100644 --- a/tests/report/ai/report.md +++ b/tests/report/ai/report.md @@ -4,19 +4,22 @@ ### 分析范围 -- raw logs 范围: 当前仓库内 `tests/logs/ai/backend/tools/**` -- 已分析 scope: `ai/backend/tools/docx/basic`、`ai/backend/tools/docx/boundary`、`ai/backend/tools/docx/scenario`、`ai/backend/tools/terminal/basic`、`ai/backend/tools/terminal/boundary`、`ai/backend/tools/terminal/scenario`、`ai/backend/tools/kb/basic`、`ai/backend/tools/kb/boundary` -- 未纳入本次分析: Stage 2 `ai/backend/human/**` 与旧版 flat scope `ai/backend/tools/docx`、`ai/backend/tools/terminal`、`ai/backend/tools/kb` +- raw logs 范围: 当前仓库内 `tests/logs/ai/backend/tools/**`、`tests/logs/ai/backend/human/**` +- 已分析 scope: `ai/backend/tools/docx/basic`、`ai/backend/tools/docx/boundary`、`ai/backend/tools/docx/scenario`、`ai/backend/tools/terminal/basic`、`ai/backend/tools/terminal/boundary`、`ai/backend/tools/terminal/scenario`、`ai/backend/tools/kb/basic`、`ai/backend/tools/kb/boundary`、`ai/backend/human/basic`、`ai/backend/human/boundary`、`ai/backend/human/scenario`、`ai/backend/human/live` +- 说明: 当前 `ai/backend/human/live` analysis window 只包含本轮补跑并已落日志的 `human-live-direct-kb-answer`、`human-live-evidence-insufficient-safe-stop`、`human-live-kb-docx-bridge`;此前已实现的 `human-live-direct-docx-edit`、`human-live-terminal-docx-handoff`、`human-live-long-task-project-brief` 未在本轮重跑,因此不计入当前 `human/live` 的 latestCaseCount +- 未纳入本次分析: 旧版 flat scope `ai/backend/tools/docx`、`ai/backend/tools/terminal`、`ai/backend/tools/kb` ### 总体结论 -- 历史总数据: `40` 次 raw runs,其中 `40` 次 passed、`0` 次 failed,historicalSuccessRate=`100.0%` -- 最新总数据: `20 / 20` 个 case 的最新一轮 run 为 passed,latestSuccessRate=`100.0%` -- 当前结论: report 已重置为 Stage 1 `backend/tools` 口径,当前新的分层 scope 下 `docx / terminal / kb` 三组工具基线都已跑通。 +- 历史总数据: `168` 次 raw runs,其中 `147` 次 passed、`21` 次 failed,historicalSuccessRate=`87.5%` +- 最新总数据: `39` 个 case 中有 `38` 个 case 的最新一轮 run 为 passed,latestSuccessRate=`97.4%` +- 当前结论: Stage 1 `backend/tools` 仍保持 `20 / 20 latest passed`;Stage 2 `backend/human` 的 `basic / boundary / scenario` 最新口径为 `16 / 16 passed`,但当前 `live` analysis window 仅 `2 / 3 passed`,`human-live-kb-docx-bridge` 仍未形成稳定 live 交付。 +- 解释: `historicalSuccessRate` 已包含 2026-03-18 为诊断 `human-live-kb-docx-bridge` 所做的多轮 live 补跑,因此阶段判断应优先看 latest 口径,而不是把这些诊断性失败等同于稳定回归失败。 ### 下一阶段建议 -- 先保持当前 Stage 1 `tools` 分层目录和命名稳定,继续把它作为后续 `human` / `entire` 的底层回归基线。 +- 保持当前 Stage 1 `tools` 分层目录和 Stage 2 `human` non-live 基线稳定。 +- 下一步优先修复 `human-live-kb-docx-bridge` 的 live 收敛与跨工具 handoff,再补跑本轮未覆盖的其他 `human live` case,之后再进入 `context` 阶段。 ## backend @@ -43,16 +46,16 @@ - `boundary`: `4` 次 raw runs,最新 `2 / 2 passed` - `scenario`: `6` 次 raw runs,最新 `3 / 3 passed` -##### 最新证据 +##### 证据 -- `tool-docx-search-modes-and-scope` | `20260318-131740-749` | passed | `tests/logs/ai/backend/tools/docx/basic/tool-docx-search-modes-and-scope/20260318-131740-749/meta.json` | `tests/logs/ai/backend/tools/docx/basic/tool-docx-search-modes-and-scope/20260318-131740-749/timeline.jsonl` | `tests/logs/ai/backend/tools/docx/basic/tool-docx-search-modes-and-scope/20260318-131740-749/artifacts.json` -- `tool-docx-structure-read` | `20260318-131740-748` | passed | `tests/logs/ai/backend/tools/docx/basic/tool-docx-structure-read/20260318-131740-748/meta.json` | `tests/logs/ai/backend/tools/docx/basic/tool-docx-structure-read/20260318-131740-748/timeline.jsonl` | `tests/logs/ai/backend/tools/docx/basic/tool-docx-structure-read/20260318-131740-748/artifacts.json` -- `tool-docx-targeting-and-ambiguous-search` | `20260318-131740-749` | passed | `tests/logs/ai/backend/tools/docx/basic/tool-docx-targeting-and-ambiguous-search/20260318-131740-749/meta.json` | `tests/logs/ai/backend/tools/docx/basic/tool-docx-targeting-and-ambiguous-search/20260318-131740-749/timeline.jsonl` | `tests/logs/ai/backend/tools/docx/basic/tool-docx-targeting-and-ambiguous-search/20260318-131740-749/artifacts.json` -- `tool-docx-preview-validate-rollback` | `20260318-131740-749` | passed | `tests/logs/ai/backend/tools/docx/boundary/tool-docx-preview-validate-rollback/20260318-131740-749/meta.json` | `tests/logs/ai/backend/tools/docx/boundary/tool-docx-preview-validate-rollback/20260318-131740-749/timeline.jsonl` | `tests/logs/ai/backend/tools/docx/boundary/tool-docx-preview-validate-rollback/20260318-131740-749/artifacts.json` -- `tool-docx-save-conflict-and-overwrite` | `20260318-131740-749` | passed | `tests/logs/ai/backend/tools/docx/boundary/tool-docx-save-conflict-and-overwrite/20260318-131740-749/meta.json` | `tests/logs/ai/backend/tools/docx/boundary/tool-docx-save-conflict-and-overwrite/20260318-131740-749/timeline.jsonl` | `tests/logs/ai/backend/tools/docx/boundary/tool-docx-save-conflict-and-overwrite/20260318-131740-749/artifacts.json` -- `tool-docx-mixed-structural-edit` | `20260318-131740-763` | passed | `tests/logs/ai/backend/tools/docx/scenario/tool-docx-mixed-structural-edit/20260318-131740-763/meta.json` | `tests/logs/ai/backend/tools/docx/scenario/tool-docx-mixed-structural-edit/20260318-131740-763/timeline.jsonl` | `tests/logs/ai/backend/tools/docx/scenario/tool-docx-mixed-structural-edit/20260318-131740-763/artifacts.json` -- `tool-docx-multi-document-session-isolation` | `20260318-131740-749` | passed | `tests/logs/ai/backend/tools/docx/scenario/tool-docx-multi-document-session-isolation/20260318-131740-749/meta.json` | `tests/logs/ai/backend/tools/docx/scenario/tool-docx-multi-document-session-isolation/20260318-131740-749/timeline.jsonl` | `tests/logs/ai/backend/tools/docx/scenario/tool-docx-multi-document-session-isolation/20260318-131740-749/artifacts.json` -- `tool-docx-table-and-inline-rewrite` | `20260318-131740-770` | passed | `tests/logs/ai/backend/tools/docx/scenario/tool-docx-table-and-inline-rewrite/20260318-131740-770/meta.json` | `tests/logs/ai/backend/tools/docx/scenario/tool-docx-table-and-inline-rewrite/20260318-131740-770/timeline.jsonl` | `tests/logs/ai/backend/tools/docx/scenario/tool-docx-table-and-inline-rewrite/20260318-131740-770/artifacts.json` +- `tool-docx-search-modes-and-scope` | run=`20260318-131740-749` | status=`passed` +- `tool-docx-structure-read` | run=`20260318-131740-748` | status=`passed` +- `tool-docx-targeting-and-ambiguous-search` | run=`20260318-131740-749` | status=`passed` +- `tool-docx-preview-validate-rollback` | run=`20260318-131740-749` | status=`passed` +- `tool-docx-save-conflict-and-overwrite` | run=`20260318-131740-749` | status=`passed` +- `tool-docx-mixed-structural-edit` | run=`20260318-131740-763` | status=`passed` +- `tool-docx-multi-document-session-isolation` | run=`20260318-131740-749` | status=`passed` +- `tool-docx-table-and-inline-rewrite` | run=`20260318-131740-770` | status=`passed` ##### 失败分析 @@ -83,15 +86,15 @@ - `boundary`: `6` 次 raw runs,最新 `3 / 3 passed` - `scenario`: `4` 次 raw runs,最新 `2 / 2 passed` -##### 最新证据 +##### 证据 -- `tool-terminal-path-resolution` | `20260318-131741-370` | passed | `tests/logs/ai/backend/tools/terminal/basic/tool-terminal-path-resolution/20260318-131741-370/meta.json` | `tests/logs/ai/backend/tools/terminal/basic/tool-terminal-path-resolution/20260318-131741-370/timeline.jsonl` | `tests/logs/ai/backend/tools/terminal/basic/tool-terminal-path-resolution/20260318-131741-370/artifacts.json` -- `tool-terminal-workspace-material-discovery` | `20260318-131741-389` | passed | `tests/logs/ai/backend/tools/terminal/basic/tool-terminal-workspace-material-discovery/20260318-131741-389/meta.json` | `tests/logs/ai/backend/tools/terminal/basic/tool-terminal-workspace-material-discovery/20260318-131741-389/timeline.jsonl` | `tests/logs/ai/backend/tools/terminal/basic/tool-terminal-workspace-material-discovery/20260318-131741-389/artifacts.json` -- `tool-terminal-approval-and-boundary` | `20260318-131741-430` | passed | `tests/logs/ai/backend/tools/terminal/boundary/tool-terminal-approval-and-boundary/20260318-131741-430/meta.json` | `tests/logs/ai/backend/tools/terminal/boundary/tool-terminal-approval-and-boundary/20260318-131741-430/timeline.jsonl` | `tests/logs/ai/backend/tools/terminal/boundary/tool-terminal-approval-and-boundary/20260318-131741-430/artifacts.json` -- `tool-terminal-approval-denied-follow-up` | `20260318-131741-505` | passed | `tests/logs/ai/backend/tools/terminal/boundary/tool-terminal-approval-denied-follow-up/20260318-131741-505/meta.json` | `tests/logs/ai/backend/tools/terminal/boundary/tool-terminal-approval-denied-follow-up/20260318-131741-505/timeline.jsonl` | `tests/logs/ai/backend/tools/terminal/boundary/tool-terminal-approval-denied-follow-up/20260318-131741-505/artifacts.json` -- `tool-terminal-failure-and-long-output` | `20260318-131740-773` | passed | `tests/logs/ai/backend/tools/terminal/boundary/tool-terminal-failure-and-long-output/20260318-131740-773/meta.json` | `tests/logs/ai/backend/tools/terminal/boundary/tool-terminal-failure-and-long-output/20260318-131740-773/timeline.jsonl` | `tests/logs/ai/backend/tools/terminal/boundary/tool-terminal-failure-and-long-output/20260318-131740-773/artifacts.json` -- `tool-terminal-artifact-producing-command` | `20260318-131741-439` | passed | `tests/logs/ai/backend/tools/terminal/scenario/tool-terminal-artifact-producing-command/20260318-131741-439/meta.json` | `tests/logs/ai/backend/tools/terminal/scenario/tool-terminal-artifact-producing-command/20260318-131741-439/timeline.jsonl` | `tests/logs/ai/backend/tools/terminal/scenario/tool-terminal-artifact-producing-command/20260318-131741-439/artifacts.json` -- `tool-terminal-batch-material-summary` | `20260318-131741-439` | passed | `tests/logs/ai/backend/tools/terminal/scenario/tool-terminal-batch-material-summary/20260318-131741-439/meta.json` | `tests/logs/ai/backend/tools/terminal/scenario/tool-terminal-batch-material-summary/20260318-131741-439/timeline.jsonl` | `tests/logs/ai/backend/tools/terminal/scenario/tool-terminal-batch-material-summary/20260318-131741-439/artifacts.json` +- `tool-terminal-path-resolution` | run=`20260318-131741-370` | status=`passed` +- `tool-terminal-workspace-material-discovery` | run=`20260318-131741-389` | status=`passed` +- `tool-terminal-approval-and-boundary` | run=`20260318-131741-430` | status=`passed` +- `tool-terminal-approval-denied-follow-up` | run=`20260318-131741-505` | status=`passed` +- `tool-terminal-failure-and-long-output` | run=`20260318-131740-773` | status=`passed` +- `tool-terminal-artifact-producing-command` | run=`20260318-131741-439` | status=`passed` +- `tool-terminal-batch-material-summary` | run=`20260318-131741-439` | status=`passed` ##### 失败分析 @@ -121,13 +124,13 @@ - `basic`: `8` 次 raw runs,最新 `4 / 4 passed` - `boundary`: `2` 次 raw runs,最新 `1 / 1 passed` -##### 最新证据 +##### 证据 -- `tool-kb-fts-search` | `20260318-131741-574` | passed | `tests/logs/ai/backend/tools/kb/basic/tool-kb-fts-search/20260318-131741-574/meta.json` | `tests/logs/ai/backend/tools/kb/basic/tool-kb-fts-search/20260318-131741-574/timeline.jsonl` | `tests/logs/ai/backend/tools/kb/basic/tool-kb-fts-search/20260318-131741-574/artifacts.json` -- `tool-kb-hybrid-search` | `20260318-131741-664` | passed | `tests/logs/ai/backend/tools/kb/basic/tool-kb-hybrid-search/20260318-131741-664/meta.json` | `tests/logs/ai/backend/tools/kb/basic/tool-kb-hybrid-search/20260318-131741-664/timeline.jsonl` | `tests/logs/ai/backend/tools/kb/basic/tool-kb-hybrid-search/20260318-131741-664/artifacts.json` -- `tool-kb-list-documents-and-scope-filter` | `20260318-131741-814` | passed | `tests/logs/ai/backend/tools/kb/basic/tool-kb-list-documents-and-scope-filter/20260318-131741-814/meta.json` | `tests/logs/ai/backend/tools/kb/basic/tool-kb-list-documents-and-scope-filter/20260318-131741-814/timeline.jsonl` | `tests/logs/ai/backend/tools/kb/basic/tool-kb-list-documents-and-scope-filter/20260318-131741-814/artifacts.json` -- `tool-kb-vector-search` | `20260318-131741-823` | passed | `tests/logs/ai/backend/tools/kb/basic/tool-kb-vector-search/20260318-131741-823/meta.json` | `tests/logs/ai/backend/tools/kb/basic/tool-kb-vector-search/20260318-131741-823/timeline.jsonl` | `tests/logs/ai/backend/tools/kb/basic/tool-kb-vector-search/20260318-131741-823/artifacts.json` -- `tool-kb-empty-scope-and-no-result` | `20260318-131741-542` | passed | `tests/logs/ai/backend/tools/kb/boundary/tool-kb-empty-scope-and-no-result/20260318-131741-542/meta.json` | `tests/logs/ai/backend/tools/kb/boundary/tool-kb-empty-scope-and-no-result/20260318-131741-542/timeline.jsonl` | `tests/logs/ai/backend/tools/kb/boundary/tool-kb-empty-scope-and-no-result/20260318-131741-542/artifacts.json` +- `tool-kb-fts-search` | run=`20260318-131741-574` | status=`passed` +- `tool-kb-hybrid-search` | run=`20260318-131741-664` | status=`passed` +- `tool-kb-list-documents-and-scope-filter` | run=`20260318-131741-814` | status=`passed` +- `tool-kb-vector-search` | run=`20260318-131741-823` | status=`passed` +- `tool-kb-empty-scope-and-no-result` | run=`20260318-131741-542` | status=`passed` ##### 失败分析 @@ -136,3 +139,62 @@ ##### 当前评估 - `kb` 已覆盖 `hybrid / vector / fts / list_documents` 与 `empty scope / no-result`,且当前最新口径为 `5 / 5 passed`。 + +### human + +##### 通过率 + +- 历史总数据: + - totalRuns: 128 + - passedRuns: 107 + - failedRuns: 21 + - historicalSuccessRate: 83.6% +- 最新总数据: + - latestCaseCount: 19 + - latestPassedCases: 18 + - latestFailedCases: 1 + - latestSuccessRate: 94.7% + +##### 分层统计 + +- `basic`: `8` 次 raw runs,最新 `2 / 2 passed` +- `boundary`: `24` 次 raw runs,最新 `6 / 6 passed` +- `scenario`: `33` 次 raw runs,最新 `8 / 8 passed` +- `live`: `11` 次 raw runs,最新 `2 / 3 passed` + +##### 证据 + +- `human-direct-docx-edit` | run=`20260318-151550-605` | status=`passed` +- `human-direct-kb-answer` | run=`20260318-151551-706` | status=`passed` +- `human-approval-blocked-recovery` | run=`20260318-151551-755` | status=`passed` +- `human-evidence-insufficient-safe-stop` | run=`20260318-151551-710` | status=`passed` +- `human-no-redundant-recheck` | run=`20260318-151551-803` | status=`passed` +- `human-partial-delivery-on-budget-limit` | run=`20260318-151551-723` | status=`passed` +- `human-tool-failure-recovery` | run=`20260318-151550-618` | status=`passed` +- `human-workspace-blocked-recovery` | run=`20260318-151551-679` | status=`passed` +- `human-kb-docx-bridge` | run=`20260318-151550-641` | status=`passed` +- `human-long-task-approval-branch` | run=`20260318-151551-710` | status=`passed` +- `human-long-task-project-brief` | run=`20260318-151550-604` | status=`passed` +- `human-long-task-with-corrections` | run=`20260318-151550-604` | status=`passed` +- `human-multi-source-merge-delivery` | run=`20260318-151550-604` | status=`passed` +- `human-multi-turn-refinement` | run=`20260318-151550-605` | status=`passed` +- `human-review-handoff-structured-delivery` | run=`20260318-151550-605` | status=`passed` +- `human-terminal-docx-handoff` | run=`20260318-151550-606` | status=`passed` +- `human-live-direct-kb-answer` | run=`20260318-145113-697` | status=`passed` +- `human-live-evidence-insufficient-safe-stop` | run=`20260318-150735-678` | status=`passed` +- `human-live-kb-docx-bridge` | run=`20260318-151207-536` | status=`failed` + +##### 失败分析 + +- 当前 latest 口径下唯一 failed case 是 `human-live-kb-docx-bridge`。 +- 2026-03-18 的 primary `gpt-5` 最新诊断 run `20260318-150310-015` 返回 `invalid_input`,summary=`524 status code (no body)`,事件只留下两次 `kb_hybrid_search_chunks`,说明链路在 live 模型侧已出现上游不稳定。 +- 2026-03-18 的 stronger `gpt-5.2` 诊断 run `20260318-150906-789` 没有触发 524,但在 `kb_hybrid_search_chunks -> kb_list_documents` 后直接判定步数不足,未进入 `docx-editing` handoff。 +- 2026-03-18 的更收敛 objective 诊断 run `20260318-151207-536` 仍然失败,表现为 `bounded_loop_error`,工具轨迹错误转入四次 `terminal_run_command` 加一次 `kb_hybrid_search_chunks`,没有产出 preview artifact。 +- 因此,当前 bridge 失败同时暴露两类问题:一是 primary live 服务侧不稳定;二是 human 在 `KB -> DOCX` 这类跨工具任务上的 live 路由和收敛仍不稳定。 + +##### 当前评估 + +- `basic / boundary / scenario` 当前 latest 全通过,说明 Stage 2 的脚本化回归基线已经稳定。 +- 新增补齐的 `human-live-direct-kb-answer` 与 `human-live-evidence-insufficient-safe-stop` 已形成 live 正向证据,其中前者还促成了 `KB` 收敛提示和 `localSourceStatus` 判定逻辑的修正。 +- 当前 `human/live` latestCaseCount 只有 `3`,因为本轮没有重跑此前已实现的 `human-live-direct-docx-edit`、`human-live-terminal-docx-handoff`、`human-live-long-task-project-brief`;因此 Stage 2 live 结论目前仍只能算“部分补齐 + 明确暴露 bridge 缺口”。 +- 在 `human-live-kb-docx-bridge` 稳定前,不建议把 Stage 2 视为可关闭阶段。