feat: enhance structured output strategy with capacity error handling and session management
This commit is contained in:
1 parent
b48859389d
commit
60221bcff2
11 files changed
+364
-77
No files matched your search
+1
-1
@@ -141,7 +141,7 @@ skill 不按目录强行拆分,也不等同于工具数量。一个 skill 可
|
||||
|
||||
AI 结构化调用按配置和模型维度记忆成功策略。首次没有历史策略时,统一按 `json_schema -> json_object -> prompt_json` 尝试;某个策略成功后,`LlmStrategyPreferenceStore` 以 `configId + modelName` 为键保存 `transport` 和结构化输出模式,后续调用直接优先使用已保存策略。供应商不会在执行器里写死专用策略。
|
||||
|
||||
结构化能力不支持、返回内容无法解析或不符合 Schema 时属于“策略降级”;HTTP 503、429、超时和 `ResourceExhausted` 属于“服务重试”,不会被误判成能力降级。`ResourceExhausted` 使用指数退避,避免短时间内重复请求进一步加重服务端容量压力。策略只有在完整调用成功后才会保存,失败调用不会污染历史偏好。
|
||||
结构化能力不支持、返回内容无法解析或不符合 Schema 时属于“策略降级”。结构化策略遇到 `ResourceExhausted` 时,会先在同一次调用内尝试下一个策略;后续策略成功后保存该策略,避免下一轮再次命中已拥塞的请求形态。只有所有候选策略都返回容量错误,才进入服务重试并使用指数退避。其他 HTTP 503、429 和超时仍直接进入服务重试。策略只有在完整调用成功后才会保存,失败调用不会污染历史偏好。
|
||||
|
||||
### 会话连续性
|
||||
|
||||
|
||||
@@ -100,10 +100,10 @@ AI unit / 辅助测试目录:
|
||||
#### `structured-output-resource-exhausted-backoff`
|
||||
|
||||
- 测试类型:`unit`
|
||||
- What:供应商返回 `503 ResourceExhausted` 时保持服务重试语义,并按 2、4、8、15 秒退避。
|
||||
- Why:服务容量错误不是结构化能力不支持;过短的固定重试会放大服务端限流并阻止策略探测完成。
|
||||
- How:直接验证重试分类结果和当前 attempt 对应的 delay;Workbench 真实 UI E2E 在运行中出现重试时检查过程流不同时显示“让俺想想”。
|
||||
- Oracle:ResourceExhausted 不触发策略持久化;重试状态可见且不与思考占位并存。
|
||||
- What:结构化策略返回 `503 ResourceExhausted` 时先尝试下一个候选策略;全部候选都容量失败时再按 2、4、8、15 秒退避。
|
||||
- Why:容量错误不等于永久不支持,但不同结构化请求形态可能落到不同服务路径;只原样重试会让每个新对话重复同样的三次等待,也无法形成有价值的策略记忆。
|
||||
- How:native 结构化输出测试让 `json_schema`、`json_object` 返回容量错误,`prompt_json` 成功,并验证第二次调用直接命中已保存偏好;执行器集成测试覆盖所有候选均失败后的外层退避和最终成功;Workbench 真实 UI E2E 在运行中出现重试时检查过程流不同时显示“让俺想想”。
|
||||
- Oracle:首轮后续策略成功时保存最终策略,第二轮只调用保存的策略;所有策略都容量失败时才产生外层重试事件;重试状态可见且不与思考占位并存。
|
||||
- Not:不要求真实 provider 在每次 E2E 都主动返回 503。
|
||||
|
||||
### 4.1 `tool`
|
||||
|
||||
@@ -100,7 +100,6 @@ export const TEXT_FILE_EDIT_TOOL_SPEC: HumanToolSpec<
|
||||
description: [
|
||||
'创建或修改普通文本文件,如 txt、md、csv、json,代码等 UTF-8 普通文本文件。',
|
||||
'无法创建或修改结构化文件如 docx、pptx、pdf、xlsx 等。',
|
||||
'每次调用执行一个 action。',
|
||||
'action=create:创建新文件并写入完整内容,不覆盖已有文件。',
|
||||
'action=rewrite:使用新内容完整改写已有文件。',
|
||||
'action=edit:对已有文件执行一个或多个精确文本修改;每个 oldText 必须恰好匹配一次。',
|
||||
|
||||
@@ -689,6 +689,7 @@ const invokeStructuredChatNative = async <T extends Record<string, unknown>>(
|
||||
const strategyErrors: string[] = []
|
||||
let latestRawText: string | undefined
|
||||
let latestOutputError: unknown
|
||||
let latestCapacityError: unknown
|
||||
|
||||
for (const strategy of attemptOrder) {
|
||||
const { method, transport } = resolveStructuredOutputStrategy(strategy)
|
||||
@@ -728,12 +729,26 @@ const invokeStructuredChatNative = async <T extends Record<string, unknown>>(
|
||||
continue
|
||||
}
|
||||
if (errorAnalysis.kind === 'retryable') {
|
||||
if (
|
||||
isResourceExhaustedError(
|
||||
error,
|
||||
`${errorAnalysis.info.rawText} ${errorAnalysis.info.code} ${errorAnalysis.info.message}`
|
||||
)
|
||||
) {
|
||||
strategyErrors.push(`${strategyLabel}: ${reason}`)
|
||||
latestCapacityError = error
|
||||
continue
|
||||
}
|
||||
throw createLlmProviderError(error, true)
|
||||
}
|
||||
throw createLlmProviderError(error, false)
|
||||
}
|
||||
}
|
||||
|
||||
if (latestCapacityError) {
|
||||
throw createLlmProviderError(latestCapacityError, true)
|
||||
}
|
||||
|
||||
throw new LlmOutputError({
|
||||
message: '模型返回内容无法解析为要求的结构。',
|
||||
issues: [
|
||||
|
||||
@@ -546,6 +546,24 @@ textarea,
|
||||
@apply bg-[#EFEFEE] text-[#37352F];
|
||||
}
|
||||
|
||||
.nc-sidebar-session-overflow {
|
||||
display: grid;
|
||||
grid-template-rows: 0fr;
|
||||
opacity: 0;
|
||||
transition:
|
||||
grid-template-rows 160ms ease,
|
||||
opacity 120ms ease;
|
||||
}
|
||||
|
||||
.nc-sidebar-session-overflow-expanded {
|
||||
grid-template-rows: 1fr;
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.nc-sidebar-session-toggle {
|
||||
@apply flex h-7 w-full items-center justify-center gap-1 rounded-[6px] text-[12px] font-medium text-[#91918E] transition-colors hover:bg-[#EFEFEE] hover:text-[#37352F] focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-[#0B6BCB]/30;
|
||||
}
|
||||
|
||||
.nc-sidebar-update-button {
|
||||
position: relative;
|
||||
display: inline-flex;
|
||||
|
||||
@@ -106,6 +106,7 @@ import SidebarUpdateControl from '../components/update/SidebarUpdateControl'
|
||||
type ApprovalBatchRequiredEvent = Extract<AiRunEvent, { type: 'approval_batch_required' }>
|
||||
type UserTakeoverRequiredEvent = Extract<AiRunEvent, { type: 'user_takeover_required' }>
|
||||
type ApprovalBatchDecisionAction = 'approve' | 'reject' | 'guide'
|
||||
const COLLAPSED_WORKSPACE_SESSION_LIMIT = 5
|
||||
type WorkbenchE2EWindow = Window &
|
||||
typeof globalThis & {
|
||||
__NCURATOR_E2E_SET_WORKBENCH_DRAFT__?: (draft: ComposerDraftValue) => void
|
||||
@@ -656,6 +657,9 @@ const ChatPage: React.FC = () => {
|
||||
return clampWorkbenchSidebarWidth(stored ? Number(stored) : WORKBENCH_SIDEBAR_DEFAULT_WIDTH)
|
||||
})
|
||||
const [collapsedWorkspaceIds, setCollapsedWorkspaceIds] = useState<Set<string>>(() => new Set())
|
||||
const [expandedWorkspaceSessionIds, setExpandedWorkspaceSessionIds] = useState<Set<string>>(
|
||||
() => new Set()
|
||||
)
|
||||
const [contentPanelCollapsed, setContentPanelCollapsed] = useState(true)
|
||||
const [contentPanelShellMounted, setContentPanelShellMounted] = useState(false)
|
||||
const [contentPanelShellVisible, setContentPanelShellVisible] = useState(false)
|
||||
@@ -1386,6 +1390,18 @@ const ChatPage: React.FC = () => {
|
||||
})
|
||||
}, [])
|
||||
|
||||
const toggleWorkspaceSessionsExpanded = useCallback((workspaceId: string): void => {
|
||||
setExpandedWorkspaceSessionIds((current) => {
|
||||
const next = new Set(current)
|
||||
if (next.has(workspaceId)) {
|
||||
next.delete(workspaceId)
|
||||
} else {
|
||||
next.add(workspaceId)
|
||||
}
|
||||
return next
|
||||
})
|
||||
}, [])
|
||||
|
||||
const createNewSession = useCallback(
|
||||
(workspaceId?: string): void => {
|
||||
saveComposerDraftForSession(currentSessionIdRef.current, composerDraftValueRef.current)
|
||||
@@ -3212,6 +3228,64 @@ const ChatPage: React.FC = () => {
|
||||
{workspaces.map((workspace) => {
|
||||
const workspaceSessions = sessionsByWorkspace.get(workspace.id) || []
|
||||
const workspaceCollapsed = collapsedWorkspaceIds.has(workspace.id)
|
||||
const workspaceSessionsExpanded = expandedWorkspaceSessionIds.has(workspace.id)
|
||||
const visibleWorkspaceSessions = workspaceSessions.slice(
|
||||
0,
|
||||
COLLAPSED_WORKSPACE_SESSION_LIMIT
|
||||
)
|
||||
const overflowWorkspaceSessions = workspaceSessions.slice(
|
||||
COLLAPSED_WORKSPACE_SESSION_LIMIT
|
||||
)
|
||||
const hasOverflowSessions = overflowWorkspaceSessions.length > 0
|
||||
const renderSession = (session: ChatSessionListItem): React.JSX.Element => (
|
||||
<div
|
||||
key={session.id}
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
data-testid={`${workbenchSelectors.sessionItemPrefix}-${session.id}`}
|
||||
className={`nc-sidebar-item group relative ${
|
||||
currentSessionId === session.id ? 'nc-sidebar-item-active' : 'bg-transparent'
|
||||
} ${
|
||||
deletingSessionIds.has(session.id) ? 'pointer-events-none opacity-60' : ''
|
||||
} px-3 py-2`}
|
||||
onClick={() => selectSession(session)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === 'Enter' || event.key === ' ') {
|
||||
event.preventDefault()
|
||||
selectSession(session)
|
||||
}
|
||||
}}
|
||||
>
|
||||
<div className="flex min-w-0 items-center gap-2 pl-4">
|
||||
<div className="min-w-0 flex-1 truncate text-[13px] font-medium text-[#37352F]">
|
||||
{session.title}
|
||||
</div>
|
||||
<div className="flex h-4 shrink-0 items-center gap-1 text-[11px] text-[#91918E] group-hover:hidden">
|
||||
{session.isRunning ? (
|
||||
<span className="nc-session-running-spinner" aria-label="任务运行中" />
|
||||
) : session.hasUnreadCompletion ? (
|
||||
<span className="nc-session-unread-dot" aria-label="任务已完成,未查看" />
|
||||
) : (
|
||||
<>
|
||||
<HiClock className="h-3 w-3" />
|
||||
<span>{formatRelativeTime(session.created_at)}</span>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
<Button
|
||||
type="text"
|
||||
size="small"
|
||||
icon={<HiTrash className="h-3.5 w-3.5" />}
|
||||
className="!absolute !right-1 !top-1/2 !z-10 !h-6 !w-6 !-translate-y-1/2 !rounded-[6px] !p-0 opacity-0 transition-opacity group-hover:opacity-100"
|
||||
loading={deletingSessionIds.has(session.id)}
|
||||
disabled={deletingSessionIds.has(session.id)}
|
||||
data-testid={`${workbenchSelectors.sessionDeleteButtonPrefix}-${session.id}`}
|
||||
onMouseDown={(event) => event.stopPropagation()}
|
||||
onClick={(event) => requestDeleteSession(event, session)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
return (
|
||||
<div key={workspace.id} className="nc-sidebar-workspace-group">
|
||||
<div className="nc-sidebar-workspace-header group">
|
||||
@@ -3248,62 +3322,48 @@ const ChatPage: React.FC = () => {
|
||||
</div>
|
||||
</div>
|
||||
{workspaceCollapsed ? null : (
|
||||
<div className="mt-1 space-y-1">
|
||||
{workspaceSessions.map((session) => (
|
||||
<div className="mt-1 flex flex-col gap-1">
|
||||
{visibleWorkspaceSessions.map(renderSession)}
|
||||
{hasOverflowSessions ? (
|
||||
<div
|
||||
key={session.id}
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
data-testid={`${workbenchSelectors.sessionItemPrefix}-${session.id}`}
|
||||
className={`nc-sidebar-item group relative ${
|
||||
currentSessionId === session.id
|
||||
? 'nc-sidebar-item-active'
|
||||
: 'bg-transparent'
|
||||
} ${deletingSessionIds.has(session.id) ? 'pointer-events-none opacity-60' : ''} px-3 py-2`}
|
||||
onClick={() => selectSession(session)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === 'Enter' || event.key === ' ') {
|
||||
event.preventDefault()
|
||||
selectSession(session)
|
||||
}
|
||||
}}
|
||||
className={`nc-sidebar-session-overflow ${
|
||||
workspaceSessionsExpanded ? 'nc-sidebar-session-overflow-expanded' : ''
|
||||
}`}
|
||||
aria-hidden={!workspaceSessionsExpanded}
|
||||
inert={!workspaceSessionsExpanded}
|
||||
>
|
||||
<div className="flex min-w-0 items-center gap-2 pl-4">
|
||||
<div className="min-w-0 flex-1 truncate text-[13px] font-medium text-[#37352F]">
|
||||
{session.title}
|
||||
<div className="min-h-0 overflow-hidden">
|
||||
<div className="flex flex-col gap-1">
|
||||
{overflowWorkspaceSessions.map(renderSession)}
|
||||
</div>
|
||||
<div className="flex h-4 shrink-0 items-center gap-1 text-[11px] text-[#91918E] group-hover:hidden">
|
||||
{session.isRunning ? (
|
||||
<span
|
||||
className="nc-session-running-spinner"
|
||||
aria-label="任务运行中"
|
||||
/>
|
||||
) : session.hasUnreadCompletion ? (
|
||||
<span
|
||||
className="nc-session-unread-dot"
|
||||
aria-label="任务已完成,未查看"
|
||||
/>
|
||||
) : (
|
||||
<>
|
||||
<HiClock className="h-3 w-3" />
|
||||
<span>{formatRelativeTime(session.created_at)}</span>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
<Button
|
||||
type="text"
|
||||
size="small"
|
||||
icon={<HiTrash className="h-3.5 w-3.5" />}
|
||||
className="!absolute !right-1 !top-1/2 !z-10 !h-6 !w-6 !-translate-y-1/2 !rounded-[6px] !p-0 opacity-0 transition-opacity group-hover:opacity-100"
|
||||
loading={deletingSessionIds.has(session.id)}
|
||||
disabled={deletingSessionIds.has(session.id)}
|
||||
data-testid={`${workbenchSelectors.sessionDeleteButtonPrefix}-${session.id}`}
|
||||
onMouseDown={(event) => event.stopPropagation()}
|
||||
onClick={(event) => requestDeleteSession(event, session)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
) : null}
|
||||
{hasOverflowSessions ? (
|
||||
<button
|
||||
type="button"
|
||||
className="nc-sidebar-session-toggle workbench-no-drag"
|
||||
onClick={() => toggleWorkspaceSessionsExpanded(workspace.id)}
|
||||
data-testid={`${workbenchSelectors.workspaceSessionTogglePrefix}-${workspace.id}`}
|
||||
aria-expanded={workspaceSessionsExpanded}
|
||||
aria-label={
|
||||
workspaceSessionsExpanded
|
||||
? `收起 ${workspace.name} 的对话`
|
||||
: `展开 ${workspace.name} 的其余 ${overflowWorkspaceSessions.length} 个对话`
|
||||
}
|
||||
>
|
||||
<span>
|
||||
{workspaceSessionsExpanded
|
||||
? '收起'
|
||||
: `展开其余 ${overflowWorkspaceSessions.length} 个`}
|
||||
</span>
|
||||
<HiChevronDown
|
||||
className={`h-3.5 w-3.5 transition-transform duration-150 ${
|
||||
workspaceSessionsExpanded ? 'rotate-180' : ''
|
||||
}`}
|
||||
/>
|
||||
</button>
|
||||
) : null}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -122,6 +122,7 @@ export const workbenchSelectors = {
|
||||
sidebarBrand: 'workbench-sidebar-brand',
|
||||
sessionsList: 'workbench-sessions-list',
|
||||
sessionItemPrefix: 'workbench-session-item',
|
||||
workspaceSessionTogglePrefix: 'workbench-workspace-session-toggle',
|
||||
sessionDeleteButtonPrefix: 'workbench-session-delete-button',
|
||||
sessionDeleteConfirmModal: 'workbench-session-delete-confirm-modal',
|
||||
sessionDeleteConfirmButton: 'workbench-session-delete-confirm-button',
|
||||
|
||||
@@ -12,6 +12,7 @@ vi.mock('../../../../src/main/services/llm/chatModel', () => ({
|
||||
|
||||
import { createAiLlmExecutor } from '../../../../src/main/services/ai/llmExecutor'
|
||||
import type { AiLlmDriver } from '../../../../src/main/services/ai/llmExecutor'
|
||||
import { LlmProviderError } from '../../../../src/main/services/ai/errors'
|
||||
import type { AiLlmIoRecord, AiRunEvent, LlmRuntimeConfig } from '../../../../src/shared/types'
|
||||
|
||||
const baseConfig: LlmRuntimeConfig = {
|
||||
@@ -114,16 +115,23 @@ describe('ai unit llm executor text path', () => {
|
||||
})
|
||||
|
||||
it('persists the winning structured fallback strategy for the next run', async () => {
|
||||
const saveConfig = vi.fn(async () => undefined)
|
||||
let currentConfig = baseConfig
|
||||
const preferredStrategies: Array<unknown> = []
|
||||
const saveConfig = vi.fn(async (nextConfig: LlmRuntimeConfig) => {
|
||||
currentConfig = nextConfig
|
||||
})
|
||||
const driver: AiLlmDriver = {
|
||||
chatStructured: async () => ({
|
||||
output: { answer: 'from prompt json' },
|
||||
rawText: '{"answer":"from prompt json"}',
|
||||
method: 'promptJson',
|
||||
transport: 'completions',
|
||||
structuredOutputMode: 'prompt_json',
|
||||
strategy: { transport: 'completions', mode: 'prompt_json' }
|
||||
})
|
||||
chatStructured: async (input) => {
|
||||
preferredStrategies.push(input.preferredStrategy)
|
||||
return {
|
||||
output: { answer: 'from prompt json' },
|
||||
rawText: '{"answer":"from prompt json"}',
|
||||
method: 'promptJson',
|
||||
transport: 'completions',
|
||||
structuredOutputMode: 'prompt_json',
|
||||
strategy: { transport: 'completions', mode: 'prompt_json' }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const executor = createAiLlmExecutor({
|
||||
@@ -132,21 +140,29 @@ describe('ai unit llm executor text path', () => {
|
||||
sendEvent: () => undefined
|
||||
},
|
||||
runtimeSettings: { llmStreamingMode: 'force_non_stream' },
|
||||
getConfig: async () => baseConfig,
|
||||
getConfig: async () => currentConfig,
|
||||
saveConfig,
|
||||
checkCancelled: () => undefined,
|
||||
driver
|
||||
})
|
||||
|
||||
const result = await executor.chatStructured({
|
||||
const request = {
|
||||
caller: 'human.brain',
|
||||
schemaName: 'unit_schema',
|
||||
schema: z.object({ answer: z.string() }),
|
||||
system: 'Return JSON only.',
|
||||
user: 'Say hello.'
|
||||
})
|
||||
} as const
|
||||
const firstResult = await executor.chatStructured(request)
|
||||
const secondResult = await executor.chatStructured(request)
|
||||
|
||||
expect(result.output).toEqual({ answer: 'from prompt json' })
|
||||
expect(firstResult.output).toEqual({ answer: 'from prompt json' })
|
||||
expect(secondResult.output).toEqual({ answer: 'from prompt json' })
|
||||
expect(preferredStrategies).toEqual([
|
||||
undefined,
|
||||
{ transport: 'completions', mode: 'prompt_json' }
|
||||
])
|
||||
expect(saveConfig).toHaveBeenCalledTimes(1)
|
||||
expect(saveConfig).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
preferredTransport: 'completions',
|
||||
@@ -155,6 +171,86 @@ describe('ai unit llm executor text path', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('recovers when ResourceExhausted fails twice and the third attempt succeeds', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const events: AiRunEvent[] = []
|
||||
const saveConfig = vi.fn(async () => undefined)
|
||||
let callCount = 0
|
||||
const driver: AiLlmDriver = {
|
||||
chatStructured: async () => {
|
||||
callCount += 1
|
||||
if (callCount < 3) {
|
||||
throw new LlmProviderError({
|
||||
message: '503 ResourceExhausted: Worker local total request limit reached (48/48)',
|
||||
httpStatus: 503,
|
||||
providerCode: 'service_unavailable',
|
||||
retryable: true
|
||||
})
|
||||
}
|
||||
return {
|
||||
output: { answer: 'recovered on attempt 3' },
|
||||
rawText: '{"answer":"recovered on attempt 3"}',
|
||||
method: 'promptJson',
|
||||
transport: 'completions',
|
||||
structuredOutputMode: 'prompt_json',
|
||||
strategy: { transport: 'completions', mode: 'prompt_json' }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const executor = createAiLlmExecutor({
|
||||
eventCtx: {
|
||||
runId: 'run-resource-exhausted-recovery',
|
||||
sendEvent: (event) => events.push(event)
|
||||
},
|
||||
runtimeSettings: { llmStreamingMode: 'force_non_stream' },
|
||||
getConfig: async () => baseConfig,
|
||||
saveConfig,
|
||||
checkCancelled: () => undefined,
|
||||
driver
|
||||
})
|
||||
|
||||
const resultPromise = executor.chatStructured({
|
||||
caller: 'human.brain',
|
||||
schemaName: 'unit_schema',
|
||||
schema: z.object({ answer: z.string() }),
|
||||
system: 'Return JSON only.',
|
||||
user: 'List files.'
|
||||
})
|
||||
|
||||
await vi.advanceTimersByTimeAsync(2_000)
|
||||
await vi.advanceTimersByTimeAsync(4_000)
|
||||
const result = await resultPromise
|
||||
|
||||
const retryStarted = events.filter(
|
||||
(event) =>
|
||||
event.type === 'assistant_process_group_started' && event.display.kind === 'retry'
|
||||
)
|
||||
expect(result.output).toEqual({ answer: 'recovered on attempt 3' })
|
||||
expect(callCount).toBe(3)
|
||||
expect(retryStarted.map((event) => event.display)).toEqual([
|
||||
expect.objectContaining({ kind: 'retry', attempt: 2, maxAttempts: 5 }),
|
||||
expect.objectContaining({ kind: 'retry', attempt: 3, maxAttempts: 5 })
|
||||
])
|
||||
expect(
|
||||
events.some((event) => event.type === 'assistant_process_group_completed')
|
||||
).toBe(true)
|
||||
expect(
|
||||
events.some((event) => event.type === 'assistant_process_group_failed')
|
||||
).toBe(false)
|
||||
expect(saveConfig).toHaveBeenCalledTimes(1)
|
||||
expect(saveConfig).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
preferredTransport: 'completions',
|
||||
preferredStructuredOutputMode: 'prompt_json'
|
||||
})
|
||||
)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('forwards streamed text deltas through the observer and records stream metadata', async () => {
|
||||
const events: AiRunEvent[] = []
|
||||
const llmIoRecords: AiLlmIoRecord[] = []
|
||||
|
||||
@@ -385,6 +385,51 @@ describe('ai unit structured output auto fallback', () => {
|
||||
expect(callOrder).toEqual(['completions/json_schema'])
|
||||
})
|
||||
|
||||
it('tries another structured strategy after provider capacity exhaustion', async () => {
|
||||
const callOrder: string[] = []
|
||||
const capacityError = Object.assign(
|
||||
new Error('ResourceExhausted: Worker local total request limit reached (48/48)'),
|
||||
{ status: 503 }
|
||||
)
|
||||
configureCreateChatModel(
|
||||
{
|
||||
'completions/json_schema': { type: 'error', error: capacityError },
|
||||
'completions/json_object': { type: 'error', error: capacityError },
|
||||
'completions/prompt_json': {
|
||||
type: 'success',
|
||||
rawText: JSON.stringify({ answer: 'capacity fallback worked' })
|
||||
}
|
||||
},
|
||||
callOrder
|
||||
)
|
||||
|
||||
const firstResult = await invokeStructuredChat({
|
||||
...baseInput,
|
||||
cfg: nvidiaCfg
|
||||
})
|
||||
|
||||
expect(firstResult.output).toEqual({ answer: 'capacity fallback worked' })
|
||||
expect(firstResult.strategy).toEqual({ transport: 'completions', mode: 'prompt_json' })
|
||||
expect(callOrder).toEqual([
|
||||
'completions/json_schema',
|
||||
'completions/json_object',
|
||||
'completions/prompt_json'
|
||||
])
|
||||
|
||||
callOrder.length = 0
|
||||
const secondResult = await invokeStructuredChat({
|
||||
...baseInput,
|
||||
cfg: {
|
||||
...nvidiaCfg,
|
||||
preferredTransport: 'completions',
|
||||
preferredStructuredOutputMode: 'prompt_json'
|
||||
}
|
||||
})
|
||||
|
||||
expect(secondResult.output).toEqual({ answer: 'capacity fallback worked' })
|
||||
expect(callOrder).toEqual(['completions/prompt_json'])
|
||||
})
|
||||
|
||||
it('surfaces non-retryable provider failures without trying another strategy', async () => {
|
||||
const callOrder: string[] = []
|
||||
configureCreateChatModel(
|
||||
|
||||
+40
-1
@@ -1,4 +1,5 @@
|
||||
import { describe, it } from 'vitest'
|
||||
import type { AiRunEvent } from '../../../../../src/shared/types'
|
||||
import { runE2ECase } from '../../../../harness/e2e/run-e2e-case'
|
||||
import {
|
||||
WORKBENCH_LIVE_AI_E2E_TIMEOUT_MS,
|
||||
@@ -17,6 +18,20 @@ const FIRST_PROMPT =
|
||||
'请基于当前工作区的 human-input.txt,用一句话告诉我里面用于 workspace read validation 的事实。'
|
||||
const SECOND_PROMPT = '再用一句话说明这条事实为什么适合做 Workbench 读文件验证。'
|
||||
|
||||
const getStructuredStrategies = (events: AiRunEvent[]) =>
|
||||
events
|
||||
.filter(
|
||||
(event): event is Extract<AiRunEvent, { type: 'activity' }> =>
|
||||
event.type === 'activity' &&
|
||||
event.actionType === 'llm_structured_output' &&
|
||||
event.status === 'completed'
|
||||
)
|
||||
.map((event) => ({
|
||||
strategy: event.details?.strategy || null,
|
||||
fallbackStrategies: event.details?.fallbackStrategies || [],
|
||||
schemaName: event.details?.schemaName || null
|
||||
}))
|
||||
|
||||
const readSessionListText = async (page: import('playwright').Page): Promise<string> =>
|
||||
((await getWorkbenchLocator(page, workbenchSelectors.sessionsList).textContent()) || '').trim()
|
||||
|
||||
@@ -70,6 +85,7 @@ describe('workbench conversation e2e', () => {
|
||||
{ page, blockerPolicy: 'auto_approve' }
|
||||
)
|
||||
recordWorkbenchAiEvidence(runtime, firstEvidence, 0.8)
|
||||
const firstStructuredStrategies = getStructuredStrategies(firstEvidence.events)
|
||||
const afterFirstSessionsText = await readSessionListText(page)
|
||||
const afterFirstCreatedSessionCount = await page.evaluate(
|
||||
async (startedAt) =>
|
||||
@@ -88,6 +104,7 @@ describe('workbench conversation e2e', () => {
|
||||
{ page, blockerPolicy: 'auto_approve' }
|
||||
)
|
||||
recordWorkbenchAiEvidence(runtime, secondEvidence, 0.8)
|
||||
const secondStructuredStrategies = getStructuredStrategies(secondEvidence.events)
|
||||
const afterSecondSessionsText = await readSessionListText(page)
|
||||
|
||||
return {
|
||||
@@ -101,7 +118,9 @@ describe('workbench conversation e2e', () => {
|
||||
firstTitlePrefix: FIRST_PROMPT.slice(0, 20),
|
||||
secondTitlePrefix: SECOND_PROMPT.slice(0, 20),
|
||||
firstFinalAnswer: firstEvidence.finalAnswerText,
|
||||
secondFinalAnswer: secondEvidence.finalAnswerText
|
||||
secondFinalAnswer: secondEvidence.finalAnswerText,
|
||||
firstStructuredStrategies,
|
||||
secondStructuredStrategies
|
||||
}
|
||||
},
|
||||
assert: async (result, runtime) => {
|
||||
@@ -121,6 +140,26 @@ describe('workbench conversation e2e', () => {
|
||||
data: result
|
||||
}
|
||||
)
|
||||
const savedStrategy = result.firstStructuredStrategies.at(-1)?.strategy
|
||||
const secondRunFirstStrategy =
|
||||
result.secondStructuredStrategies[0]?.fallbackStrategies[0]
|
||||
await runtime.assertThat(
|
||||
Boolean(
|
||||
savedStrategy &&
|
||||
secondRunFirstStrategy &&
|
||||
secondRunFirstStrategy.transport === savedStrategy.transport &&
|
||||
secondRunFirstStrategy.mode === savedStrategy.mode
|
||||
),
|
||||
{
|
||||
name: 'workbench-structured-output-strategy-profile-reuse',
|
||||
message:
|
||||
'第一轮结构化输出成功后应保存策略;第二轮首个结构化请求必须直接从该策略开始,不能重新从默认策略链探测。',
|
||||
data: {
|
||||
firstStructuredStrategies: result.firstStructuredStrategies,
|
||||
secondStructuredStrategies: result.secondStructuredStrategies
|
||||
}
|
||||
}
|
||||
)
|
||||
await assertNoWorkbenchFakeSignals(runtime)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1759,15 +1759,29 @@ export const waitForWorkbenchAiRunOutcome = async (
|
||||
|
||||
for (;;) {
|
||||
const events = await collectWorkbenchAiRunEvents(runtime)
|
||||
const runFailed = events.find((event) => event.type === 'run_failed')
|
||||
const runStarted = events
|
||||
.filter(
|
||||
(event): event is Extract<AiRunEvent, { type: 'run_started' }> =>
|
||||
event.type === 'run_started' && event.createdAt >= startedAtMs
|
||||
)
|
||||
.at(-1)
|
||||
if (!runStarted) {
|
||||
if (Date.now() >= deadline) {
|
||||
throw new Error(`AI run did not start within ${timeoutMs}ms`)
|
||||
}
|
||||
await page.waitForTimeout(250)
|
||||
continue
|
||||
}
|
||||
const runEvents = events.filter((event) => event.runId === runStarted.runId)
|
||||
const runFailed = runEvents.find((event) => event.type === 'run_failed')
|
||||
if (runFailed?.type === 'run_failed') {
|
||||
throw new Error(`AI run failed while waiting for completion: ${runFailed.message}`)
|
||||
}
|
||||
|
||||
const runCompleted = events.find((event) => event.type === 'run_completed')
|
||||
const runCompleted = runEvents.find((event) => event.type === 'run_completed')
|
||||
if (runCompleted?.type === 'run_completed') {
|
||||
const finalAnswerText =
|
||||
events
|
||||
runEvents
|
||||
.filter(
|
||||
(event): event is Extract<AiRunEvent, { type: 'answer_completed' }> =>
|
||||
event.type === 'answer_completed'
|
||||
@@ -1779,20 +1793,20 @@ export const waitForWorkbenchAiRunOutcome = async (
|
||||
evidence: {
|
||||
modelName,
|
||||
durationMs: Date.now() - startedAtMs,
|
||||
events,
|
||||
events: runEvents,
|
||||
finalAnswerText
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const approvalRequired = events
|
||||
const approvalRequired = runEvents
|
||||
.filter(
|
||||
(event): event is Extract<AiRunEvent, { type: 'approval_batch_required' }> =>
|
||||
event.type === 'approval_batch_required'
|
||||
)
|
||||
.at(-1)
|
||||
const approvalAlreadyDecided = approvalRequired
|
||||
? events.some(
|
||||
? runEvents.some(
|
||||
(event) =>
|
||||
event.type === 'approval_batch_decision' && event.batchId === approvalRequired.batchId
|
||||
)
|
||||
|
||||
Reference in new issue
Block a user