Advance human live tests and simplify report evidence
This commit is contained in:
hy committed
1 parent
b3fcd23fa0
commit
bb3a374b61
12 files changed
+890
-55
No files matched your search
@@ -171,6 +171,261 @@ const toResultData = (value: JsonValue | undefined, key: string): JsonObject | u
|
||||
return { [key]: value }
|
||||
}
|
||||
|
||||
type DecisionDocumentRef = {
|
||||
documentName?: string
|
||||
documentId?: string
|
||||
}
|
||||
|
||||
const clipText = (value: string, maxChars = 120): string => {
|
||||
const text = value.trim()
|
||||
if (text.length <= maxChars) return text
|
||||
return `${text.slice(0, maxChars - 3)}...`
|
||||
}
|
||||
|
||||
const parseJsonValueText = (value: string): JsonValue | undefined => {
|
||||
const trimmed = value.trim()
|
||||
if (!trimmed) return undefined
|
||||
try {
|
||||
return JSON.parse(trimmed) as JsonValue
|
||||
} catch {
|
||||
const arrayStart = trimmed.indexOf('[')
|
||||
const arrayEnd = trimmed.lastIndexOf(']')
|
||||
if (arrayStart >= 0 && arrayEnd > arrayStart) {
|
||||
try {
|
||||
return JSON.parse(trimmed.slice(arrayStart, arrayEnd + 1)) as JsonValue
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
const objectStart = trimmed.indexOf('{')
|
||||
const objectEnd = trimmed.lastIndexOf('}')
|
||||
if (objectStart >= 0 && objectEnd > objectStart) {
|
||||
try {
|
||||
return JSON.parse(trimmed.slice(objectStart, objectEnd + 1)) as JsonValue
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
const extractDocumentRefFromRecord = (value: JsonObject): DecisionDocumentRef | null => {
|
||||
const documentName = toOptionalText(value.document_name) || toOptionalText(value.name)
|
||||
const documentId = toOptionalText(value.document_id) || toOptionalText(value.id)
|
||||
if (!documentName && !documentId) return null
|
||||
return {
|
||||
documentName,
|
||||
documentId
|
||||
}
|
||||
}
|
||||
|
||||
const collectDocumentRefsFromValue = (value: JsonValue | undefined): DecisionDocumentRef[] => {
|
||||
if (Array.isArray(value)) {
|
||||
return value.flatMap((item) => collectDocumentRefsFromValue(item))
|
||||
}
|
||||
|
||||
if (!isJsonObject(value)) return []
|
||||
|
||||
const refs: DecisionDocumentRef[] = []
|
||||
const directRef = extractDocumentRefFromRecord(value)
|
||||
if (directRef) refs.push(directRef)
|
||||
|
||||
const raw = toOptionalText(value.raw)
|
||||
if (raw) {
|
||||
refs.push(...collectDocumentRefsFromValue(parseJsonValueText(raw)))
|
||||
}
|
||||
|
||||
if (Array.isArray(value.items)) {
|
||||
refs.push(...collectDocumentRefsFromValue(value.items))
|
||||
}
|
||||
|
||||
return refs
|
||||
}
|
||||
|
||||
const dedupeDocumentRefs = (refs: DecisionDocumentRef[]): DecisionDocumentRef[] => {
|
||||
const seen = new Set<string>()
|
||||
const unique: DecisionDocumentRef[] = []
|
||||
|
||||
for (const ref of refs) {
|
||||
const key = `${ref.documentId || ''}::${ref.documentName || ''}`
|
||||
if (!ref.documentId && !ref.documentName) continue
|
||||
if (seen.has(key)) continue
|
||||
seen.add(key)
|
||||
unique.push(ref)
|
||||
}
|
||||
|
||||
return unique
|
||||
}
|
||||
|
||||
const formatDocumentRef = (ref: DecisionDocumentRef): string | undefined => {
|
||||
if (ref.documentName && ref.documentId) return `${ref.documentName} (${ref.documentId})`
|
||||
return ref.documentName || ref.documentId
|
||||
}
|
||||
|
||||
const countValueItems = (value: JsonValue | undefined): number => {
|
||||
if (Array.isArray(value)) return value.length
|
||||
if (isJsonObject(value) && Array.isArray(value.items)) return value.items.length
|
||||
return value === undefined ? 0 : 1
|
||||
}
|
||||
|
||||
const extractPrimaryEvidenceText = (value: JsonValue | undefined): string | undefined => {
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) {
|
||||
const nested = extractPrimaryEvidenceText(item)
|
||||
if (nested) return nested
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
if (typeof value === 'string') {
|
||||
const text = value.trim()
|
||||
return text ? clipText(text) : undefined
|
||||
}
|
||||
|
||||
if (!isJsonObject(value)) return undefined
|
||||
|
||||
const directText =
|
||||
toOptionalText(value.text) || toOptionalText(value.summary) || toOptionalText(value.message)
|
||||
if (directText) return clipText(directText)
|
||||
|
||||
const raw = toOptionalText(value.raw)
|
||||
if (raw) {
|
||||
const nested = extractPrimaryEvidenceText(parseJsonValueText(raw))
|
||||
if (nested) return nested
|
||||
}
|
||||
|
||||
if (Array.isArray(value.items)) {
|
||||
return extractPrimaryEvidenceText(value.items)
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
const resolveToolFamily = (toolName: string | undefined): string | undefined => {
|
||||
const normalized = String(toolName || '').trim()
|
||||
if (!normalized) return undefined
|
||||
const separatorIndex = normalized.indexOf('_')
|
||||
return separatorIndex > 0 ? normalized.slice(0, separatorIndex) : normalized
|
||||
}
|
||||
|
||||
const countTrailingSameToolName = (
|
||||
rows: Array<{ toolName: string; toolFamily: string; summary: string }>
|
||||
): number => {
|
||||
if (rows.length === 0) return 0
|
||||
let count = 1
|
||||
for (let index = rows.length - 1; index > 0; index -= 1) {
|
||||
if (rows[index]?.toolName !== rows[index - 1]?.toolName) break
|
||||
count += 1
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
const countTrailingSameToolFamily = (
|
||||
rows: Array<{ toolName: string; toolFamily: string; summary: string }>
|
||||
): number => {
|
||||
if (rows.length === 0) return 0
|
||||
let count = 1
|
||||
for (let index = rows.length - 1; index > 0; index -= 1) {
|
||||
if (rows[index]?.toolFamily !== rows[index - 1]?.toolFamily) break
|
||||
count += 1
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
const countTrailingSameSummary = (
|
||||
rows: Array<{ toolName: string; toolFamily: string; summary: string }>
|
||||
): number => {
|
||||
if (rows.length === 0) return 0
|
||||
let count = 1
|
||||
for (let index = rows.length - 1; index > 0; index -= 1) {
|
||||
if (rows[index]?.summary !== rows[index - 1]?.summary) break
|
||||
count += 1
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
const buildDecisionStateText = (input: { state: HumanRunState; ctx: AiRunContext }): string => {
|
||||
const { state, ctx } = input
|
||||
const remainingSteps = Math.max(0, state.maxSteps - state.steps)
|
||||
const objective = String(state.input.objective || '')
|
||||
const recentToolRows = state.stepHistory
|
||||
.filter(
|
||||
(step) =>
|
||||
step.action === 'invoke_tool' && typeof step.toolName === 'string' && step.toolName.length > 0
|
||||
)
|
||||
.slice(-4)
|
||||
.map((step) => ({
|
||||
toolName: String(step.toolName || ''),
|
||||
toolFamily: resolveToolFamily(String(step.toolName || '')) || 'unknown',
|
||||
resultStatus: step.resultStatus || 'unknown',
|
||||
summary: clipText(String(step.summary || ''))
|
||||
}))
|
||||
|
||||
let lastObservationSignals: JsonValue = {
|
||||
kind: 'none'
|
||||
}
|
||||
|
||||
if (state.lastObservation?.kind === 'skill_loaded') {
|
||||
lastObservationSignals = {
|
||||
kind: 'skill_loaded',
|
||||
skillId: state.lastObservation.skillId,
|
||||
activeSkillIds: state.lastObservation.activeSkillIds,
|
||||
activeToolNames: state.lastObservation.activeToolNames
|
||||
}
|
||||
} else if (state.lastObservation?.kind === 'terminal') {
|
||||
lastObservationSignals = {
|
||||
kind: 'terminal',
|
||||
status: state.lastObservation.status,
|
||||
code: state.lastObservation.code,
|
||||
summary: clipText(state.lastObservation.summary),
|
||||
hasData: state.lastObservation.data !== undefined
|
||||
}
|
||||
} else if (state.lastObservation?.kind === 'tool_result') {
|
||||
const uniqueRefs = dedupeDocumentRefs(collectDocumentRefsFromValue(state.lastObservation.data))
|
||||
const primarySourceRef = uniqueRefs.map((ref) => formatDocumentRef(ref)).find(Boolean)
|
||||
lastObservationSignals = {
|
||||
kind: 'tool_result',
|
||||
toolName: state.lastObservation.toolName,
|
||||
toolFamily: resolveToolFamily(state.lastObservation.toolName) || 'unknown',
|
||||
ok: state.lastObservation.ok,
|
||||
code: state.lastObservation.code,
|
||||
itemCount: countValueItems(state.lastObservation.data),
|
||||
uniqueSourceRefCount: uniqueRefs.length,
|
||||
primarySourceRef: primarySourceRef || 'none',
|
||||
primaryEvidenceText: extractPrimaryEvidenceText(state.lastObservation.data) || 'none',
|
||||
summary: clipText(state.lastObservation.summary)
|
||||
}
|
||||
}
|
||||
|
||||
return toJsonText({
|
||||
stepBudget: {
|
||||
currentStep: state.steps + 1,
|
||||
maxSteps: state.maxSteps,
|
||||
remainingSteps,
|
||||
lowBudget: remainingSteps <= 2
|
||||
},
|
||||
taskSignals: {
|
||||
deliveryKind: state.deliveryKind,
|
||||
workspaceBound: Boolean(ctx.workspace),
|
||||
selectedDocumentCount: ctx.selectedDocumentIds?.length || 0,
|
||||
mentionsDocx: /\.docx\b|docx/i.test(objective),
|
||||
mentionsPreview: /preview|预览/i.test(objective),
|
||||
mentionsSave: /save|保存|另存/i.test(objective)
|
||||
},
|
||||
progressSignals: {
|
||||
activeSkillCount: state.activeSkillIds.length,
|
||||
activeToolCount: state.activeTools.length,
|
||||
recentToolCalls: recentToolRows,
|
||||
trailingSameToolCount: countTrailingSameToolName(recentToolRows),
|
||||
trailingSameToolFamilyCount: countTrailingSameToolFamily(recentToolRows),
|
||||
trailingSameSummaryCount: countTrailingSameSummary(recentToolRows),
|
||||
recentErrorCount: recentToolRows.filter((row) => row.resultStatus === 'error').length
|
||||
},
|
||||
lastObservationSignals
|
||||
})
|
||||
}
|
||||
|
||||
const buildDecisionPrompt = (input: { state: HumanRunState; ctx: AiRunContext }) => {
|
||||
const { state } = input
|
||||
const prompt = buildHumanBrainPrompt({
|
||||
@@ -181,6 +436,7 @@ const buildDecisionPrompt = (input: { state: HumanRunState; ctx: AiRunContext })
|
||||
availableSkillCardsText: toJsonText(state.availableSkillCards),
|
||||
loadedSkillsText: serializeLoadedSkills(state),
|
||||
activeToolsText: serializeActiveTools(state),
|
||||
decisionStateText: buildDecisionStateText(input),
|
||||
lastObservationText: state.lastObservation ? toJsonText(state.lastObservation) : 'none',
|
||||
stepHistoryText:
|
||||
state.stepHistory.length > 0
|
||||
|
||||
@@ -98,10 +98,14 @@ const deriveLocalSourceStatus = (state: HumanRunState): {
|
||||
summary: string
|
||||
} => {
|
||||
const joined = [state.summary, state.deliveryMarkdown].join('\n')
|
||||
const mentionsInsufficient =
|
||||
joined.includes('本地资料不足') ||
|
||||
joined.includes('是否允许继续使用非本地信息') ||
|
||||
joined.includes('非本地信息')
|
||||
const insufficientMarkers = [
|
||||
'本地资料不足',
|
||||
'本地证据不足',
|
||||
'当前本地资料不足',
|
||||
'当前本地证据不足',
|
||||
'本地范围不足'
|
||||
]
|
||||
const mentionsInsufficient = insufficientMarkers.some((marker) => joined.includes(marker))
|
||||
if (mentionsInsufficient) {
|
||||
return {
|
||||
status: 'insufficient',
|
||||
|
||||
@@ -9,6 +9,7 @@ export type HumanBrainPromptInput = {
|
||||
availableSkillCardsText: string
|
||||
loadedSkillsText: string
|
||||
activeToolsText: string
|
||||
decisionStateText: string
|
||||
lastObservationText: string
|
||||
stepHistoryText: string
|
||||
}
|
||||
@@ -19,7 +20,9 @@ export const buildHumanBrainPrompt = (input: HumanBrainPromptInput) => {
|
||||
'【工具边界】只有 active tools 中列出的工具可以 invoke_tool。',
|
||||
'【完成标准】仅当任务目标已满足且证据充分时才 complete_task；summary 必须写清结论与关键证据。',
|
||||
'【结果收敛】若任务目标已明确要求 resultData 字段，且最近 observation 已经提供这些字段所需证据，下一步必须 complete_task 并写入 resultData，不要继续为同一证据反复调用 KB 或 shell。',
|
||||
'【KB 收敛】若 kb_hybrid_search_chunks 的 observation 已经出现唯一的 document_name/document_id，或首条文本已经直接给出答案，不要再调用 kb_list_documents / 再次搜索来“确认同一证据”；应直接 complete_task。',
|
||||
'【证据收敛】若最近 observation 已经足以唯一支撑当前结论或下一交付，优先 complete_task；不要为了“再次确认同一证据”继续调用等价工具。',
|
||||
'【无新增信息】若最近连续几步没有带来新信息，优先切换能力、缩小目标或明确资料不足；不要只是在同类工具之间来回试探。',
|
||||
'【最小路径】优先完成满足 objective 与 delivery kind 的最小交付路径；若任务只要求 answer、handoff 或 preview，不要扩展成非必需的保存、重复检索或额外产物。',
|
||||
'【阻塞标准】workspace_required/approval_denied 等不可恢复条件必须 block_task。',
|
||||
'【反循环】最近一步没有带来新信息时，禁止重复相同 toolName + args；应切换 skill、调整参数或收敛。',
|
||||
'【禁止过度复核】同一文件名、同一命令输出或同一 KB 证据已经足够支撑结论时，不要再做“再次确认”式工具调用。',
|
||||
@@ -45,6 +48,7 @@ export const buildHumanBrainPrompt = (input: HumanBrainPromptInput) => {
|
||||
{ label: 'Available Skill Cards', value: input.availableSkillCardsText },
|
||||
{ label: 'Loaded Skills', value: input.loadedSkillsText },
|
||||
{ label: 'Active Tools', value: input.activeToolsText },
|
||||
{ label: 'Decision State', value: input.decisionStateText },
|
||||
{ label: 'Last Observation', value: input.lastObservationText },
|
||||
{ label: 'Recent Steps', value: input.stepHistoryText }
|
||||
])
|
||||
|
||||
Reference in new issue
Block a user