优化工作流实现
This commit is contained in:
1 parent
904a6c53f4
commit
ba005662e4
10 files changed
+931
-492
No files matched your search
@@ -23,30 +23,26 @@ import { Video, Send, Type, Play } from 'lucide-react';
|
||||
import { useBoard } from '@plait-board/react-board';
|
||||
import { getSelectedElements, ATTACHED_ELEMENT_CLASS_NAME } from '@plait/core';
|
||||
import { useI18n } from '../../i18n';
|
||||
import { useTaskQueue } from '../../hooks/useTaskQueue';
|
||||
import { TaskType, TaskStatus } from '../../types/task.types';
|
||||
import { TaskStatus } from '../../types/task.types';
|
||||
import { taskQueueService } from '../../services/task-queue-service';
|
||||
import { processSelectedContentForAI } from '../../utils/selection-utils';
|
||||
import { VIDEO_MODEL_CONFIGS } from '../../constants/video-model-config';
|
||||
import type { VideoModel } from '../../types/video.types';
|
||||
import { useTextSelection } from '../../hooks/useTextSelection';
|
||||
import { usePromptHistory } from '../../hooks/usePromptHistory';
|
||||
import { useChatDrawerControl } from '../../contexts/ChatDrawerContext';
|
||||
import { AI_INSTRUCTIONS } from '../../constants/prompts';
|
||||
import {
|
||||
SmartSuggestionPanel,
|
||||
import {
|
||||
SmartSuggestionPanel,
|
||||
useTriggerDetection,
|
||||
insertToInput,
|
||||
type PromptItem,
|
||||
} from './smart-suggestion-panel';
|
||||
import { agentExecutor } from '../../services/agent';
|
||||
import { initializeMCP, setCanvasBoard } from '../../mcp';
|
||||
import { initializeMCP, setCanvasBoard, mcpRegistry } from '../../mcp';
|
||||
import type { MCPTaskResult } from '../../mcp/types';
|
||||
import { parseAIInput } from '../../utils/ai-input-parser';
|
||||
import { convertToWorkflow, type WorkflowDefinition } from './workflow-converter';
|
||||
import { useWorkflowControl } from '../../contexts/WorkflowContext';
|
||||
import { geminiSettings } from '../../utils/settings-manager';
|
||||
import type { WorkflowMessageData } from '../../types/chat.types';
|
||||
import type { AgentExecutionContext } from '../../mcp/types';
|
||||
import classNames from 'classnames';
|
||||
import './ai-input-bar.scss';
|
||||
|
||||
@@ -205,8 +201,6 @@ export const AIInputBar: React.FC<AIInputBarProps> = React.memo(({ className })
|
||||
|
||||
const { language } = useI18n();
|
||||
|
||||
// 只获取需要的函数,避免整个对象变化导致重渲染
|
||||
const { createTask } = useTaskQueue();
|
||||
const { history, addHistory, removeHistory } = usePromptHistory();
|
||||
const chatDrawerControl = useChatDrawerControl();
|
||||
const workflowControl = useWorkflowControl();
|
||||
@@ -295,6 +289,18 @@ export const AIInputBar: React.FC<AIInputBarProps> = React.memo(({ className })
|
||||
if (newStatus !== step.status) {
|
||||
workflowControl.updateStep(step.id, newStatus, stepResult, stepError);
|
||||
|
||||
// 如果步骤失败,将同批次中其他 running 状态的步骤标记为 skipped
|
||||
if (newStatus === 'failed') {
|
||||
const stepBatchId = step.options?.batchId;
|
||||
if (stepBatchId) {
|
||||
workflow.steps.forEach((s) => {
|
||||
if (s.id !== step.id && s.options?.batchId === stepBatchId && s.status === 'running') {
|
||||
workflowControl.updateStep(s.id, 'skipped', undefined, '前置任务失败');
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 同步更新 ChatDrawer 中的工作流消息
|
||||
const updatedWorkflow = workflowControl.getWorkflow();
|
||||
if (updatedWorkflow) {
|
||||
@@ -449,221 +455,110 @@ export const AIInputBar: React.FC<AIInputBarProps> = React.memo(({ className })
|
||||
textModel, // 传递全局文本模型
|
||||
});
|
||||
|
||||
// 获取最终 prompt(用于任务创建)
|
||||
const finalPrompt = parsedParams.prompt;
|
||||
// 统一处理:遍历工作流步骤,通过 MCP Registry 执行
|
||||
console.log(`[AIInputBar] Executing workflow: ${workflow.steps.length} steps, scenario: ${parsedParams.scenario}`);
|
||||
|
||||
// 根据场景处理
|
||||
if (parsedParams.scenario === 'direct_generation') {
|
||||
// 场景 1-3: 直接生成(无额外内容)
|
||||
// 直接创建任务添加到任务队列
|
||||
|
||||
const { generationType, modelId, count, size, duration } = parsedParams;
|
||||
|
||||
console.log(`[AIInputBar] Direct generation: type=${generationType}, model=${modelId}, count=${count}, size=${size}, duration=${duration}`);
|
||||
|
||||
// 将参考图片转换为 uploadedImages 格式(与 AI 图片/视频弹窗一致)
|
||||
const uploadedImages = referenceImages.map((url, index) => ({
|
||||
type: 'url' as const,
|
||||
url,
|
||||
name: `reference-${index + 1}`,
|
||||
}));
|
||||
|
||||
// 根据数量创建多个任务,并更新工作流步骤状态
|
||||
// 注意:任务创建后状态为 PENDING,需要等任务实际完成后才更新步骤为 completed
|
||||
// 这里使用并行创建任务,步骤状态通过 taskQueueService 的事件订阅来更新
|
||||
const createdTaskIds: string[] = [];
|
||||
const createdTaskIds: string[] = [];
|
||||
|
||||
// 生成批次ID和全局计数器,用于区分同一批次中的不同任务(与批量图片生成弹窗保持一致)
|
||||
const batchId = `input_${Date.now()}`;
|
||||
let globalIndex = 0;
|
||||
// 创建标准回调(所有工具都可使用,不需要的会忽略)
|
||||
const createStepCallbacks = (currentStep: typeof workflow.steps[0], stepStartTime: number) => ({
|
||||
// 流式输出回调
|
||||
onChunk: (chunk: string) => {
|
||||
updateThinkingContentRef.current(chunk);
|
||||
},
|
||||
// 动态添加步骤回调
|
||||
onAddSteps: (newSteps: Array<{ id: string; mcp: string; args: Record<string, unknown>; description: string; status: string }>) => {
|
||||
// 当前步骤完成
|
||||
workflowControl.updateStep(currentStep.id, 'completed', { analysis: 'completed' }, undefined, Date.now() - stepStartTime);
|
||||
|
||||
for (let i = 0; i < count; i++) {
|
||||
const stepId = `step-${i + 1}`;
|
||||
// 添加新步骤到工作流
|
||||
workflowControl.addSteps(newSteps.map(s => ({
|
||||
...s,
|
||||
status: s.status as 'pending' | 'running' | 'completed' | 'failed' | 'skipped',
|
||||
})));
|
||||
|
||||
// 更新步骤为运行中(表示任务正在排队或执行中)
|
||||
workflowControl.updateStep(stepId, 'running');
|
||||
|
||||
try {
|
||||
globalIndex++;
|
||||
let task;
|
||||
if (generationType === 'image') {
|
||||
task = createTask(
|
||||
{
|
||||
prompt: finalPrompt,
|
||||
size: size || '1x1',
|
||||
uploadedImages: uploadedImages.length > 0 ? uploadedImages : undefined,
|
||||
model: modelId,
|
||||
// 批量参数:确保同一批次中的不同任务有唯一哈希(与批量图片生成弹窗保持一致)
|
||||
batchId,
|
||||
batchIndex: i + 1,
|
||||
batchTotal: count,
|
||||
globalIndex,
|
||||
},
|
||||
TaskType.IMAGE
|
||||
);
|
||||
} else {
|
||||
// 视频任务
|
||||
const modelConfig = VIDEO_MODEL_CONFIGS[modelId as VideoModel] || VIDEO_MODEL_CONFIGS['veo3'];
|
||||
task = createTask(
|
||||
{
|
||||
prompt: finalPrompt,
|
||||
size: size || '16x9',
|
||||
duration: parseInt(duration || modelConfig.defaultDuration, 10),
|
||||
model: modelId,
|
||||
uploadedImages: uploadedImages.length > 0 ? uploadedImages : undefined,
|
||||
// 批量参数:确保同一批次中的不同任务有唯一哈希(与批量图片生成弹窗保持一致)
|
||||
batchId,
|
||||
batchIndex: i + 1,
|
||||
batchTotal: count,
|
||||
globalIndex,
|
||||
},
|
||||
TaskType.VIDEO
|
||||
);
|
||||
}
|
||||
|
||||
if (task) {
|
||||
createdTaskIds.push(task.id);
|
||||
// 记录任务 ID 到步骤中,便于后续状态同步
|
||||
workflowControl.updateStep(stepId, 'running', { taskId: task.id });
|
||||
} else {
|
||||
// 任务创建失败
|
||||
workflowControl.updateStep(stepId, 'failed', undefined, '任务创建失败');
|
||||
}
|
||||
} catch (stepError) {
|
||||
// 更新步骤为失败
|
||||
workflowControl.updateStep(stepId, 'failed', undefined, String(stepError));
|
||||
}
|
||||
}
|
||||
|
||||
// 同步更新 ChatDrawer 中的工作流消息
|
||||
const updatedWorkflow = workflowControl.getWorkflow();
|
||||
if (updatedWorkflow) {
|
||||
updateWorkflowMessageRef.current(toWorkflowMessageData(updatedWorkflow));
|
||||
}
|
||||
} else {
|
||||
// 场景 4: Agent 流程
|
||||
// 有额外内容,需要调用 Agent
|
||||
console.log('[AIInputBar] Using Agent mode with model:', parsedParams.modelId);
|
||||
|
||||
// 更新分析步骤为运行中
|
||||
const analyzeStartTime = Date.now();
|
||||
workflowControl.updateStep('step-analyze', 'running');
|
||||
// 同步更新 ChatDrawer 中的工作流消息
|
||||
const runningWorkflow = workflowControl.getWorkflow();
|
||||
if (runningWorkflow) {
|
||||
updateWorkflowMessageRef.current(toWorkflowMessageData(runningWorkflow));
|
||||
}
|
||||
|
||||
// 构建 Agent 执行上下文
|
||||
const agentContext: AgentExecutionContext = {
|
||||
userInstruction: parsedParams.userInstruction,
|
||||
rawInput: parsedParams.rawInput,
|
||||
model: {
|
||||
id: parsedParams.modelId,
|
||||
type: parsedParams.generationType,
|
||||
isExplicit: parsedParams.isModelExplicit,
|
||||
},
|
||||
params: {
|
||||
count: parsedParams.count,
|
||||
size: parsedParams.size,
|
||||
duration: parsedParams.duration,
|
||||
},
|
||||
selection,
|
||||
finalPrompt,
|
||||
};
|
||||
|
||||
// 记录当前工具调用的名称,用于日志
|
||||
let currentToolName = '';
|
||||
|
||||
const result = await agentExecutor.execute(agentContext, {
|
||||
model: parsedParams.modelId,
|
||||
onChunk: (chunk) => {
|
||||
console.log('[AIInputBar] Agent chunk:', chunk);
|
||||
// 流式追加 AI 思考内容到日志
|
||||
updateThinkingContentRef.current(chunk);
|
||||
},
|
||||
onToolCall: (toolCall) => {
|
||||
console.log('[AIInputBar] Agent calling tool:', toolCall.name);
|
||||
currentToolName = toolCall.name;
|
||||
|
||||
// 分析步骤完成,添加工具调用步骤
|
||||
if (workflowControl.getWorkflow()?.steps.find(s => s.id === 'step-analyze')?.status === 'running') {
|
||||
workflowControl.updateStep('step-analyze', 'completed', { analysis: 'completed' }, undefined, Date.now() - analyzeStartTime);
|
||||
}
|
||||
|
||||
// 动态添加工具调用步骤
|
||||
const newStepId = `step-tool-${Date.now()}`;
|
||||
workflowControl.addSteps([{
|
||||
id: newStepId,
|
||||
mcp: toolCall.name,
|
||||
args: toolCall.arguments || {},
|
||||
description: `执行 ${toolCall.name}`,
|
||||
status: 'running',
|
||||
}]);
|
||||
|
||||
// 追加工具调用日志
|
||||
// 追加工具调用日志
|
||||
newSteps.forEach(s => {
|
||||
appendAgentLogRef.current({
|
||||
type: 'tool_call',
|
||||
timestamp: Date.now(),
|
||||
toolName: toolCall.name,
|
||||
args: toolCall.arguments || {},
|
||||
toolName: s.mcp,
|
||||
args: s.args,
|
||||
});
|
||||
});
|
||||
|
||||
// 同步更新 ChatDrawer 中的工作流消息
|
||||
const toolCallWorkflow = workflowControl.getWorkflow();
|
||||
if (toolCallWorkflow) {
|
||||
updateWorkflowMessageRef.current(toWorkflowMessageData(toolCallWorkflow));
|
||||
}
|
||||
},
|
||||
onToolResult: (toolResult) => {
|
||||
console.log('[AIInputBar] Tool result:', toolResult);
|
||||
updateWorkflowMessageRef.current(toWorkflowMessageData(workflowControl.getWorkflow()!));
|
||||
},
|
||||
// 更新步骤状态回调
|
||||
onUpdateStep: (stepId: string, status: string, result?: unknown, error?: string) => {
|
||||
workflowControl.updateStep(stepId, status as 'pending' | 'running' | 'completed' | 'failed' | 'skipped', result, error);
|
||||
|
||||
// 追加工具结果日志
|
||||
appendAgentLogRef.current({
|
||||
type: 'tool_result',
|
||||
timestamp: Date.now(),
|
||||
toolName: currentToolName,
|
||||
success: toolResult.success,
|
||||
data: toolResult.data,
|
||||
error: toolResult.error,
|
||||
resultType: toolResult.type,
|
||||
});
|
||||
// 追加工具结果日志
|
||||
appendAgentLogRef.current({
|
||||
type: 'tool_result',
|
||||
timestamp: Date.now(),
|
||||
toolName: stepId,
|
||||
success: status === 'completed',
|
||||
data: result,
|
||||
error,
|
||||
});
|
||||
|
||||
// MCP 工具已经完成了生成任务并返回了 URL
|
||||
// 这里只需要更新工作流状态,不需要再创建任务
|
||||
// 因为 MCP 工具内部已经调用了生成 API 并等待完成
|
||||
updateWorkflowMessageRef.current(toWorkflowMessageData(workflowControl.getWorkflow()!));
|
||||
},
|
||||
});
|
||||
|
||||
// 同步更新 ChatDrawer 中的工作流消息
|
||||
const toolResultWorkflow = workflowControl.getWorkflow();
|
||||
if (toolResultWorkflow) {
|
||||
updateWorkflowMessageRef.current(toWorkflowMessageData(toolResultWorkflow));
|
||||
}
|
||||
let workflowFailed = false;
|
||||
|
||||
// 如果生成成功,结果已经在 toolResult.data 中
|
||||
// TODO: 这里可以直接把生成的图片/视频添加到画布上
|
||||
// 目前 MCP 工具返回的是 URL,需要决定是否自动添加到画布
|
||||
if (toolResult.success && toolResult.data) {
|
||||
const data = toolResult.data as any;
|
||||
console.log('[AIInputBar] Generation completed, URL:', data.url);
|
||||
// 未来可以在这里调用 addToCanvas(data.url, toolResult.type)
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
if (!result.success && result.error) {
|
||||
console.error('[AIInputBar] Agent execution failed:', result.error);
|
||||
// 更新分析步骤为失败(如果还在运行中)
|
||||
const currentWorkflow = workflowControl.getWorkflow();
|
||||
const analyzeStep = currentWorkflow?.steps.find(s => s.id === 'step-analyze');
|
||||
if (analyzeStep?.status === 'running') {
|
||||
workflowControl.updateStep('step-analyze', 'failed', undefined, result.error, Date.now() - analyzeStartTime);
|
||||
// 同步更新 ChatDrawer 中的工作流消息
|
||||
const failedWorkflow = workflowControl.getWorkflow();
|
||||
if (failedWorkflow) {
|
||||
updateWorkflowMessageRef.current(toWorkflowMessageData(failedWorkflow));
|
||||
}
|
||||
}
|
||||
for (const step of workflow.steps) {
|
||||
// 如果工作流已失败,跳过剩余步骤
|
||||
if (workflowFailed) {
|
||||
workflowControl.updateStep(step.id, 'skipped');
|
||||
updateWorkflowMessageRef.current(toWorkflowMessageData(workflowControl.getWorkflow()!));
|
||||
continue;
|
||||
}
|
||||
|
||||
const stepStartTime = Date.now();
|
||||
|
||||
// 更新步骤为运行中
|
||||
workflowControl.updateStep(step.id, 'running');
|
||||
updateWorkflowMessageRef.current(toWorkflowMessageData(workflowControl.getWorkflow()!));
|
||||
|
||||
try {
|
||||
// 合并步骤选项和标准回调(工具自行决定是否使用回调)
|
||||
const executeOptions = {
|
||||
...step.options,
|
||||
...createStepCallbacks(step, stepStartTime),
|
||||
};
|
||||
|
||||
// 通过 MCP Registry 执行工具
|
||||
const result = await mcpRegistry.executeTool(
|
||||
{ name: step.mcp, arguments: step.args },
|
||||
executeOptions
|
||||
) as MCPTaskResult;
|
||||
|
||||
// 根据结果更新步骤状态
|
||||
const currentStepStatus = workflowControl.getWorkflow()?.steps.find(s => s.id === step.id)?.status;
|
||||
|
||||
if (!result.success) {
|
||||
// 执行失败,标记工作流失败
|
||||
workflowControl.updateStep(step.id, 'failed', undefined, result.error || '执行失败', Date.now() - stepStartTime);
|
||||
workflowFailed = true;
|
||||
} else if (result.taskId) {
|
||||
// 队列模式:记录任务 ID(状态保持 running,等任务完成后更新)
|
||||
createdTaskIds.push(result.taskId);
|
||||
workflowControl.updateStep(step.id, 'running', { taskId: result.taskId });
|
||||
} else if (currentStepStatus === 'running') {
|
||||
// 同步模式且未被回调更新:标记为完成
|
||||
workflowControl.updateStep(step.id, 'completed', result.data, undefined, Date.now() - stepStartTime);
|
||||
}
|
||||
} catch (stepError) {
|
||||
// 更新步骤为失败,标记工作流失败
|
||||
workflowControl.updateStep(step.id, 'failed', undefined, String(stepError));
|
||||
workflowFailed = true;
|
||||
}
|
||||
|
||||
// 同步更新 ChatDrawer 中的工作流消息
|
||||
updateWorkflowMessageRef.current(toWorkflowMessageData(workflowControl.getWorkflow()!));
|
||||
}
|
||||
|
||||
// 清空输入并关闭面板
|
||||
@@ -679,7 +574,7 @@ export const AIInputBar: React.FC<AIInputBarProps> = React.memo(({ className })
|
||||
} finally {
|
||||
setIsSubmitting(false);
|
||||
}
|
||||
}, [prompt, selectedContent, createTask, isSubmitting, addHistory, workflowControl]);
|
||||
}, [prompt, selectedContent, isSubmitting, addHistory, workflowControl]);
|
||||
|
||||
// Handle key press
|
||||
const handleKeyDown = useCallback(
|
||||
|
||||
@@ -11,6 +11,22 @@
|
||||
|
||||
import type { ParsedGenerationParams, GenerationType, SelectionInfo } from '../../utils/ai-input-parser';
|
||||
|
||||
/**
|
||||
* 工作流步骤执行选项(批量参数等)
|
||||
*/
|
||||
export interface WorkflowStepOptions {
|
||||
/** 执行模式 */
|
||||
mode?: 'async' | 'queue';
|
||||
/** 批次 ID */
|
||||
batchId?: string;
|
||||
/** 批次索引(1-based) */
|
||||
batchIndex?: number;
|
||||
/** 批次总数 */
|
||||
batchTotal?: number;
|
||||
/** 全局索引 */
|
||||
globalIndex?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* 工作流步骤定义
|
||||
*/
|
||||
@@ -21,6 +37,8 @@ export interface WorkflowStep {
|
||||
mcp: string;
|
||||
/** 工具参数 */
|
||||
args: Record<string, unknown>;
|
||||
/** 执行选项(批量参数等) */
|
||||
options?: WorkflowStepOptions;
|
||||
/** 步骤描述 */
|
||||
description: string;
|
||||
/** 步骤状态 */
|
||||
@@ -85,8 +103,9 @@ function generateId(): string {
|
||||
|
||||
/**
|
||||
* 场景1-3: 将直接生成场景转换为工作流定义
|
||||
*
|
||||
*
|
||||
* 这些场景通过正则解析用户输入,直接生成图片/视频
|
||||
* 步骤中包含完整的工具调用信息(mcp、args、options),调用方可直接执行
|
||||
*/
|
||||
export function convertDirectGenerationToWorkflow(
|
||||
params: ParsedGenerationParams,
|
||||
@@ -104,13 +123,25 @@ export function convertDirectGenerationToWorkflow(
|
||||
duration,
|
||||
selection,
|
||||
} = params;
|
||||
|
||||
|
||||
const steps: WorkflowStep[] = [];
|
||||
|
||||
|
||||
// 生成批次 ID(用于区分同一批次中的不同任务)
|
||||
const batchId = `workflow_${Date.now()}`;
|
||||
|
||||
// 根据数量创建多个生成步骤
|
||||
for (let i = 0; i < count; i++) {
|
||||
const stepId = `step-${i + 1}`;
|
||||
|
||||
|
||||
// 通用的执行选项
|
||||
const options: WorkflowStepOptions = {
|
||||
mode: 'queue',
|
||||
batchId,
|
||||
batchIndex: i + 1,
|
||||
batchTotal: count,
|
||||
globalIndex: i + 1,
|
||||
};
|
||||
|
||||
if (generationType === 'image') {
|
||||
steps.push({
|
||||
id: stepId,
|
||||
@@ -121,9 +152,8 @@ export function convertDirectGenerationToWorkflow(
|
||||
size: size || '1x1',
|
||||
referenceImages: referenceImages.length > 0 ? referenceImages : undefined,
|
||||
},
|
||||
description: count > 1
|
||||
? `生成图片 (${i + 1}/${count})`
|
||||
: '生成图片',
|
||||
options,
|
||||
description: count > 1 ? `生成图片 (${i + 1}/${count})` : '生成图片',
|
||||
status: 'pending',
|
||||
});
|
||||
} else {
|
||||
@@ -135,17 +165,16 @@ export function convertDirectGenerationToWorkflow(
|
||||
prompt,
|
||||
model: modelId,
|
||||
size: size || '16x9',
|
||||
duration: duration || '5',
|
||||
seconds: duration || '5',
|
||||
referenceImages: referenceImages.length > 0 ? referenceImages : undefined,
|
||||
},
|
||||
description: count > 1
|
||||
? `生成视频 (${i + 1}/${count})`
|
||||
: '生成视频',
|
||||
options,
|
||||
description: count > 1 ? `生成视频 (${i + 1}/${count})` : '生成视频',
|
||||
status: 'pending',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return {
|
||||
id: generateId(),
|
||||
name: generationType === 'image' ? '图片生成' : '视频生成',
|
||||
@@ -171,9 +200,10 @@ export function convertDirectGenerationToWorkflow(
|
||||
|
||||
/**
|
||||
* 场景4: 将 Agent 流程转换为工作流定义
|
||||
*
|
||||
* 这个场景需要先调用文本模型分析用户意图,然后根据分析结果生成工作流
|
||||
* 初始只创建一个"分析"步骤,后续步骤由 AI 动态生成
|
||||
*
|
||||
* 这个场景需要先调用文本模型分析用户意图,然后根据分析结果执行工具
|
||||
* 初始只创建一个 ai_analyze 步骤,后续步骤由 AI 动态生成
|
||||
* 步骤中包含完整的工具调用信息,调用方可直接执行
|
||||
*/
|
||||
export function convertAgentFlowToWorkflow(
|
||||
params: ParsedGenerationParams,
|
||||
@@ -192,28 +222,36 @@ export function convertAgentFlowToWorkflow(
|
||||
selection,
|
||||
} = params;
|
||||
|
||||
// Agent 流程初始只有一个分析步骤
|
||||
// 构建 Agent 执行上下文(与 AgentExecutionContext 类型一致)
|
||||
const agentContext = {
|
||||
userInstruction,
|
||||
rawInput,
|
||||
model: {
|
||||
id: modelId,
|
||||
type: generationType,
|
||||
isExplicit: isModelExplicit,
|
||||
},
|
||||
params: {
|
||||
count,
|
||||
size,
|
||||
duration,
|
||||
},
|
||||
selection,
|
||||
finalPrompt: prompt,
|
||||
};
|
||||
|
||||
// Agent 流程初始只有一个 ai_analyze 步骤
|
||||
// 后续步骤会在 AI 分析后动态添加
|
||||
const steps: WorkflowStep[] = [
|
||||
{
|
||||
id: 'step-analyze',
|
||||
mcp: 'ai_analyze',
|
||||
args: {
|
||||
// 用户指令(包含额外要求)
|
||||
userInstruction,
|
||||
// 选中的文本作为生成 prompt
|
||||
selectedTexts: selection.texts,
|
||||
context: {
|
||||
generationType,
|
||||
modelId,
|
||||
isModelExplicit,
|
||||
count,
|
||||
size,
|
||||
duration,
|
||||
hasReferenceImages: referenceImages.length > 0,
|
||||
hasSelectedTexts: selection.texts.length > 0,
|
||||
hasSelectedVideos: selection.videos.length > 0,
|
||||
},
|
||||
context: agentContext,
|
||||
textModel: modelId,
|
||||
},
|
||||
options: {
|
||||
mode: 'async',
|
||||
},
|
||||
description: 'AI 分析用户意图',
|
||||
status: 'pending',
|
||||
@@ -223,7 +261,7 @@ export function convertAgentFlowToWorkflow(
|
||||
return {
|
||||
id: generateId(),
|
||||
name: 'AI 智能生成',
|
||||
description: `AI 分析用户请求并执行相应操作`,
|
||||
description: 'AI 分析用户请求并执行相应操作',
|
||||
scenarioType: 'agent_flow',
|
||||
generationType,
|
||||
steps,
|
||||
|
||||
@@ -76,79 +76,87 @@ export interface InstructionItemData {
|
||||
// AI 指令 - 用于文本模型生成工作流(AI 输入框使用)
|
||||
export const AI_INSTRUCTIONS: Record<'zh' | 'en', InstructionItemData[]> = {
|
||||
zh: [
|
||||
// {
|
||||
// content: '将选中的内容整理成思维导图',
|
||||
// scene: '知识梳理、内容结构化、学习笔记'
|
||||
// },
|
||||
// {
|
||||
// content: '为选中的主题生成详细的分析框架',
|
||||
// scene: '项目分析、问题拆解、决策支持'
|
||||
// },
|
||||
// {
|
||||
// content: '将文本内容转换为流程图',
|
||||
// scene: '流程可视化、步骤说明、操作指南'
|
||||
// },
|
||||
// {
|
||||
// content: '为这个概念生成相关的扩展内容',
|
||||
// scene: '头脑风暴、创意发散、内容扩展'
|
||||
// },
|
||||
// {
|
||||
// content: '总结并提炼关键要点',
|
||||
// scene: '内容摘要、报告总结、快速了解'
|
||||
// },
|
||||
// {
|
||||
// content: '分析这些内容之间的关联关系',
|
||||
// scene: '关系分析、逻辑梳理、知识图谱'
|
||||
// },
|
||||
// {
|
||||
// content: '为当前内容生成行动计划',
|
||||
// scene: '任务规划、项目管理、执行方案'
|
||||
// },
|
||||
// {
|
||||
// content: '对比分析这些选项的优缺点',
|
||||
// scene: '方案对比、决策分析、选型评估'
|
||||
// },
|
||||
// {
|
||||
// content: '生成一张与主题相关的配图',
|
||||
// scene: '视觉辅助、内容配图、演示美化'
|
||||
// },
|
||||
{
|
||||
content: '将选中的内容整理成思维导图',
|
||||
scene: '知识梳理、内容结构化、学习笔记'
|
||||
},
|
||||
{
|
||||
content: '为选中的主题生成详细的分析框架',
|
||||
scene: '项目分析、问题拆解、决策支持'
|
||||
},
|
||||
{
|
||||
content: '将文本内容转换为流程图',
|
||||
scene: '流程可视化、步骤说明、操作指南'
|
||||
},
|
||||
{
|
||||
content: '为这个概念生成相关的扩展内容',
|
||||
scene: '头脑风暴、创意发散、内容扩展'
|
||||
},
|
||||
{
|
||||
content: '总结并提炼关键要点',
|
||||
scene: '内容摘要、报告总结、快速了解'
|
||||
},
|
||||
{
|
||||
content: '分析这些内容之间的关联关系',
|
||||
scene: '关系分析、逻辑梳理、知识图谱'
|
||||
},
|
||||
{
|
||||
content: '为当前内容生成行动计划',
|
||||
scene: '任务规划、项目管理、执行方案'
|
||||
},
|
||||
{
|
||||
content: '对比分析这些选项的优缺点',
|
||||
scene: '方案对比、决策分析、选型评估'
|
||||
},
|
||||
{
|
||||
content: '生成一张与主题相关的配图',
|
||||
scene: '视觉辅助、内容配图、演示美化'
|
||||
content: '优化提示词',
|
||||
scene: '提示词优化、效果提升、Prompt润色'
|
||||
}
|
||||
],
|
||||
en: [
|
||||
// {
|
||||
// content: 'Organize the selected content into a mind map',
|
||||
// scene: 'Knowledge organization, content structuring, study notes'
|
||||
// },
|
||||
// {
|
||||
// content: 'Generate a detailed analysis framework for the selected topic',
|
||||
// scene: 'Project analysis, problem breakdown, decision support'
|
||||
// },
|
||||
// {
|
||||
// content: 'Convert the text content into a flowchart',
|
||||
// scene: 'Process visualization, step explanation, operation guide'
|
||||
// },
|
||||
// {
|
||||
// content: 'Generate related extended content for this concept',
|
||||
// scene: 'Brainstorming, creative expansion, content extension'
|
||||
// },
|
||||
// {
|
||||
// content: 'Summarize and extract key points',
|
||||
// scene: 'Content summary, report summary, quick overview'
|
||||
// },
|
||||
// {
|
||||
// content: 'Analyze the relationships between these contents',
|
||||
// scene: 'Relationship analysis, logic organization, knowledge graph'
|
||||
// },
|
||||
// {
|
||||
// content: 'Generate an action plan for the current content',
|
||||
// scene: 'Task planning, project management, execution plan'
|
||||
// },
|
||||
// {
|
||||
// content: 'Compare and analyze the pros and cons of these options',
|
||||
// scene: 'Solution comparison, decision analysis, selection evaluation'
|
||||
// },
|
||||
// {
|
||||
// content: 'Generate an image related to the topic',
|
||||
// scene: 'Visual aid, content illustration, presentation enhancement'
|
||||
// },
|
||||
{
|
||||
content: 'Organize the selected content into a mind map',
|
||||
scene: 'Knowledge organization, content structuring, study notes'
|
||||
},
|
||||
{
|
||||
content: 'Generate a detailed analysis framework for the selected topic',
|
||||
scene: 'Project analysis, problem breakdown, decision support'
|
||||
},
|
||||
{
|
||||
content: 'Convert the text content into a flowchart',
|
||||
scene: 'Process visualization, step explanation, operation guide'
|
||||
},
|
||||
{
|
||||
content: 'Generate related extended content for this concept',
|
||||
scene: 'Brainstorming, creative expansion, content extension'
|
||||
},
|
||||
{
|
||||
content: 'Summarize and extract key points',
|
||||
scene: 'Content summary, report summary, quick overview'
|
||||
},
|
||||
{
|
||||
content: 'Analyze the relationships between these contents',
|
||||
scene: 'Relationship analysis, logic organization, knowledge graph'
|
||||
},
|
||||
{
|
||||
content: 'Generate an action plan for the current content',
|
||||
scene: 'Task planning, project management, execution plan'
|
||||
},
|
||||
{
|
||||
content: 'Compare and analyze the pros and cons of these options',
|
||||
scene: 'Solution comparison, decision analysis, selection evaluation'
|
||||
},
|
||||
{
|
||||
content: 'Generate an image related to the topic',
|
||||
scene: 'Visual aid, content illustration, presentation enhancement'
|
||||
content: 'Optimize the prompt',
|
||||
scene: 'Prompt optimization, Quality improvement, Prompt polishing'
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
@@ -26,6 +26,7 @@ import { mcpRegistry } from './registry';
|
||||
import { imageGenerationTool } from './tools/image-generation';
|
||||
import { videoGenerationTool } from './tools/video-generation';
|
||||
import { canvasInsertionTool } from './tools/canvas-insertion';
|
||||
import { aiAnalyzeTool } from './tools/ai-analyze';
|
||||
|
||||
/**
|
||||
* 初始化 MCP 模块,注册所有内置工具
|
||||
@@ -35,6 +36,7 @@ export function initializeMCP(): void {
|
||||
imageGenerationTool,
|
||||
videoGenerationTool,
|
||||
canvasInsertionTool,
|
||||
aiAnalyzeTool,
|
||||
]);
|
||||
console.log('[MCP] Initialized with built-in tools');
|
||||
}
|
||||
@@ -4,7 +4,7 @@
|
||||
* 管理所有 MCP 工具的注册、查询和执行
|
||||
*/
|
||||
|
||||
import type { MCPTool, MCPResult, ToolCall } from './types';
|
||||
import type { MCPTool, MCPResult, ToolCall, MCPExecuteOptions } from './types';
|
||||
|
||||
/**
|
||||
* MCP 工具注册中心
|
||||
@@ -84,10 +84,12 @@ class MCPRegistry {
|
||||
|
||||
/**
|
||||
* 执行工具调用
|
||||
* @param toolCall - 工具调用信息
|
||||
* @param options - 执行选项(可选,用于指定执行模式等)
|
||||
*/
|
||||
async executeTool(toolCall: ToolCall): Promise<MCPResult> {
|
||||
async executeTool(toolCall: ToolCall, options?: MCPExecuteOptions): Promise<MCPResult> {
|
||||
const tool = this.tools.get(toolCall.name);
|
||||
|
||||
|
||||
if (!tool) {
|
||||
return {
|
||||
success: false,
|
||||
@@ -97,8 +99,8 @@ class MCPRegistry {
|
||||
}
|
||||
|
||||
try {
|
||||
console.log(`[MCPRegistry] Executing tool "${toolCall.name}" with args:`, toolCall.arguments);
|
||||
const result = await tool.execute(toolCall.arguments);
|
||||
console.log(`[MCPRegistry] Executing tool "${toolCall.name}" with args:`, toolCall.arguments, 'options:', options);
|
||||
const result = await tool.execute(toolCall.arguments, options);
|
||||
console.log(`[MCPRegistry] Tool "${toolCall.name}" execution completed:`, result.success);
|
||||
return result;
|
||||
} catch (error: any) {
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
/**
|
||||
* AI 分析 MCP 工具
|
||||
*
|
||||
* 调用文本模型分析用户意图,返回需要执行的后续工作流步骤
|
||||
* 这使得 Agent 流程也可以统一用工作流方式处理
|
||||
*
|
||||
* 使用标准的 MCPExecuteOptions 回调:
|
||||
* - onChunk: 流式输出 AI 思考内容
|
||||
* - onAddSteps: 动态添加工作流步骤
|
||||
* - onUpdateStep: 更新步骤状态
|
||||
*/
|
||||
|
||||
import type { MCPTool, MCPResult, MCPExecuteOptions, AgentExecutionContext, WorkflowStepInfo } from '../types';
|
||||
import { agentExecutor } from '../../services/agent';
|
||||
|
||||
/**
|
||||
* AI 分析参数
|
||||
*/
|
||||
export interface AIAnalyzeParams {
|
||||
/** Agent 执行上下文 */
|
||||
context: AgentExecutionContext;
|
||||
/** 使用的文本模型 */
|
||||
textModel?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* AI 分析结果
|
||||
*/
|
||||
export interface AIAnalyzeResult {
|
||||
/** 分析是否成功 */
|
||||
success: boolean;
|
||||
/** 生成的后续步骤 */
|
||||
generatedSteps?: WorkflowStepInfo[];
|
||||
/** 错误信息 */
|
||||
error?: string;
|
||||
/** AI 响应文本 */
|
||||
response?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* AI 分析 MCP 工具定义
|
||||
*/
|
||||
export const aiAnalyzeTool: MCPTool = {
|
||||
name: 'ai_analyze',
|
||||
description: `AI 分析工具。调用文本模型分析用户意图,决定需要执行的操作。
|
||||
|
||||
使用场景:
|
||||
- 用户输入包含额外要求或复杂指令
|
||||
- 需要 AI 理解用户意图并决定调用哪些工具
|
||||
- 用户选择了文本内容作为生成素材
|
||||
|
||||
不适用场景:
|
||||
- 简单的直接生成(使用 generate_image 或 generate_video)`,
|
||||
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
context: {
|
||||
type: 'object',
|
||||
description: 'Agent 执行上下文,包含用户指令、选中内容等',
|
||||
},
|
||||
textModel: {
|
||||
type: 'string',
|
||||
description: '使用的文本模型 ID',
|
||||
},
|
||||
},
|
||||
required: ['context'],
|
||||
},
|
||||
|
||||
supportedModes: ['async'],
|
||||
|
||||
execute: async (params: Record<string, unknown>, options?: MCPExecuteOptions): Promise<MCPResult> => {
|
||||
const { context, textModel } = params as unknown as AIAnalyzeParams;
|
||||
|
||||
if (!context) {
|
||||
return {
|
||||
success: false,
|
||||
error: '缺少必填参数 context',
|
||||
type: 'error',
|
||||
};
|
||||
}
|
||||
|
||||
const startTime = Date.now();
|
||||
const generatedSteps: WorkflowStepInfo[] = [];
|
||||
|
||||
try {
|
||||
console.log('[AIAnalyzeTool] Starting analysis with context:', {
|
||||
userInstruction: context.userInstruction?.substring(0, 50),
|
||||
model: context.model,
|
||||
hasSelection: Object.values(context.selection).some(arr => arr.length > 0),
|
||||
});
|
||||
|
||||
const result = await agentExecutor.execute(context, {
|
||||
model: textModel || context.model.id,
|
||||
onChunk: (chunk) => {
|
||||
console.log('[AIAnalyzeTool] Chunk:', chunk);
|
||||
// 使用标准回调
|
||||
options?.onChunk?.(chunk);
|
||||
},
|
||||
onToolCall: (toolCall) => {
|
||||
console.log('[AIAnalyzeTool] Tool call:', toolCall.name);
|
||||
|
||||
// 创建新的工作流步骤
|
||||
const newStep: WorkflowStepInfo = {
|
||||
id: `step-tool-${Date.now()}-${Math.random().toString(36).substring(2, 6)}`,
|
||||
mcp: toolCall.name,
|
||||
args: toolCall.arguments || {},
|
||||
description: getToolDescription(toolCall.name, toolCall.arguments),
|
||||
status: 'running',
|
||||
};
|
||||
|
||||
generatedSteps.push(newStep);
|
||||
|
||||
// 使用标准回调通知调用方添加步骤
|
||||
options?.onAddSteps?.([newStep]);
|
||||
},
|
||||
onToolResult: (toolResult) => {
|
||||
console.log('[AIAnalyzeTool] Tool result:', toolResult);
|
||||
|
||||
// 更新最后一个步骤的状态
|
||||
if (generatedSteps.length > 0) {
|
||||
const lastStep = generatedSteps[generatedSteps.length - 1];
|
||||
lastStep.status = toolResult.success ? 'completed' : 'failed';
|
||||
|
||||
// 使用标准回调通知调用方更新步骤状态
|
||||
options?.onUpdateStep?.(
|
||||
lastStep.id,
|
||||
lastStep.status,
|
||||
toolResult.data,
|
||||
toolResult.error
|
||||
);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
if (!result.success) {
|
||||
return {
|
||||
success: false,
|
||||
error: result.error || 'AI 分析失败',
|
||||
type: 'error',
|
||||
data: {
|
||||
duration,
|
||||
generatedSteps,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
type: 'text',
|
||||
data: {
|
||||
response: result.response,
|
||||
generatedSteps,
|
||||
duration,
|
||||
},
|
||||
};
|
||||
} catch (error: any) {
|
||||
console.error('[AIAnalyzeTool] Analysis failed:', error);
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: error.message || 'AI 分析失败',
|
||||
type: 'error',
|
||||
data: {
|
||||
duration: Date.now() - startTime,
|
||||
generatedSteps,
|
||||
},
|
||||
};
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* 根据工具名称生成描述
|
||||
*/
|
||||
function getToolDescription(toolName: string, args?: Record<string, unknown>): string {
|
||||
switch (toolName) {
|
||||
case 'generate_image':
|
||||
return `生成图片: ${((args?.prompt as string) || '').substring(0, 30)}...`;
|
||||
case 'generate_video':
|
||||
return `生成视频: ${((args?.prompt as string) || '').substring(0, 30)}...`;
|
||||
case 'canvas_insertion':
|
||||
return '插入到画布';
|
||||
default:
|
||||
return `执行 ${toolName}`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 便捷方法:执行 AI 分析
|
||||
*/
|
||||
export async function analyzeWithAI(
|
||||
context: AgentExecutionContext,
|
||||
options?: MCPExecuteOptions
|
||||
): Promise<AIAnalyzeResult> {
|
||||
const result = await aiAnalyzeTool.execute(
|
||||
{ context } as unknown as Record<string, unknown>,
|
||||
options
|
||||
);
|
||||
|
||||
const data = result.data as { generatedSteps?: WorkflowStepInfo[]; response?: string } | undefined;
|
||||
|
||||
return {
|
||||
success: result.success,
|
||||
generatedSteps: data?.generatedSteps,
|
||||
response: data?.response,
|
||||
error: result.error,
|
||||
};
|
||||
}
|
||||
@@ -1,16 +1,21 @@
|
||||
/**
|
||||
* 图片生成 MCP 工具
|
||||
*
|
||||
*
|
||||
* 封装现有的图片生成服务,提供标准化的 MCP 工具接口
|
||||
* 支持两种执行模式:
|
||||
* - async: 直接调用 API 等待返回(Agent 流程)
|
||||
* - queue: 创建任务加入队列(直接生成流程)
|
||||
*/
|
||||
|
||||
import type { MCPTool, MCPResult } from '../types';
|
||||
import type { MCPTool, MCPResult, MCPExecuteOptions, MCPTaskResult } from '../types';
|
||||
import { defaultGeminiClient } from '../../utils/gemini-api';
|
||||
import { taskQueueService } from '../../services/task-queue-service';
|
||||
import { TaskType } from '../../types/task.types';
|
||||
|
||||
/**
|
||||
* 图片生成参数
|
||||
*/
|
||||
interface ImageGenerationParams {
|
||||
export interface ImageGenerationParams {
|
||||
/** 图片描述提示词 */
|
||||
prompt: string;
|
||||
/** 图片尺寸,格式如 '1x1', '16x9', '9x16' */
|
||||
@@ -19,6 +24,154 @@ interface ImageGenerationParams {
|
||||
referenceImages?: string[];
|
||||
/** 图片质量 */
|
||||
quality?: '1k' | '2k' | '4k';
|
||||
/** AI 模型 */
|
||||
model?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* 直接调用 API 生成图片(async 模式)
|
||||
*/
|
||||
async function executeAsync(params: ImageGenerationParams): Promise<MCPResult> {
|
||||
const { prompt, size, referenceImages, quality } = params;
|
||||
|
||||
if (!prompt || typeof prompt !== 'string') {
|
||||
return {
|
||||
success: false,
|
||||
error: '缺少必填参数 prompt',
|
||||
type: 'error',
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
console.log('[ImageGenerationTool] Generating image with params:', {
|
||||
prompt: prompt.substring(0, 100) + (prompt.length > 100 ? '...' : ''),
|
||||
size,
|
||||
referenceImages: referenceImages?.length || 0,
|
||||
quality,
|
||||
});
|
||||
|
||||
// 调用 Gemini 图片生成 API
|
||||
const result = await defaultGeminiClient.generateImage(prompt, {
|
||||
size: size || '1x1',
|
||||
image: referenceImages && referenceImages.length > 0 ? referenceImages : undefined,
|
||||
response_format: 'url',
|
||||
quality: quality || '1k',
|
||||
});
|
||||
|
||||
console.log('[ImageGenerationTool] Generation response:', result);
|
||||
|
||||
// 解析响应
|
||||
if (result.data && Array.isArray(result.data) && result.data.length > 0) {
|
||||
const imageData = result.data[0];
|
||||
let imageUrl: string;
|
||||
|
||||
if (imageData.url) {
|
||||
imageUrl = imageData.url;
|
||||
} else if (imageData.b64_json) {
|
||||
imageUrl = `data:image/png;base64,${imageData.b64_json}`;
|
||||
} else {
|
||||
return {
|
||||
success: false,
|
||||
error: 'API 未返回有效的图片数据',
|
||||
type: 'error',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
url: imageUrl,
|
||||
format: 'png',
|
||||
prompt,
|
||||
size: size || '1x1',
|
||||
},
|
||||
type: 'image',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: 'API 未返回有效的图片数据',
|
||||
type: 'error',
|
||||
};
|
||||
} catch (error: any) {
|
||||
console.error('[ImageGenerationTool] Generation failed:', error);
|
||||
|
||||
// 提取更详细的错误信息
|
||||
let errorMessage = error.message || '图片生成失败';
|
||||
if (error.apiErrorBody) {
|
||||
errorMessage = `${errorMessage} - ${JSON.stringify(error.apiErrorBody)}`;
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: errorMessage,
|
||||
type: 'error',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建任务加入队列(queue 模式)
|
||||
*/
|
||||
function executeQueue(params: ImageGenerationParams, options: MCPExecuteOptions): MCPTaskResult {
|
||||
const { prompt, size, referenceImages, model } = params;
|
||||
|
||||
if (!prompt || typeof prompt !== 'string') {
|
||||
return {
|
||||
success: false,
|
||||
error: '缺少必填参数 prompt',
|
||||
type: 'error',
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
// 将参考图片转换为 uploadedImages 格式
|
||||
const uploadedImages = referenceImages?.map((url, index) => ({
|
||||
type: 'url' as const,
|
||||
url,
|
||||
name: `reference-${index + 1}`,
|
||||
}));
|
||||
|
||||
// 创建任务
|
||||
const task = taskQueueService.createTask(
|
||||
{
|
||||
prompt,
|
||||
size: size || '1x1',
|
||||
uploadedImages: uploadedImages && uploadedImages.length > 0 ? uploadedImages : undefined,
|
||||
model: model || 'imagen-3.0-generate-002',
|
||||
// 批量参数
|
||||
batchId: options.batchId,
|
||||
batchIndex: options.batchIndex,
|
||||
batchTotal: options.batchTotal,
|
||||
globalIndex: options.globalIndex,
|
||||
},
|
||||
TaskType.IMAGE
|
||||
);
|
||||
|
||||
console.log('[ImageGenerationTool] Created task:', task.id);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
taskId: task.id,
|
||||
prompt,
|
||||
size: size || '1x1',
|
||||
model: model || 'imagen-3.0-generate-002',
|
||||
},
|
||||
type: 'image',
|
||||
taskId: task.id,
|
||||
task,
|
||||
};
|
||||
} catch (error: any) {
|
||||
console.error('[ImageGenerationTool] Failed to create task:', error);
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: error.message || '创建任务失败',
|
||||
type: 'error',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -27,7 +180,7 @@ interface ImageGenerationParams {
|
||||
export const imageGenerationTool: MCPTool = {
|
||||
name: 'generate_image',
|
||||
description: `生成图片工具。根据用户的文字描述生成图片。
|
||||
|
||||
|
||||
使用场景:
|
||||
- 用户想要创建、生成、绘制图片
|
||||
- 用户描述了想要的图片内容
|
||||
@@ -63,87 +216,45 @@ export const imageGenerationTool: MCPTool = {
|
||||
enum: ['1k', '2k', '4k'],
|
||||
default: '1k',
|
||||
},
|
||||
model: {
|
||||
type: 'string',
|
||||
description: '图片生成模型',
|
||||
default: 'imagen-3.0-generate-002',
|
||||
},
|
||||
},
|
||||
required: ['prompt'],
|
||||
},
|
||||
|
||||
execute: async (params: Record<string, unknown>): Promise<MCPResult> => {
|
||||
const { prompt, size, referenceImages, quality } = params as ImageGenerationParams;
|
||||
supportedModes: ['async', 'queue'],
|
||||
|
||||
if (!prompt || typeof prompt !== 'string') {
|
||||
return {
|
||||
success: false,
|
||||
error: '缺少必填参数 prompt',
|
||||
type: 'error',
|
||||
};
|
||||
execute: async (params: Record<string, unknown>, options?: MCPExecuteOptions): Promise<MCPResult> => {
|
||||
const typedParams = params as unknown as ImageGenerationParams;
|
||||
const mode = options?.mode || 'async';
|
||||
|
||||
if (mode === 'queue') {
|
||||
return executeQueue(typedParams, options || {});
|
||||
}
|
||||
|
||||
try {
|
||||
console.log('[ImageGenerationTool] Generating image with params:', {
|
||||
prompt: prompt.substring(0, 100) + (prompt.length > 100 ? '...' : ''),
|
||||
size,
|
||||
referenceImages: referenceImages?.length || 0,
|
||||
quality,
|
||||
});
|
||||
|
||||
// 调用 Gemini 图片生成 API
|
||||
const result = await defaultGeminiClient.generateImage(prompt, {
|
||||
size: size || '1x1',
|
||||
image: referenceImages && referenceImages.length > 0 ? referenceImages : undefined,
|
||||
response_format: 'url',
|
||||
quality: quality || '1k',
|
||||
});
|
||||
|
||||
console.log('[ImageGenerationTool] Generation response:', result);
|
||||
|
||||
// 解析响应
|
||||
if (result.data && Array.isArray(result.data) && result.data.length > 0) {
|
||||
const imageData = result.data[0];
|
||||
let imageUrl: string;
|
||||
|
||||
if (imageData.url) {
|
||||
imageUrl = imageData.url;
|
||||
} else if (imageData.b64_json) {
|
||||
imageUrl = `data:image/png;base64,${imageData.b64_json}`;
|
||||
} else {
|
||||
return {
|
||||
success: false,
|
||||
error: 'API 未返回有效的图片数据',
|
||||
type: 'error',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
url: imageUrl,
|
||||
format: 'png',
|
||||
prompt,
|
||||
size: size || '1x1',
|
||||
},
|
||||
type: 'image',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: 'API 未返回有效的图片数据',
|
||||
type: 'error',
|
||||
};
|
||||
} catch (error: any) {
|
||||
console.error('[ImageGenerationTool] Generation failed:', error);
|
||||
|
||||
// 提取更详细的错误信息
|
||||
let errorMessage = error.message || '图片生成失败';
|
||||
if (error.apiErrorBody) {
|
||||
errorMessage = `${errorMessage} - ${JSON.stringify(error.apiErrorBody)}`;
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: errorMessage,
|
||||
type: 'error',
|
||||
};
|
||||
}
|
||||
return executeAsync(typedParams);
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* 便捷方法:直接生成图片(async 模式)
|
||||
*/
|
||||
export async function generateImage(params: ImageGenerationParams): Promise<MCPResult> {
|
||||
return imageGenerationTool.execute(params as unknown as Record<string, unknown>, { mode: 'async' });
|
||||
}
|
||||
|
||||
/**
|
||||
* 便捷方法:创建图片生成任务(queue 模式)
|
||||
*/
|
||||
export function createImageTask(
|
||||
params: ImageGenerationParams,
|
||||
options?: Omit<MCPExecuteOptions, 'mode'>
|
||||
): MCPTaskResult {
|
||||
return imageGenerationTool.execute(params as unknown as Record<string, unknown>, {
|
||||
...options,
|
||||
mode: 'queue',
|
||||
}) as unknown as MCPTaskResult;
|
||||
}
|
||||
@@ -2,5 +2,11 @@
|
||||
* MCP 工具统一导出
|
||||
*/
|
||||
|
||||
export { imageGenerationTool } from './image-generation';
|
||||
export { videoGenerationTool } from './video-generation';
|
||||
export { imageGenerationTool, generateImage, createImageTask } from './image-generation';
|
||||
export type { ImageGenerationParams } from './image-generation';
|
||||
|
||||
export { videoGenerationTool, generateVideo, createVideoTask } from './video-generation';
|
||||
export type { VideoGenerationParams } from './video-generation';
|
||||
|
||||
export { aiAnalyzeTool, analyzeWithAI } from './ai-analyze';
|
||||
export type { AIAnalyzeParams, AIAnalyzeResult } from './ai-analyze';
|
||||
@@ -1,17 +1,23 @@
|
||||
/**
|
||||
* 视频生成 MCP 工具
|
||||
*
|
||||
*
|
||||
* 封装现有的视频生成服务,提供标准化的 MCP 工具接口
|
||||
* 支持两种执行模式:
|
||||
* - async: 直接调用 API 等待返回(Agent 流程)
|
||||
* - queue: 创建任务加入队列(直接生成流程)
|
||||
*/
|
||||
|
||||
import type { MCPTool, MCPResult } from '../types';
|
||||
import type { MCPTool, MCPResult, MCPExecuteOptions, MCPTaskResult } from '../types';
|
||||
import { videoAPIService } from '../../services/video-api-service';
|
||||
import { taskQueueService } from '../../services/task-queue-service';
|
||||
import { TaskType } from '../../types/task.types';
|
||||
import type { VideoModel } from '../../types/video.types';
|
||||
import { VIDEO_MODEL_CONFIGS } from '../../constants/video-model-config';
|
||||
|
||||
/**
|
||||
* 视频生成参数
|
||||
*/
|
||||
interface VideoGenerationParams {
|
||||
export interface VideoGenerationParams {
|
||||
/** 视频描述提示词 */
|
||||
prompt: string;
|
||||
/** 视频模型 */
|
||||
@@ -24,6 +30,170 @@ interface VideoGenerationParams {
|
||||
referenceImages?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* 直接调用 API 生成视频(async 模式)
|
||||
*/
|
||||
async function executeAsync(params: VideoGenerationParams): Promise<MCPResult> {
|
||||
const {
|
||||
prompt,
|
||||
model = 'veo3',
|
||||
seconds = '8',
|
||||
size = '1280x720',
|
||||
referenceImages,
|
||||
} = params;
|
||||
|
||||
if (!prompt || typeof prompt !== 'string') {
|
||||
return {
|
||||
success: false,
|
||||
error: '缺少必填参数 prompt',
|
||||
type: 'error',
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
console.log('[VideoGenerationTool] Generating video with params:', {
|
||||
prompt: prompt.substring(0, 100) + (prompt.length > 100 ? '...' : ''),
|
||||
model,
|
||||
seconds,
|
||||
size,
|
||||
referenceImages: referenceImages?.length || 0,
|
||||
});
|
||||
|
||||
// 准备参考图片
|
||||
let inputReferences: Array<{ type: 'url'; url: string }> | undefined;
|
||||
if (referenceImages && referenceImages.length > 0) {
|
||||
inputReferences = referenceImages.map((url) => ({
|
||||
type: 'url' as const,
|
||||
url,
|
||||
}));
|
||||
}
|
||||
|
||||
// 调用视频生成 API(带轮询)
|
||||
const result = await videoAPIService.generateVideoWithPolling(
|
||||
{
|
||||
model: model as VideoModel,
|
||||
prompt,
|
||||
seconds,
|
||||
size,
|
||||
inputReferences,
|
||||
},
|
||||
{
|
||||
interval: 5000, // 每 5 秒轮询一次
|
||||
onProgress: (progress, status) => {
|
||||
console.log(`[VideoGenerationTool] Progress: ${progress}% (${status})`);
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
console.log('[VideoGenerationTool] Generation completed:', result);
|
||||
|
||||
// 提取视频 URL
|
||||
const videoUrl = result.video_url || result.url;
|
||||
if (!videoUrl) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'API 未返回有效的视频 URL',
|
||||
type: 'error',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
url: videoUrl,
|
||||
format: 'mp4',
|
||||
prompt,
|
||||
model,
|
||||
seconds,
|
||||
size,
|
||||
},
|
||||
type: 'video',
|
||||
};
|
||||
} catch (error: any) {
|
||||
console.error('[VideoGenerationTool] Generation failed:', error);
|
||||
|
||||
// 提取更详细的错误信息
|
||||
let errorMessage = error.message || '视频生成失败';
|
||||
if (error.apiErrorBody) {
|
||||
errorMessage = `${errorMessage} - ${JSON.stringify(error.apiErrorBody)}`;
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: errorMessage,
|
||||
type: 'error',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建任务加入队列(queue 模式)
|
||||
*/
|
||||
function executeQueue(params: VideoGenerationParams, options: MCPExecuteOptions): MCPTaskResult {
|
||||
const { prompt, model = 'veo3', seconds, size, referenceImages } = params;
|
||||
|
||||
if (!prompt || typeof prompt !== 'string') {
|
||||
return {
|
||||
success: false,
|
||||
error: '缺少必填参数 prompt',
|
||||
type: 'error',
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
// 获取模型默认配置
|
||||
const modelConfig = VIDEO_MODEL_CONFIGS[model as VideoModel] || VIDEO_MODEL_CONFIGS['veo3'];
|
||||
|
||||
// 将参考图片转换为 uploadedImages 格式
|
||||
const uploadedImages = referenceImages?.map((url, index) => ({
|
||||
type: 'url' as const,
|
||||
url,
|
||||
name: `reference-${index + 1}`,
|
||||
}));
|
||||
|
||||
// 创建任务
|
||||
const task = taskQueueService.createTask(
|
||||
{
|
||||
prompt,
|
||||
size: size || '16x9',
|
||||
duration: parseInt(seconds || modelConfig.defaultDuration, 10),
|
||||
model,
|
||||
uploadedImages: uploadedImages && uploadedImages.length > 0 ? uploadedImages : undefined,
|
||||
// 批量参数
|
||||
batchId: options.batchId,
|
||||
batchIndex: options.batchIndex,
|
||||
batchTotal: options.batchTotal,
|
||||
globalIndex: options.globalIndex,
|
||||
},
|
||||
TaskType.VIDEO
|
||||
);
|
||||
|
||||
console.log('[VideoGenerationTool] Created task:', task.id);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
taskId: task.id,
|
||||
prompt,
|
||||
size: size || '16x9',
|
||||
duration: parseInt(seconds || modelConfig.defaultDuration, 10),
|
||||
model,
|
||||
},
|
||||
type: 'video',
|
||||
taskId: task.id,
|
||||
task,
|
||||
};
|
||||
} catch (error: any) {
|
||||
console.error('[VideoGenerationTool] Failed to create task:', error);
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: error.message || '创建任务失败',
|
||||
type: 'error',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 视频生成 MCP 工具定义
|
||||
*/
|
||||
@@ -77,96 +247,36 @@ export const videoGenerationTool: MCPTool = {
|
||||
required: ['prompt'],
|
||||
},
|
||||
|
||||
execute: async (params: Record<string, unknown>): Promise<MCPResult> => {
|
||||
const {
|
||||
prompt,
|
||||
model = 'veo3',
|
||||
seconds = '8',
|
||||
size = '1280x720',
|
||||
referenceImages
|
||||
} = params as VideoGenerationParams;
|
||||
supportedModes: ['async', 'queue'],
|
||||
|
||||
if (!prompt || typeof prompt !== 'string') {
|
||||
return {
|
||||
success: false,
|
||||
error: '缺少必填参数 prompt',
|
||||
type: 'error',
|
||||
};
|
||||
execute: async (params: Record<string, unknown>, options?: MCPExecuteOptions): Promise<MCPResult> => {
|
||||
const typedParams = params as unknown as VideoGenerationParams;
|
||||
const mode = options?.mode || 'async';
|
||||
|
||||
if (mode === 'queue') {
|
||||
return executeQueue(typedParams, options || {});
|
||||
}
|
||||
|
||||
try {
|
||||
console.log('[VideoGenerationTool] Generating video with params:', {
|
||||
prompt: prompt.substring(0, 100) + (prompt.length > 100 ? '...' : ''),
|
||||
model,
|
||||
seconds,
|
||||
size,
|
||||
referenceImages: referenceImages?.length || 0,
|
||||
});
|
||||
|
||||
// 准备参考图片
|
||||
let inputReferences: Array<{ type: 'url'; url: string }> | undefined;
|
||||
if (referenceImages && referenceImages.length > 0) {
|
||||
inputReferences = referenceImages.map(url => ({
|
||||
type: 'url' as const,
|
||||
url,
|
||||
}));
|
||||
}
|
||||
|
||||
// 调用视频生成 API(带轮询)
|
||||
const result = await videoAPIService.generateVideoWithPolling(
|
||||
{
|
||||
model: model as VideoModel,
|
||||
prompt,
|
||||
seconds,
|
||||
size,
|
||||
inputReferences,
|
||||
},
|
||||
{
|
||||
interval: 5000, // 每 5 秒轮询一次
|
||||
onProgress: (progress, status) => {
|
||||
console.log(`[VideoGenerationTool] Progress: ${progress}% (${status})`);
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
console.log('[VideoGenerationTool] Generation completed:', result);
|
||||
|
||||
// 提取视频 URL
|
||||
const videoUrl = result.video_url || result.url;
|
||||
if (!videoUrl) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'API 未返回有效的视频 URL',
|
||||
type: 'error',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
url: videoUrl,
|
||||
format: 'mp4',
|
||||
prompt,
|
||||
model,
|
||||
seconds,
|
||||
size,
|
||||
},
|
||||
type: 'video',
|
||||
};
|
||||
} catch (error: any) {
|
||||
console.error('[VideoGenerationTool] Generation failed:', error);
|
||||
|
||||
// 提取更详细的错误信息
|
||||
let errorMessage = error.message || '视频生成失败';
|
||||
if (error.apiErrorBody) {
|
||||
errorMessage = `${errorMessage} - ${JSON.stringify(error.apiErrorBody)}`;
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: errorMessage,
|
||||
type: 'error',
|
||||
};
|
||||
}
|
||||
return executeAsync(typedParams);
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* 便捷方法:直接生成视频(async 模式)
|
||||
*/
|
||||
export async function generateVideo(params: VideoGenerationParams): Promise<MCPResult> {
|
||||
return videoGenerationTool.execute(params as unknown as Record<string, unknown>, { mode: 'async' });
|
||||
}
|
||||
|
||||
/**
|
||||
* 便捷方法:创建视频生成任务(queue 模式)
|
||||
*/
|
||||
export function createVideoTask(
|
||||
params: VideoGenerationParams,
|
||||
options?: Omit<MCPExecuteOptions, 'mode'>
|
||||
): MCPTaskResult {
|
||||
return videoGenerationTool.execute(params as unknown as Record<string, unknown>, {
|
||||
...options,
|
||||
mode: 'queue',
|
||||
}) as unknown as MCPTaskResult;
|
||||
}
|
||||
@@ -38,6 +38,60 @@ export interface JSONSchemaProperty {
|
||||
required?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* MCP 工具执行模式
|
||||
*/
|
||||
export type MCPExecuteMode = 'async' | 'queue';
|
||||
|
||||
/**
|
||||
* 工作流步骤信息(用于回调)
|
||||
*/
|
||||
export interface WorkflowStepInfo {
|
||||
id: string;
|
||||
mcp: string;
|
||||
args: Record<string, unknown>;
|
||||
description: string;
|
||||
status: 'pending' | 'running' | 'completed' | 'failed' | 'skipped';
|
||||
}
|
||||
|
||||
/**
|
||||
* MCP 工具执行回调(标准回调,所有工具都可使用)
|
||||
*/
|
||||
export interface MCPExecuteCallbacks {
|
||||
/** 流式输出回调(用于 AI 思考过程) */
|
||||
onChunk?: (content: string) => void;
|
||||
/** 动态添加工作流步骤回调 */
|
||||
onAddSteps?: (steps: WorkflowStepInfo[]) => void;
|
||||
/** 更新步骤状态回调 */
|
||||
onUpdateStep?: (stepId: string, status: WorkflowStepInfo['status'], result?: unknown, error?: string, duration?: number) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* MCP 工具执行选项
|
||||
*/
|
||||
export interface MCPExecuteOptions extends MCPExecuteCallbacks {
|
||||
/** 执行模式:async(等待API返回)或 queue(加入任务队列) */
|
||||
mode?: MCPExecuteMode;
|
||||
/** 批次 ID(用于批量任务去重) */
|
||||
batchId?: string;
|
||||
/** 批次索引(1-based) */
|
||||
batchIndex?: number;
|
||||
/** 批次总数 */
|
||||
batchTotal?: number;
|
||||
/** 全局索引(用于批量任务排序) */
|
||||
globalIndex?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* 队列模式返回的任务信息
|
||||
*/
|
||||
export interface MCPTaskResult extends MCPResult {
|
||||
/** 创建的任务 ID(队列模式) */
|
||||
taskId?: string;
|
||||
/** 创建的任务对象(队列模式) */
|
||||
task?: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* MCP 工具定义
|
||||
*/
|
||||
@@ -48,8 +102,10 @@ export interface MCPTool {
|
||||
description: string;
|
||||
/** 输入参数 Schema */
|
||||
inputSchema: JSONSchema;
|
||||
/** 工具执行函数 */
|
||||
execute: (params: Record<string, unknown>) => Promise<MCPResult>;
|
||||
/** 工具执行函数(默认 async 模式) */
|
||||
execute: (params: Record<string, unknown>, options?: MCPExecuteOptions) => Promise<MCPResult>;
|
||||
/** 支持的执行模式(默认只支持 async) */
|
||||
supportedModes?: MCPExecuteMode[];
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in new issue
Block a user