From d5dd9cfe34dc307e6ae63bc1f860effb04267ee7 Mon Sep 17 00:00:00 2001 From: 7418 <7418@openclaw.ai> Date: Fri, 27 Feb 2026 00:55:24 +0800 Subject: [PATCH] feat: unify reference image data structure, add Nano Banana 2 model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reference Image 统一重构: - 新增 ReferenceImage 类型 (src/types/index.ts)，统一承载 base64 和 localPath 两种来源 - 重写 image-ref-store.ts：单一 Map 替代原来的 imageGenRefStore + lastGeneratedImagePaths 双通路 - sessionStorage 持久化 last-generated 路径，解决页面刷新后数据丢失 - buildReferenceImages() 作为唯一的 merge 入口，消除各组件手写拼接逻辑 - ImageGenConfirmation props 从两个 (referenceImagePaths + referenceImagesData) 合并为单一 referenceImages?: ReferenceImage[] - 预览渲染改为单循环，根据 data/localPath 决定 img src，上传图和结果图同时显示 - API 调用时再拆分回 base64 和路径两个字段，后端无需改动 - StreamingMessage / MessageItem 各自 5 行手动 merge 逻辑替换为一行 buildReferenceImages() 调用 - ChatView 改用 setLastGeneratedImages() 和 transferPendingToMessage() 替代直接操作 Map - MessageInput 改用 setRefImages() / deleteRefImages() 替代直接操作 Map 新增 Gemini 3.1 Flash Image 模型: - 新增 Nano Banana 2 (gemini-3.1-flash-image-preview) 作为默认图片生成模型 - 模型列表: Nano Banana 2 > Nano Banana Pro > Nano Banana - 更新 ProviderManager、image-generator.ts、media/generate route 的默认值 其他已有改动一并提交: - batch image generation 功能文件 - 依赖更新 (package.json / package-lock.json) - i18n、db schema、AppShell、uploads route 等相关改动 Co-Authored-By: Claude Opus 4.6 --- package-lock.json | 92 ++++ package.json | 2 + src/app/api/media/generate/route.ts | 168 +----- src/app/api/media/jobs/[id]/cancel/route.ts | 38 ++ src/app/api/media/jobs/[id]/items/route.ts | 49 ++ src/app/api/media/jobs/[id]/pause/route.ts | 38 ++ src/app/api/media/jobs/[id]/progress/route.ts | 89 ++++ src/app/api/media/jobs/[id]/resume/route.ts | 42 ++ src/app/api/media/jobs/[id]/route.ts | 63 +++ src/app/api/media/jobs/[id]/start/route.ts | 42 ++ .../api/media/jobs/[id]/sync-context/route.ts | 99 ++++ src/app/api/media/jobs/plan/route.ts | 161 ++++++ src/app/api/media/jobs/route.ts | 73 +++ src/app/api/uploads/route.ts | 5 +- src/components/chat/ChatView.tsx | 138 ++--- src/components/chat/ImageGenConfirmation.tsx | 56 +- src/components/chat/ImageGenToggle.tsx | 6 +- src/components/chat/MessageInput.tsx | 65 ++- src/components/chat/MessageItem.tsx | 56 +- src/components/chat/StreamingMessage.tsx | 72 ++- .../chat/batch-image-gen/BatchContextSync.tsx | 79 +++ .../BatchExecutionDashboard.tsx | 117 +++++ .../batch-image-gen/BatchExecutionItem.tsx | 79 +++ .../BatchPlanInlinePreview.tsx | 127 +++++ .../chat/batch-image-gen/BatchPlanRow.tsx | 91 ++++ src/components/chat/batch-image-gen/index.ts | 5 + src/components/layout/AppShell.tsx | 4 + src/components/settings/ProviderManager.tsx | 127 +++-- src/hooks/useBatchImageGen.ts | 485 ++++++++++++++++++ src/i18n/en.ts | 50 +- src/i18n/zh.ts | 50 +- src/lib/db.ts | 351 ++++++++++++- src/lib/image-generator.ts | 179 +++++++ src/lib/image-ref-store.ts | 123 +++++ src/lib/job-executor.ts | 362 +++++++++++++ src/lib/text-generator.ts | 119 +++++ src/types/index.ts | 149 ++++++ 37 files changed, 3508 insertions(+), 343 deletions(-) create mode 100644 src/app/api/media/jobs/[id]/cancel/route.ts create mode 100644 src/app/api/media/jobs/[id]/items/route.ts create mode 100644 src/app/api/media/jobs/[id]/pause/route.ts create mode 100644 src/app/api/media/jobs/[id]/progress/route.ts create mode 100644 src/app/api/media/jobs/[id]/resume/route.ts create mode 100644 src/app/api/media/jobs/[id]/route.ts create mode 100644 src/app/api/media/jobs/[id]/start/route.ts create mode 100644 src/app/api/media/jobs/[id]/sync-context/route.ts create mode 100644 src/app/api/media/jobs/plan/route.ts create mode 100644 src/app/api/media/jobs/route.ts create mode 100644 src/components/chat/batch-image-gen/BatchContextSync.tsx create mode 100644 src/components/chat/batch-image-gen/BatchExecutionDashboard.tsx create mode 100644 src/components/chat/batch-image-gen/BatchExecutionItem.tsx create mode 100644 src/components/chat/batch-image-gen/BatchPlanInlinePreview.tsx create mode 100644 src/components/chat/batch-image-gen/BatchPlanRow.tsx create mode 100644 src/components/chat/batch-image-gen/index.ts create mode 100644 src/hooks/useBatchImageGen.ts create mode 100644 src/lib/image-generator.ts create mode 100644 src/lib/image-ref-store.ts create mode 100644 src/lib/job-executor.ts create mode 100644 src/lib/text-generator.ts diff --git a/package-lock.json b/package-lock.json index 137bc2e..f30de64 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,9 @@ "name": "codepilot", "version": "0.15.1", "dependencies": { + "@ai-sdk/anthropic": "^3.0.47", "@ai-sdk/google": "^3.0.31", + "@ai-sdk/openai": "^3.0.34", "@anthropic-ai/claude-agent-sdk": "^0.2.33", "@google/genai": "^1.43.0", "@hugeicons/core-free-icons": "^3.1.1", @@ -66,6 +68,51 @@ "wait-on": "^9.0.3" } }, + "node_modules/@ai-sdk/anthropic": { + "version": "3.0.47", + "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-3.0.47.tgz", + "integrity": "sha512-E6Z3i/xvxGDxRskMMbuX9+xDK4l5LesrP2O7YQ0CcbAkYP25qTo/kYGf/AsJrLkNIY23HeO/kheUWtG1XZllDA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.8", + "@ai-sdk/provider-utils": "4.0.15" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/anthropic/node_modules/@ai-sdk/provider": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.8.tgz", + "integrity": "sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/anthropic/node_modules/@ai-sdk/provider-utils": { + "version": "4.0.15", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.15.tgz", + "integrity": "sha512-8XiKWbemmCbvNN0CLR9u3PQiet4gtEVIrX4zzLxnCj06AwsEDJwJVBbKrEI4t6qE8XRSIvU2irka0dcpziKW6w==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.8", + "@standard-schema/spec": "^1.1.0", + "eventsource-parser": "^3.0.6" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/@ai-sdk/gateway": { "version": "3.0.36", "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.36.tgz", @@ -128,6 +175,51 @@ "zod": "^3.25.76 || ^4.1.8" } }, + "node_modules/@ai-sdk/openai": { + "version": "3.0.34", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-3.0.34.tgz", + "integrity": "sha512-rnSsjV7cqIUcYdHgTcXRvN+xn0+wv/DB/U1SCxCVOKDsD9xaNgDjQTuMeJ+dUTzpH4Gn+XS0dJQELCWVIFYQVg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.8", + "@ai-sdk/provider-utils": "4.0.15" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/openai/node_modules/@ai-sdk/provider": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.8.tgz", + "integrity": "sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/openai/node_modules/@ai-sdk/provider-utils": { + "version": "4.0.15", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.15.tgz", + "integrity": "sha512-8XiKWbemmCbvNN0CLR9u3PQiet4gtEVIrX4zzLxnCj06AwsEDJwJVBbKrEI4t6qE8XRSIvU2irka0dcpziKW6w==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.8", + "@standard-schema/spec": "^1.1.0", + "eventsource-parser": "^3.0.6" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/@ai-sdk/provider": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.7.tgz", diff --git a/package.json b/package.json index 872f1ad..fe07897 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,9 @@ "electron:pack:linux": "npm run electron:build && electron-builder --linux --config electron-builder.yml" }, "dependencies": { + "@ai-sdk/anthropic": "^3.0.47", "@ai-sdk/google": "^3.0.31", + "@ai-sdk/openai": "^3.0.34", "@anthropic-ai/claude-agent-sdk": "^0.2.33", "@google/genai": "^1.43.0", "@hugeicons/core-free-icons": "^3.1.1", diff --git a/src/app/api/media/generate/route.ts b/src/app/api/media/generate/route.ts index 42e112e..f7332c2 100644 --- a/src/app/api/media/generate/route.ts +++ b/src/app/api/media/generate/route.ts @@ -1,14 +1,5 @@ import { NextRequest } from 'next/server'; -import { generateImage, NoImageGeneratedError } from 'ai'; -import { createGoogleGenerativeAI } from '@ai-sdk/google'; -import { getDb, getSession } from '@/lib/db'; -import crypto from 'crypto'; -import fs from 'fs'; -import path from 'path'; -import os from 'os'; - -const dataDir = process.env.CLAUDE_GUI_DATA_DIR || path.join(os.homedir(), '.codepilot'); -const MEDIA_DIR = path.join(dataDir, '.codepilot-media'); +import { generateSingleImage, NoImageGeneratedError } from '@/lib/image-generator'; interface GenerateRequest { prompt: string; @@ -25,8 +16,6 @@ export const dynamic = 'force-dynamic'; export const maxDuration = 300; export async function POST(request: NextRequest) { - const startTime = Date.now(); - try { const body: GenerateRequest = await request.json(); @@ -37,156 +26,29 @@ export async function POST(request: NextRequest) { ); } - const db = getDb(); - const provider = db.prepare( - "SELECT api_key FROM api_providers WHERE provider_type = 'gemini-image' AND api_key != '' LIMIT 1" - ).get() as { api_key: string } | undefined; - - if (!provider) { - return new Response( - JSON.stringify({ error: 'No Gemini Image provider configured. Please add a provider with type "gemini-image" in Settings.' }), - { status: 400, headers: { 'Content-Type': 'application/json' } } - ); - } - - const requestedModel = body.model || 'gemini-3-pro-image-preview'; - const aspectRatio = (body.aspectRatio || '1:1') as `${number}:${number}`; - const imageSize = body.imageSize || '1K'; - - const google = createGoogleGenerativeAI({ apiKey: provider.api_key }); - - // Build prompt: plain string or { text, images } for reference images - // Support both base64 referenceImages and on-disk referenceImagePaths - let refImageData: string[] = []; - if (body.referenceImages && body.referenceImages.length > 0) { - refImageData = body.referenceImages.map(img => img.data); - } else if (body.referenceImagePaths && body.referenceImagePaths.length > 0) { - for (const filePath of body.referenceImagePaths) { - if (fs.existsSync(filePath)) { - const buf = fs.readFileSync(filePath); - refImageData.push(buf.toString('base64')); - } - } - } - const prompt = refImageData.length > 0 - ? { text: body.prompt, images: refImageData } - : body.prompt; - - const { images } = await generateImage({ - model: google.image(requestedModel), - prompt, - providerOptions: { - google: { - imageConfig: { aspectRatio, imageSize }, - }, - }, - maxRetries: 3, - abortSignal: AbortSignal.timeout(120_000), + const result = await generateSingleImage({ + prompt: body.prompt, + model: body.model, + aspectRatio: body.aspectRatio, + imageSize: body.imageSize, + referenceImages: body.referenceImages, + referenceImagePaths: body.referenceImagePaths, + sessionId: body.sessionId, }); - const elapsed = Date.now() - startTime; - console.log(`[media/generate] ${requestedModel} ${imageSize} completed in ${elapsed}ms`); - - // Ensure media directory exists - if (!fs.existsSync(MEDIA_DIR)) { - fs.mkdirSync(MEDIA_DIR, { recursive: true }); - } - - // Write images to disk - const savedImages: Array<{ mimeType: string; localPath: string }> = []; - - for (const img of images) { - const ext = img.mediaType === 'image/jpeg' ? '.jpg' - : img.mediaType === 'image/webp' ? '.webp' - : '.png'; - const filename = `${Date.now()}-${crypto.randomBytes(8).toString('hex')}${ext}`; - const filePath = path.join(MEDIA_DIR, filename); - - fs.writeFileSync(filePath, Buffer.from(img.uint8Array)); - - savedImages.push({ - mimeType: img.mediaType, - localPath: filePath, - }); - } - - // Copy images to project directory if sessionId is provided - if (body.sessionId) { - try { - const session = getSession(body.sessionId); - if (session?.working_directory) { - const projectImgDir = path.join(session.working_directory, '.codepilot-images'); - if (!fs.existsSync(projectImgDir)) { - fs.mkdirSync(projectImgDir, { recursive: true }); - } - for (const saved of savedImages) { - const destPath = path.join(projectImgDir, path.basename(saved.localPath)); - fs.copyFileSync(saved.localPath, destPath); - } - console.log(`[media/generate] Copied ${savedImages.length} image(s) to ${projectImgDir}`); - } - } catch (copyErr) { - console.warn('[media/generate] Failed to copy images to project directory:', copyErr); - } - } - - // Save reference images to disk for gallery display - const savedRefImages: Array<{ mimeType: string; localPath: string }> = []; - if (refImageData.length > 0) { - const refMimeTypes = body.referenceImages - ? body.referenceImages.map(img => img.mimeType) - : body.referenceImagePaths - ? body.referenceImagePaths.map(() => 'image/png') - : []; - for (let i = 0; i < refImageData.length; i++) { - const mime = refMimeTypes[i] || 'image/png'; - const ext = mime === 'image/jpeg' ? '.jpg' : mime === 'image/webp' ? '.webp' : '.png'; - const filename = `ref-${Date.now()}-${crypto.randomBytes(4).toString('hex')}${ext}`; - const filePath = path.join(MEDIA_DIR, filename); - fs.writeFileSync(filePath, Buffer.from(refImageData[i], 'base64')); - savedRefImages.push({ mimeType: mime, localPath: filePath }); - } - } - - // DB record - const id = crypto.randomBytes(16).toString('hex'); - const now = new Date().toISOString().replace('T', ' ').split('.')[0]; - const localPath = savedImages.length > 0 ? savedImages[0].localPath : ''; - - const metadata: Record = { - imageCount: savedImages.length, - elapsedMs: elapsed, - model: requestedModel, - }; - if (savedRefImages.length > 0) { - metadata.referenceImages = savedRefImages; - } - - db.prepare( - `INSERT INTO media_generations (id, type, status, provider, model, prompt, aspect_ratio, image_size, local_path, thumbnail_path, session_id, message_id, tags, metadata, error, created_at, completed_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - id, 'image', 'completed', 'gemini', requestedModel, body.prompt, - aspectRatio, imageSize, localPath, '', - body.sessionId || null, null, - '[]', JSON.stringify(metadata), - null, now, now - ); - return new Response( JSON.stringify({ - id, + id: result.mediaGenerationId, text: '', - images: savedImages, - model: requestedModel, - imageSize, - elapsedMs: elapsed, + images: result.images, + model: body.model || 'gemini-3.1-flash-image-preview', + imageSize: body.imageSize || '1K', + elapsedMs: result.elapsedMs, }), { status: 200, headers: { 'Content-Type': 'application/json' } } ); } catch (error) { - const elapsed = Date.now() - startTime; - console.error(`[media/generate] Failed after ${elapsed}ms:`, error); + console.error('[media/generate] Failed:', error); if (NoImageGeneratedError.isInstance(error)) { return new Response( diff --git a/src/app/api/media/jobs/[id]/cancel/route.ts b/src/app/api/media/jobs/[id]/cancel/route.ts new file mode 100644 index 0000000..5ac59f9 --- /dev/null +++ b/src/app/api/media/jobs/[id]/cancel/route.ts @@ -0,0 +1,38 @@ +import { NextRequest } from 'next/server'; +import { getMediaJob } from '@/lib/db'; +import { cancelJob } from '@/lib/job-executor'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** + * POST /api/media/jobs/:id/cancel — Cancel a running or paused job + */ +export async function POST( + _request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + try { + const { id } = await params; + const job = getMediaJob(id); + if (!job) { + return Response.json({ error: 'Job not found' }, { status: 404 }); + } + + if (job.status !== 'running' && job.status !== 'paused' && job.status !== 'planned') { + return Response.json( + { error: `Cannot cancel job with status "${job.status}".` }, + { status: 409 } + ); + } + + cancelJob(id); + return Response.json({ success: true, jobId: id, status: 'cancelled' }); + } catch (error) { + console.error('[api/media/jobs/[id]/cancel] POST failed:', error); + return Response.json( + { error: error instanceof Error ? error.message : 'Failed to cancel job' }, + { status: 500 } + ); + } +} diff --git a/src/app/api/media/jobs/[id]/items/route.ts b/src/app/api/media/jobs/[id]/items/route.ts new file mode 100644 index 0000000..5b1a745 --- /dev/null +++ b/src/app/api/media/jobs/[id]/items/route.ts @@ -0,0 +1,49 @@ +import { NextRequest } from 'next/server'; +import { getMediaJob, getMediaJobItems, updateMediaJobItem } from '@/lib/db'; +import type { UpdateMediaJobItemsRequest } from '@/types'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** + * PUT /api/media/jobs/:id/items — Batch edit items (during review phase) + */ +export async function PUT( + request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + try { + const { id } = await params; + const job = getMediaJob(id); + if (!job) { + return Response.json({ error: 'Job not found' }, { status: 404 }); + } + + if (job.status !== 'planned' && job.status !== 'draft') { + return Response.json( + { error: 'Items can only be edited when job is in planned or draft status' }, + { status: 409 } + ); + } + + const body: UpdateMediaJobItemsRequest = await request.json(); + + for (const itemUpdate of body.items) { + updateMediaJobItem(itemUpdate.id, { + prompt: itemUpdate.prompt, + aspectRatio: itemUpdate.aspectRatio, + imageSize: itemUpdate.imageSize, + tags: itemUpdate.tags, + }); + } + + const items = getMediaJobItems(id); + return Response.json({ job, items }); + } catch (error) { + console.error('[api/media/jobs/[id]/items] PUT failed:', error); + return Response.json( + { error: error instanceof Error ? error.message : 'Failed to update items' }, + { status: 500 } + ); + } +} diff --git a/src/app/api/media/jobs/[id]/pause/route.ts b/src/app/api/media/jobs/[id]/pause/route.ts new file mode 100644 index 0000000..08d68ff --- /dev/null +++ b/src/app/api/media/jobs/[id]/pause/route.ts @@ -0,0 +1,38 @@ +import { NextRequest } from 'next/server'; +import { getMediaJob } from '@/lib/db'; +import { pauseJob } from '@/lib/job-executor'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** + * POST /api/media/jobs/:id/pause — Pause a running job + */ +export async function POST( + _request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + try { + const { id } = await params; + const job = getMediaJob(id); + if (!job) { + return Response.json({ error: 'Job not found' }, { status: 404 }); + } + + if (job.status !== 'running') { + return Response.json( + { error: `Cannot pause job with status "${job.status}". Must be "running".` }, + { status: 409 } + ); + } + + pauseJob(id); + return Response.json({ success: true, jobId: id, status: 'paused' }); + } catch (error) { + console.error('[api/media/jobs/[id]/pause] POST failed:', error); + return Response.json( + { error: error instanceof Error ? error.message : 'Failed to pause job' }, + { status: 500 } + ); + } +} diff --git a/src/app/api/media/jobs/[id]/progress/route.ts b/src/app/api/media/jobs/[id]/progress/route.ts new file mode 100644 index 0000000..1a140ce --- /dev/null +++ b/src/app/api/media/jobs/[id]/progress/route.ts @@ -0,0 +1,89 @@ +import { NextRequest } from 'next/server'; +import { getMediaJob, getMediaJobItems } from '@/lib/db'; +import { addProgressListener, isJobRunning } from '@/lib/job-executor'; +import type { JobProgressEvent } from '@/types'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** + * GET /api/media/jobs/:id/progress — SSE real-time progress stream + */ +export async function GET( + _request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + const { id } = await params; + const job = getMediaJob(id); + if (!job) { + return Response.json({ error: 'Job not found' }, { status: 404 }); + } + + const encoder = new TextEncoder(); + + const stream = new ReadableStream({ + start(controller) { + const send = (event: string, data: unknown) => { + try { + controller.enqueue(encoder.encode(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`)); + } catch { + // Stream may be closed + } + }; + + // Send current snapshot immediately + const items = getMediaJobItems(id); + const snapshot = { + total: items.length, + completed: items.filter(i => i.status === 'completed').length, + failed: items.filter(i => i.status === 'failed').length, + processing: items.filter(i => i.status === 'processing').length, + }; + send('snapshot', { jobId: id, status: job.status, progress: snapshot, items }); + + // If the job is not running, send done immediately + if (!isJobRunning(id) && job.status !== 'running') { + send('done', { jobId: id, status: job.status }); + controller.close(); + return; + } + + // Register progress listener + const cleanup = addProgressListener(id, (event: JobProgressEvent) => { + send(event.type, event); + + // Close stream on terminal events + if (event.type === 'job_completed' || event.type === 'job_cancelled') { + send('done', { jobId: id }); + cleanup(); + controller.close(); + } + }); + + // Heartbeat to keep connection alive + const heartbeat = setInterval(() => { + try { + controller.enqueue(encoder.encode(': heartbeat\n\n')); + } catch { + clearInterval(heartbeat); + cleanup(); + } + }, 15000); + + // Cleanup on client disconnect (AbortSignal) + _request.signal.addEventListener('abort', () => { + clearInterval(heartbeat); + cleanup(); + try { controller.close(); } catch { /* already closed */ } + }); + }, + }); + + return new Response(stream, { + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }, + }); +} diff --git a/src/app/api/media/jobs/[id]/resume/route.ts b/src/app/api/media/jobs/[id]/resume/route.ts new file mode 100644 index 0000000..7447d65 --- /dev/null +++ b/src/app/api/media/jobs/[id]/resume/route.ts @@ -0,0 +1,42 @@ +import { NextRequest } from 'next/server'; +import { getMediaJob } from '@/lib/db'; +import { startJob } from '@/lib/job-executor'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** + * POST /api/media/jobs/:id/resume — Resume a paused job + */ +export async function POST( + _request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + try { + const { id } = await params; + const job = getMediaJob(id); + if (!job) { + return Response.json({ error: 'Job not found' }, { status: 404 }); + } + + if (job.status !== 'paused') { + return Response.json( + { error: `Cannot resume job with status "${job.status}". Must be "paused".` }, + { status: 409 } + ); + } + + // Resume by calling startJob (which handles paused → running) + startJob(id).catch(err => { + console.error(`[api/media/jobs/${id}/resume] Background execution error:`, err); + }); + + return Response.json({ success: true, jobId: id, status: 'running' }); + } catch (error) { + console.error('[api/media/jobs/[id]/resume] POST failed:', error); + return Response.json( + { error: error instanceof Error ? error.message : 'Failed to resume job' }, + { status: 500 } + ); + } +} diff --git a/src/app/api/media/jobs/[id]/route.ts b/src/app/api/media/jobs/[id]/route.ts new file mode 100644 index 0000000..9d98425 --- /dev/null +++ b/src/app/api/media/jobs/[id]/route.ts @@ -0,0 +1,63 @@ +import { NextRequest } from 'next/server'; +import { getMediaJob, getMediaJobItems, deleteMediaJob } from '@/lib/db'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** + * GET /api/media/jobs/:id — Get job detail with items + */ +export async function GET( + _request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + try { + const { id } = await params; + const job = getMediaJob(id); + if (!job) { + return Response.json({ error: 'Job not found' }, { status: 404 }); + } + + const items = getMediaJobItems(id); + return Response.json({ job, items }); + } catch (error) { + console.error('[api/media/jobs/[id]] GET failed:', error); + return Response.json( + { error: error instanceof Error ? error.message : 'Failed to get job' }, + { status: 500 } + ); + } +} + +/** + * DELETE /api/media/jobs/:id — Delete a job + */ +export async function DELETE( + _request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + try { + const { id } = await params; + const job = getMediaJob(id); + if (!job) { + return Response.json({ error: 'Job not found' }, { status: 404 }); + } + + // Only allow deleting jobs that are not currently running + if (job.status === 'running') { + return Response.json( + { error: 'Cannot delete a running job. Pause or cancel it first.' }, + { status: 409 } + ); + } + + deleteMediaJob(id); + return Response.json({ success: true }); + } catch (error) { + console.error('[api/media/jobs/[id]] DELETE failed:', error); + return Response.json( + { error: error instanceof Error ? error.message : 'Failed to delete job' }, + { status: 500 } + ); + } +} diff --git a/src/app/api/media/jobs/[id]/start/route.ts b/src/app/api/media/jobs/[id]/start/route.ts new file mode 100644 index 0000000..4ff50a4 --- /dev/null +++ b/src/app/api/media/jobs/[id]/start/route.ts @@ -0,0 +1,42 @@ +import { NextRequest } from 'next/server'; +import { getMediaJob } from '@/lib/db'; +import { startJob } from '@/lib/job-executor'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** + * POST /api/media/jobs/:id/start — Start executing a job + */ +export async function POST( + _request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + try { + const { id } = await params; + const job = getMediaJob(id); + if (!job) { + return Response.json({ error: 'Job not found' }, { status: 404 }); + } + + if (job.status !== 'planned' && job.status !== 'paused') { + return Response.json( + { error: `Cannot start job from status "${job.status}". Must be "planned" or "paused".` }, + { status: 409 } + ); + } + + // Start job asynchronously (don't await — it runs in the background) + startJob(id).catch(err => { + console.error(`[api/media/jobs/${id}/start] Background execution error:`, err); + }); + + return Response.json({ success: true, jobId: id, status: 'running' }); + } catch (error) { + console.error('[api/media/jobs/[id]/start] POST failed:', error); + return Response.json( + { error: error instanceof Error ? error.message : 'Failed to start job' }, + { status: 500 } + ); + } +} diff --git a/src/app/api/media/jobs/[id]/sync-context/route.ts b/src/app/api/media/jobs/[id]/sync-context/route.ts new file mode 100644 index 0000000..391a926 --- /dev/null +++ b/src/app/api/media/jobs/[id]/sync-context/route.ts @@ -0,0 +1,99 @@ +import { NextRequest } from 'next/server'; +import { getMediaJob, getMediaJobItems, addMessage, createContextEvent, markContextEventSynced } from '@/lib/db'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** + * POST /api/media/jobs/:id/sync-context — Sync completed results to LLM conversation + */ +export async function POST( + request: NextRequest, + { params }: { params: Promise<{ id: string }> } +) { + try { + const { id } = await params; + const job = getMediaJob(id); + if (!job) { + return Response.json({ error: 'Job not found' }, { status: 404 }); + } + + const body = await request.json().catch(() => ({})); + const syncMode = body.syncMode || 'manual'; + + if (!job.session_id) { + return Response.json( + { error: 'Job has no associated session. Cannot sync to conversation.' }, + { status: 400 } + ); + } + + const items = getMediaJobItems(id); + const completedItems = items.filter(i => i.status === 'completed'); + + if (completedItems.length === 0) { + return Response.json( + { error: 'No completed items to sync' }, + { status: 400 } + ); + } + + // Build summary message + const lines: string[] = [ + `## Batch Image Generation Complete`, + ``, + `**${completedItems.length}/${items.length}** images generated successfully.`, + ``, + ]; + + for (const item of completedItems) { + const tags = JSON.parse(item.tags || '[]') as string[]; + const tagStr = tags.length > 0 ? ` [${tags.join(', ')}]` : ''; + lines.push(`- **#${item.idx + 1}**: ${item.prompt.slice(0, 100)}${item.prompt.length > 100 ? '...' : ''}${tagStr}`); + if (item.result_media_generation_id) { + lines.push(` - Generation ID: ${item.result_media_generation_id}`); + } + } + + const failedItems = items.filter(i => i.status === 'failed'); + if (failedItems.length > 0) { + lines.push(``, `**${failedItems.length} items failed:**`); + for (const item of failedItems) { + lines.push(`- #${item.idx + 1}: ${item.error || 'Unknown error'}`); + } + } + + const summaryText = lines.join('\n'); + + // Add as user message to the session + addMessage(job.session_id, 'user', summaryText); + + // Record context event + const payload = { + completedCount: completedItems.length, + failedCount: failedItems.length, + totalCount: items.length, + itemSummaries: completedItems.map(i => ({ + idx: i.idx, + prompt: i.prompt.slice(0, 200), + mediaGenerationId: i.result_media_generation_id, + })), + }; + + const event = createContextEvent({ + sessionId: job.session_id, + jobId: id, + payload, + syncMode, + }); + markContextEventSynced(event.id); + + return Response.json({ success: true, eventId: event.id, messageLength: summaryText.length }); + } catch (error) { + console.error('[api/media/jobs/[id]/sync-context] POST failed:', error); + return Response.json( + { error: error instanceof Error ? error.message : 'Failed to sync context' }, + { status: 500 } + ); + } +} diff --git a/src/app/api/media/jobs/plan/route.ts b/src/app/api/media/jobs/plan/route.ts new file mode 100644 index 0000000..2394dbb --- /dev/null +++ b/src/app/api/media/jobs/plan/route.ts @@ -0,0 +1,161 @@ +import { NextRequest } from 'next/server'; +import { streamTextFromProvider } from '@/lib/text-generator'; +import { getDb } from '@/lib/db'; +import fs from 'fs'; +import type { PlanMediaJobRequest } from '@/types'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; +export const maxDuration = 120; + +const PLANNER_SYSTEM_PROMPT = `You are an image generation planner. Given document content and a style prompt, you must plan a series of image generation tasks. + +Your output MUST be a valid JSON object with this exact structure: +{ + "summary": "Brief summary of the plan", + "items": [ + { + "prompt": "Detailed image generation prompt", + "aspectRatio": "1:1", + "resolution": "1K", + "tags": ["tag1", "tag2"], + "sourceRefs": ["reference to source document section"] + } + ] +} + +Guidelines: +- Each prompt should be detailed, specific, and optimized for AI image generation +- Incorporate the style prompt into each item's prompt naturally +- Choose appropriate aspect ratios based on content (1:1, 16:9, 9:16, 3:2, 4:3, etc.) +- Add relevant tags for organization +- Reference the source document sections that inspired each image +- Keep resolution at "1K" unless the content specifically needs higher resolution +- Generate the requested number of items, or determine an appropriate count from the content`; + +/** + * POST /api/media/jobs/plan — SSE streaming planner + * Does NOT write to DB — returns plan for client preview/editing + */ +export async function POST(request: NextRequest) { + try { + const body: PlanMediaJobRequest = await request.json(); + + if (!body.stylePrompt) { + return Response.json( + { error: 'stylePrompt is required' }, + { status: 400 } + ); + } + + // Resolve the provider and model from the session or defaults + const db = getDb(); + let providerId = ''; + let modelId = ''; + + if (body.sessionId) { + const session = db.prepare('SELECT provider_id, model FROM chat_sessions WHERE id = ?').get(body.sessionId) as { provider_id: string; model: string } | undefined; + if (session) { + providerId = session.provider_id; + modelId = session.model; + } + } + + // Fallback to default provider + if (!providerId) { + const defaultId = db.prepare("SELECT value FROM settings WHERE key = 'default_provider_id'").get() as { value: string } | undefined; + providerId = defaultId?.value || ''; + } + if (!modelId) { + const defaultModel = db.prepare("SELECT value FROM settings WHERE key = 'default_model'").get() as { value: string } | undefined; + modelId = defaultModel?.value || 'claude-sonnet-4-20250514'; + } + + // Read document content + let docContent = body.docContent || ''; + if (body.docPaths && body.docPaths.length > 0 && !docContent) { + const parts: string[] = []; + for (const docPath of body.docPaths) { + try { + const stat = fs.statSync(docPath); + if (stat.size > 100 * 1024) { + parts.push(`[File: ${docPath} — skipped, too large (${Math.round(stat.size / 1024)}KB)]`); + continue; + } + const content = fs.readFileSync(docPath, 'utf-8'); + parts.push(`--- File: ${docPath} ---\n${content}`); + } catch { + parts.push(`[File: ${docPath} — could not be read]`); + } + } + docContent = parts.join('\n\n'); + } + + // Build the user prompt + const countHint = body.count ? `Generate exactly ${body.count} image items.` : 'Determine the appropriate number of images based on the content.'; + const userPrompt = `Style prompt: ${body.stylePrompt} + +${countHint} + +${docContent ? `Document content:\n${docContent}` : 'No document provided — generate images based on the style prompt alone.'}`; + + // Create SSE stream + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + async start(controller) { + const send = (event: string, data: unknown) => { + controller.enqueue(encoder.encode(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`)); + }; + + send('planning_start', { message: 'Starting plan generation...' }); + + try { + let fullText = ''; + + for await (const chunk of streamTextFromProvider({ + providerId, + model: modelId, + system: PLANNER_SYSTEM_PROMPT, + prompt: userPrompt, + maxTokens: 4096, + })) { + fullText += chunk; + send('text', { chunk }); + } + + // Extract JSON from the response + const jsonMatch = fullText.match(/\{[\s\S]*\}/); + if (!jsonMatch) { + send('error', { message: 'Failed to extract plan JSON from LLM response' }); + send('done', {}); + controller.close(); + return; + } + + const plan = JSON.parse(jsonMatch[0]); + send('plan_complete', { plan }); + } catch (err) { + const message = err instanceof Error ? err.message : 'Planning failed'; + send('error', { message }); + } + + send('done', {}); + controller.close(); + }, + }); + + return new Response(stream, { + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }, + }); + } catch (error) { + console.error('[api/media/jobs/plan] POST failed:', error); + return Response.json( + { error: error instanceof Error ? error.message : 'Planning failed' }, + { status: 500 } + ); + } +} diff --git a/src/app/api/media/jobs/route.ts b/src/app/api/media/jobs/route.ts new file mode 100644 index 0000000..b5a9e2a --- /dev/null +++ b/src/app/api/media/jobs/route.ts @@ -0,0 +1,73 @@ +import { NextRequest } from 'next/server'; +import { + getAllMediaJobs, + getMediaJobsBySession, + createMediaJob, + createMediaJobItems, + getMediaJob, + getMediaJobItems, +} from '@/lib/db'; +import type { CreateMediaJobRequest } from '@/types'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +/** + * GET /api/media/jobs — List all jobs, optionally filtered by sessionId + */ +export async function GET(request: NextRequest) { + try { + const { searchParams } = new URL(request.url); + const sessionId = searchParams.get('sessionId'); + const limit = parseInt(searchParams.get('limit') || '50'); + const offset = parseInt(searchParams.get('offset') || '0'); + + const jobs = sessionId + ? getMediaJobsBySession(sessionId) + : getAllMediaJobs(limit, offset); + + return Response.json({ jobs }); + } catch (error) { + console.error('[api/media/jobs] GET failed:', error); + return Response.json( + { error: error instanceof Error ? error.message : 'Failed to list jobs' }, + { status: 500 } + ); + } +} + +/** + * POST /api/media/jobs — Create a new job with items (after plan confirmation) + */ +export async function POST(request: NextRequest) { + try { + const body: CreateMediaJobRequest = await request.json(); + + if (!body.items || body.items.length === 0) { + return Response.json( + { error: 'At least one item is required' }, + { status: 400 } + ); + } + + // Create the job + const job = createMediaJob({ + sessionId: body.sessionId, + docPaths: body.docPaths, + stylePrompt: body.stylePrompt, + batchConfig: body.batchConfig, + totalItems: body.items.length, + }); + + // Create the items + const items = createMediaJobItems(job.id, body.items); + + return Response.json({ job: getMediaJob(job.id), items }); + } catch (error) { + console.error('[api/media/jobs] POST failed:', error); + return Response.json( + { error: error instanceof Error ? error.message : 'Failed to create job' }, + { status: 500 } + ); + } +} diff --git a/src/app/api/uploads/route.ts b/src/app/api/uploads/route.ts index f93b159..39ca816 100644 --- a/src/app/api/uploads/route.ts +++ b/src/app/api/uploads/route.ts @@ -33,9 +33,10 @@ export async function GET(request: NextRequest) { }); } - // Security: only allow files within .codepilot-uploads/ directories + // Security: only allow files within known .codepilot-* directories const resolved = path.resolve(filePath); - if (!resolved.includes('.codepilot-uploads')) { + const allowedDirs = ['.codepilot-uploads', '.codepilot-media', '.codepilot-images']; + if (!allowedDirs.some(dir => resolved.includes(dir))) { return new Response(JSON.stringify({ error: 'Access denied' }), { status: 403, headers: { 'Content-Type': 'application/json' }, diff --git a/src/components/chat/ChatView.tsx b/src/components/chat/ChatView.tsx index 20f75f5..85d361a 100644 --- a/src/components/chat/ChatView.tsx +++ b/src/components/chat/ChatView.tsx @@ -6,13 +6,8 @@ import { MessageList } from './MessageList'; import { MessageInput } from './MessageInput'; import { usePanel } from '@/hooks/usePanel'; import { consumeSSEStream } from '@/hooks/useSSEStream'; - -/** - * Module-level store for reference image base64 data. - * Keyed by message ID — ImageGenConfirmation reads from here to pass - * reference images to /api/media/generate without storing base64 in DB. - */ -export const imageGenRefStore = new Map>(); +import { BatchExecutionDashboard, BatchContextSync } from './batch-image-gen'; +import { setLastGeneratedImages, transferPendingToMessage } from '@/lib/image-ref-store'; interface ToolUseInfo { id: string; @@ -115,6 +110,8 @@ export function ChatView({ sessionId, initialMessages = [], initialHasMore = fal const toolResultsRef = useRef([]); // Ref for sendMessage to allow self-referencing in timeout auto-retry without circular deps const sendMessageRef = useRef<(content: string, files?: FileAttachment[]) => Promise>(undefined); + // Pending image generation notices — flushed into the next user message so the LLM knows about generated images + const pendingImageNoticesRef = useRef([]); // Re-sync streaming content when the window regains visibility (Electron/browser tab switch) useEffect(() => { @@ -275,13 +272,22 @@ export function ChatView({ sessionId, initialMessages = [], initialHasMore = fal }, 10_000); const markActive = () => { lastEventTime = Date.now(); }; + // Flush any pending image generation notices into the prompt so + // the LLM (especially in SDK resume mode) knows about previously generated images. + let effectiveContent = content; + if (pendingImageNoticesRef.current.length > 0) { + const notices = pendingImageNoticesRef.current.join('\n\n'); + pendingImageNoticesRef.current = []; + effectiveContent = `${notices}\n\n---\n\n${content}`; + } + try { const response = await fetch('/api/chat', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ session_id: sessionId, - content, + content: effectiveContent, mode, model: currentModel, provider_id: currentProviderId, @@ -412,6 +418,11 @@ export function ChatView({ sessionId, initialMessages = [], initialHasMore = fal created_at: new Date().toISOString(), token_usage: result.tokenUsage ? JSON.stringify(result.tokenUsage) : null, }; + // Transfer pending reference images to this message ID so MessageItem can + // retrieve them. StreamingMessage uses __pending__ directly, but once the + // message transitions to MessageItem, it's keyed by message.id. + transferPendingToMessage(assistantMessage.id); + setMessages((prev) => [...prev, assistantMessage]); } } catch (error) { @@ -598,7 +609,9 @@ export function ChatView({ sessionId, initialMessages = [], initialHasMore = fal } }, [sessionId, sendMessage]); - // Listen for image generation completion — persist notice to DB only (no model call) + // Listen for image generation completion — persist notice to DB and queue for next user message. + // The notice is NOT sent as a separate LLM turn (avoids permission popups). + // Instead it's flushed into the next user message via pendingImageNoticesRef. // MessageItem hides messages matching this prefix so the user doesn't see them. useEffect(() => { const handler = (e: Event) => { @@ -607,103 +620,29 @@ export function ChatView({ sessionId, initialMessages = [], initialHasMore = fal const paths = (detail.images || []) .map((img: { localPath?: string }) => img.localPath) .filter(Boolean); - const pathInfo = paths.length > 0 ? `, file path: ${paths.join(', ')}` : ''; - const notice = `[__IMAGE_GEN_NOTICE__ prompt: "${detail.prompt}", aspect ratio: ${detail.aspectRatio}, resolution: ${detail.resolution}${pathInfo}. You can use your Read tool to view the generated image if you need to analyze or discuss it.]`; - // Persist to DB only — does NOT trigger model + const pathInfo = paths.length > 0 ? `\nGenerated image file paths:\n${paths.map((p: string) => `- ${p}`).join('\n')}` : ''; + const notice = `[Image generation completed]\n- Prompt: "${detail.prompt}"\n- Aspect ratio: ${detail.aspectRatio}\n- Resolution: ${detail.resolution}${pathInfo}`; + + // Store generated image paths so subsequent edits can use them as reference + if (paths.length > 0) { + setLastGeneratedImages(paths); + } + + // Queue for next user message so the LLM gets the context + pendingImageNoticesRef.current.push(notice); + + // Also persist to DB for history reload + const dbNotice = `[__IMAGE_GEN_NOTICE__ prompt: "${detail.prompt}", aspect ratio: ${detail.aspectRatio}, resolution: ${detail.resolution}${paths.length > 0 ? `, file path: ${paths.join(', ')}` : ''}]`; fetch('/api/chat/messages', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ session_id: sessionId, role: 'user', content: notice }), + body: JSON.stringify({ session_id: sessionId, role: 'user', content: dbNotice }), }).catch(() => {}); }; window.addEventListener('image-gen-completed', handler); return () => window.removeEventListener('image-gen-completed', handler); }, [sessionId]); - // Direct image generation — bypasses /api/chat, shows ImageGenConfirmation for parameter selection - const handleImageGenerate = useCallback(async (prompt: string, files?: FileAttachment[]) => { - if (!prompt.trim() && (!files || files.length === 0)) return; - - const displayPrompt = prompt || 'Generate an image'; - - // Build user display content — only store file metadata (no base64) for DB - let userDbContent = displayPrompt; - if (files && files.length > 0) { - const metaOnly = files.map(f => ({ id: f.id, name: f.name, type: f.type, size: f.size })); - userDbContent = `${displayPrompt}`; - } - - // Build user display content for UI — include base64 for immediate rendering - let userUiContent = displayPrompt; - if (files && files.length > 0) { - const metaWithData = files.map(f => ({ id: f.id, name: f.name, type: f.type, size: f.size, data: f.data })); - userUiContent = `${displayPrompt}`; - } - - // Add user message to UI - setMessages(prev => [...prev, { - id: 'temp-' + Date.now(), - session_id: sessionId, - role: 'user', - content: userUiContent, - created_at: new Date().toISOString(), - token_usage: null, - }]); - - // Persist user message (lightweight, no base64) - fetch('/api/chat/messages', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ session_id: sessionId, role: 'user', content: userDbContent }), - }).catch(() => {}); - - // Auto-generate title from first message if needed - if (messages.length === 0) { - const title = displayPrompt.slice(0, 50) + (displayPrompt.length > 50 ? '...' : ''); - fetch(`/api/chat/sessions/${sessionId}`, { - method: 'PATCH', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ title }), - }).then(() => { - window.dispatchEvent(new CustomEvent('session-updated')); - }).catch(() => {}); - } - - // Store reference images in module-level map for ImageGenConfirmation to pick up - const msgId = 'imggen-' + Date.now(); - if (files && files.length > 0) { - const imageFiles = files.filter(f => f.type.startsWith('image/')); - if (imageFiles.length > 0) { - imageGenRefStore.set(msgId, imageFiles.map(f => ({ mimeType: f.type, data: f.data }))); - } - } - - // Build image-gen-request — renders ImageGenConfirmation with interactive UI - const requestPayload: Record = { - prompt: displayPrompt, - aspectRatio: '1:1', - resolution: '1K', - }; - const requestContent = '```image-gen-request\n' + JSON.stringify(requestPayload) + '\n```'; - - // Add assistant message with ImageGenConfirmation UI - setMessages(prev => [...prev, { - id: msgId, - session_id: sessionId, - role: 'assistant', - content: requestContent, - created_at: new Date().toISOString(), - token_usage: null, - }]); - - // Persist assistant message so it survives page reload - fetch('/api/chat/messages', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ session_id: sessionId, role: 'assistant', content: requestContent }), - }).catch(() => {}); - }, [sessionId, messages.length]); - return (
+ {/* Batch image generation panels — shown above the input area */} + + + ; + referenceImages?: ReferenceImage[]; } type Status = 'idle' | 'generating' | 'completed' | 'error'; @@ -35,8 +35,7 @@ export function ImageGenConfirmation({ initialPrompt, initialAspectRatio, initialResolution, - referenceImagePaths, - referenceImagesData, + referenceImages, }: ImageGenConfirmationProps) { const { t } = useTranslation(); const { sessionId } = usePanel(); @@ -87,6 +86,10 @@ export function ImageGenConfirmation({ setError(null); try { + // Split unified ReferenceImage[] back into base64 data vs file paths for the API + const refData = referenceImages?.filter(r => r.data).map(r => ({ mimeType: r.mimeType, data: r.data! })); + const refPaths = referenceImages?.filter(r => r.localPath).map(r => r.localPath!); + const res = await fetch('/api/media/generate', { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -95,11 +98,12 @@ export function ImageGenConfirmation({ aspectRatio, imageSize: resolution, sessionId, - ...(referenceImagesData && referenceImagesData.length > 0 - ? { referenceImages: referenceImagesData } - : referenceImagePaths && referenceImagePaths.length > 0 - ? { referenceImagePaths } - : {}), + ...(refData && refData.length > 0 + ? { referenceImages: refData } + : {}), + ...(refPaths && refPaths.length > 0 + ? { referenceImagePaths: refPaths } + : {}), }), signal: controller.signal, }); @@ -163,7 +167,7 @@ export function ImageGenConfirmation({ } finally { abortRef.current = null; } - }, [prompt, aspectRatio, resolution, initialPrompt, sessionId, referenceImagePaths, referenceImagesData]); + }, [prompt, aspectRatio, resolution, initialPrompt, sessionId, referenceImages]); const handleRegenerate = useCallback(() => { setResult(null); @@ -185,7 +189,7 @@ export function ImageGenConfirmation({ aspectRatio={aspectRatio} imageSize={resolution} onRegenerate={handleRegenerate} - referenceImages={referenceImagesData} + referenceImages={referenceImages?.filter(r => r.data).map(r => ({ mimeType: r.mimeType, data: r.data! }))} />
); @@ -200,18 +204,20 @@ export function ImageGenConfirmation({
- {/* Reference images preview */} - {referenceImagesData && referenceImagesData.length > 0 ? ( + {/* Reference images preview — unified loop over all reference images */} + {referenceImages && referenceImages.length > 0 && (
- {referenceImagesData.map((img, i) => ( + {referenceImages.map((img, i) => (
{/* eslint-disable-next-line @next/next/no-img-element */} {`Reference @@ -219,25 +225,7 @@ export function ImageGenConfirmation({ ))}
- ) : referenceImagePaths && referenceImagePaths.length > 0 ? ( -
- -
- {referenceImagePaths.map((p, i) => ( -
- {/* eslint-disable-next-line @next/next/no-img-element */} - {`Reference -
- ))} -
-
- ) : null} + )} {/* Prompt textarea */}
diff --git a/src/components/chat/ImageGenToggle.tsx b/src/components/chat/ImageGenToggle.tsx index ef0cbb5..9b29695 100644 --- a/src/components/chat/ImageGenToggle.tsx +++ b/src/components/chat/ImageGenToggle.tsx @@ -14,12 +14,16 @@ export function ImageGenToggle() { const { state, setEnabled } = useImageGen(); const { t } = useTranslation(); + const handleToggle = () => { + setEnabled(!state.enabled); + }; + return (
- {/* Image generation toggle */} + {/* Image Agent toggle */} diff --git a/src/components/chat/MessageItem.tsx b/src/components/chat/MessageItem.tsx index 805d324..808bba4 100644 --- a/src/components/chat/MessageItem.tsx +++ b/src/components/chat/MessageItem.tsx @@ -13,14 +13,17 @@ import { Copy01Icon, Tick01Icon, ArrowDown01Icon, ArrowUp01Icon } from "@hugeico import { FileAttachmentDisplay } from './FileAttachmentDisplay'; import { ImageGenConfirmation } from './ImageGenConfirmation'; import { ImageGenCard } from './ImageGenCard'; -import { imageGenRefStore } from './ChatView'; +import { BatchPlanInlinePreview } from './batch-image-gen/BatchPlanInlinePreview'; +import { buildReferenceImages } from '@/lib/image-ref-store'; import { parseDBDate } from '@/lib/utils'; +import type { PlannerOutput } from '@/types'; interface ImageGenRequest { prompt: string; aspectRatio: string; resolution: string; referenceImages?: string[]; + useLastGenerated?: boolean; } function parseImageGenRequest(text: string): { beforeText: string; request: ImageGenRequest; afterText: string } | null { @@ -49,6 +52,7 @@ function parseImageGenRequest(text: string): { beforeText: string; request: Imag aspectRatio: String(json.aspectRatio || '1:1'), resolution: String(json.resolution || '1K'), referenceImages: Array.isArray(json.referenceImages) ? json.referenceImages : undefined, + useLastGenerated: json.useLastGenerated === true, }, afterText, }; @@ -93,6 +97,33 @@ function parseImageGenResult(text: string): { beforeText: string; result: ImageG } } +function parseBatchPlan(text: string): { beforeText: string; plan: PlannerOutput; afterText: string } | null { + const regex = /```batch-plan\s*\n?([\s\S]*?)\n?\s*```/; + const match = text.match(regex); + if (!match) return null; + try { + const json = JSON.parse(match[1]); + const beforeText = text.slice(0, match.index).trim(); + const afterText = text.slice((match.index || 0) + match[0].length).trim(); + return { + beforeText, + plan: { + summary: json.summary || '', + items: Array.isArray(json.items) ? json.items.map((item: Record) => ({ + prompt: String(item.prompt || ''), + aspectRatio: String(item.aspectRatio || '1:1'), + resolution: String(item.resolution || '1K'), + tags: Array.isArray(item.tags) ? item.tags : [], + sourceRefs: Array.isArray(item.sourceRefs) ? item.sourceRefs : [], + })) : [], + }, + afterText, + }; + } catch { + return null; + } +} + interface MessageItemProps { message: Message; } @@ -388,6 +419,18 @@ export function MessageItem({ message }: MessageItemProps) { )}
) : (() => { + // Try batch-plan first (Image Agent batch mode) + const batchPlanResult = parseBatchPlan(displayText); + if (batchPlanResult) { + return ( + <> + {batchPlanResult.beforeText && {batchPlanResult.beforeText}} + + {batchPlanResult.afterText && {batchPlanResult.afterText}} + + ); + } + // Try image-gen-result first (new direct-call format) const genResult = parseImageGenResult(displayText); if (genResult) { @@ -439,8 +482,11 @@ export function MessageItem({ message }: MessageItemProps) { // Legacy: image-gen-request (model-dependent format, for old messages) const parsed = parseImageGenRequest(displayText); if (parsed) { - // Check module-level store for reference image base64 data - const refData = imageGenRefStore.get(message.id); + const refs = buildReferenceImages( + message.id, + parsed.request.useLastGenerated || false, + parsed.request.referenceImages, + ); return ( <> {parsed.beforeText && {parsed.beforeText}} @@ -448,8 +494,7 @@ export function MessageItem({ message }: MessageItemProps) { initialPrompt={parsed.request.prompt} initialAspectRatio={parsed.request.aspectRatio} initialResolution={parsed.request.resolution} - referenceImagePaths={parsed.request.referenceImages} - referenceImagesData={refData} + referenceImages={refs.length > 0 ? refs : undefined} /> {parsed.afterText && {parsed.afterText}} @@ -458,6 +503,7 @@ export function MessageItem({ message }: MessageItemProps) { const stripped = displayText .replace(/```image-gen-request[\s\S]*?```/g, '') .replace(/```image-gen-result[\s\S]*?```/g, '') + .replace(/```batch-plan[\s\S]*?```/g, '') .trim(); return stripped ? {stripped} : null; })() diff --git a/src/components/chat/StreamingMessage.tsx b/src/components/chat/StreamingMessage.tsx index f7660ad..73f7b74 100644 --- a/src/components/chat/StreamingMessage.tsx +++ b/src/components/chat/StreamingMessage.tsx @@ -19,14 +19,17 @@ import { } from '@/components/ai-elements/confirmation'; import { Shimmer } from '@/components/ai-elements/shimmer'; import { ImageGenConfirmation } from './ImageGenConfirmation'; +import { BatchPlanInlinePreview } from './batch-image-gen/BatchPlanInlinePreview'; +import { PENDING_KEY, buildReferenceImages } from '@/lib/image-ref-store'; import type { ToolUIPart } from 'ai'; -import type { PermissionRequestEvent } from '@/types'; +import type { PermissionRequestEvent, PlannerOutput } from '@/types'; interface ImageGenRequest { prompt: string; aspectRatio: string; resolution: string; referenceImages?: string[]; + useLastGenerated?: boolean; } function parseImageGenRequest(text: string): { beforeText: string; request: ImageGenRequest; afterText: string } | null { @@ -55,6 +58,34 @@ function parseImageGenRequest(text: string): { beforeText: string; request: Imag aspectRatio: String(json.aspectRatio || '1:1'), resolution: String(json.resolution || '1K'), referenceImages: Array.isArray(json.referenceImages) ? json.referenceImages : undefined, + useLastGenerated: json.useLastGenerated === true, + }, + afterText, + }; + } catch { + return null; + } +} + +function parseBatchPlan(text: string): { beforeText: string; plan: PlannerOutput; afterText: string } | null { + const regex = /```batch-plan\s*\n?([\s\S]*?)\n?\s*```/; + const match = text.match(regex); + if (!match) return null; + try { + const json = JSON.parse(match[1]); + const beforeText = text.slice(0, match.index).trim(); + const afterText = text.slice((match.index || 0) + match[0].length).trim(); + return { + beforeText, + plan: { + summary: json.summary || '', + items: Array.isArray(json.items) ? json.items.map((item: Record) => ({ + prompt: String(item.prompt || ''), + aspectRatio: String(item.aspectRatio || '1:1'), + resolution: String(item.resolution || '1K'), + tags: Array.isArray(item.tags) ? item.tags : [], + sourceRefs: Array.isArray(item.sourceRefs) ? item.sourceRefs : [], + })) : [], }, afterText, }; @@ -492,10 +523,26 @@ export function StreamingMessage({ {/* Streaming text content rendered via Streamdown */} {content && (() => { - console.log('[StreamingMessage] content:', JSON.stringify(content.slice(0, 500))); + // Try batch-plan first (Image Agent batch mode) + const batchPlanResult = parseBatchPlan(content); + if (batchPlanResult) { + return ( + <> + {batchPlanResult.beforeText && {batchPlanResult.beforeText}} + + {batchPlanResult.afterText && {batchPlanResult.afterText}} + + ); + } + + // Try image-gen-request const parsed = parseImageGenRequest(content); - console.log('[StreamingMessage] parseImageGenRequest result:', parsed ? 'OK' : 'null', 'isStreaming:', isStreaming); if (parsed) { + const refs = buildReferenceImages( + PENDING_KEY, + parsed.request.useLastGenerated || false, + parsed.request.referenceImages, + ); return ( <> {parsed.beforeText && {parsed.beforeText}} @@ -503,22 +550,29 @@ export function StreamingMessage({ initialPrompt={parsed.request.prompt} initialAspectRatio={parsed.request.aspectRatio} initialResolution={parsed.request.resolution} - referenceImagePaths={parsed.request.referenceImages} + referenceImages={refs.length > 0 ? refs : undefined} /> {parsed.afterText && {parsed.afterText}} ); } - // Strip partial or unparseable image-gen-request blocks to avoid Shiki errors + // Strip partial or unparseable code fence blocks to avoid Shiki errors if (isStreaming) { const hasImageGenBlock = /```image-gen-request/.test(content); - const stripped = content.replace(/```image-gen-request[\s\S]*$/, '').trim(); + const hasBatchPlanBlock = /```batch-plan/.test(content); + const stripped = content + .replace(/```image-gen-request[\s\S]*$/, '') + .replace(/```batch-plan[\s\S]*$/, '') + .trim(); if (stripped) return {stripped}; - // Show shimmer while the image-gen block is being streamed - if (hasImageGenBlock) return {t('streaming.thinking')}; + // Show shimmer while the structured block is being streamed + if (hasImageGenBlock || hasBatchPlanBlock) return {t('streaming.thinking')}; return null; } - const stripped = content.replace(/```image-gen-request[\s\S]*?```/g, '').trim(); + const stripped = content + .replace(/```image-gen-request[\s\S]*?```/g, '') + .replace(/```batch-plan[\s\S]*?```/g, '') + .trim(); return stripped ? {stripped} : null; })()} diff --git a/src/components/chat/batch-image-gen/BatchContextSync.tsx b/src/components/chat/batch-image-gen/BatchContextSync.tsx new file mode 100644 index 0000000..aec7e7f --- /dev/null +++ b/src/components/chat/batch-image-gen/BatchContextSync.tsx @@ -0,0 +1,79 @@ +'use client'; + +import { useState } from 'react'; +import { useBatchImageGen } from '@/hooks/useBatchImageGen'; +import { useTranslation } from '@/hooks/useTranslation'; +import type { TranslationKey } from '@/i18n'; + +export function BatchContextSync() { + const { state, syncToLlm, resetJob } = useBatchImageGen(); + const { t } = useTranslation(); + const [synced, setSynced] = useState(false); + + if (state.phase !== 'completed' && state.phase !== 'syncing') return null; + + const handleSync = async (syncMode: 'manual' | 'auto_batch' = 'manual') => { + await syncToLlm(syncMode); + setSynced(true); + }; + + const isSyncing = state.phase === 'syncing'; + const { progress } = state; + + return ( +
+
+

+ {t('batchImageGen.completed' as TranslationKey)} +

+

+ {t('batchImageGen.completedStats' as TranslationKey) + .replace('{completed}', String(progress.completed)) + .replace('{total}', String(progress.total)) + } +

+
+ +
+ {!synced ? ( + <> + +
+ + + ) : ( + <> + + {t('batchImageGen.syncComplete' as TranslationKey)} + +
+ + + )} + + {state.error && ( +

{state.error}

+ )} +
+
+ ); +} diff --git a/src/components/chat/batch-image-gen/BatchExecutionDashboard.tsx b/src/components/chat/batch-image-gen/BatchExecutionDashboard.tsx new file mode 100644 index 0000000..e032540 --- /dev/null +++ b/src/components/chat/batch-image-gen/BatchExecutionDashboard.tsx @@ -0,0 +1,117 @@ +'use client'; + +import { useBatchImageGen } from '@/hooks/useBatchImageGen'; +import { useTranslation } from '@/hooks/useTranslation'; +import type { TranslationKey } from '@/i18n'; +import { BatchExecutionItem } from './BatchExecutionItem'; + +export function BatchExecutionDashboard() { + const { state, pauseJob, resumeJob, cancelJob, retryFailed } = useBatchImageGen(); + const { t } = useTranslation(); + + if (state.phase !== 'executing' && state.phase !== 'completed') return null; + + const { progress, items, currentJob } = state; + const isPaused = currentJob?.status === 'paused'; + const isRunning = currentJob?.status === 'running'; + const isCompleted = state.phase === 'completed'; + const progressPercent = progress.total > 0 + ? Math.round(((progress.completed + progress.failed) / progress.total) * 100) + : 0; + + const hasFailedItems = items.some(i => i.status === 'failed'); + + return ( +
+ {/* Header with progress */} +
+
+

+ {t('batchImageGen.totalProgress' as TranslationKey)} +

+ + {progress.completed}/{progress.total} + {progress.failed > 0 && ( + ({progress.failed} failed) + )} + +
+ + {/* Progress bar */} +
+
0 + ? 'linear-gradient(90deg, rgb(168, 85, 247) 0%, rgb(239, 68, 68) 100%)' + : 'rgb(168, 85, 247)', + }} + /> +
+
+ + {/* Item list */} +
+ {items.map(item => ( + + ))} +
+ + {/* Controls */} +
+ {isRunning && ( + <> + + + + )} + + {isPaused && ( + <> + + + + )} + + {isCompleted && hasFailedItems && ( + + )} + + {state.error && ( +

{state.error}

+ )} +
+
+ ); +} diff --git a/src/components/chat/batch-image-gen/BatchExecutionItem.tsx b/src/components/chat/batch-image-gen/BatchExecutionItem.tsx new file mode 100644 index 0000000..8c6c06f --- /dev/null +++ b/src/components/chat/batch-image-gen/BatchExecutionItem.tsx @@ -0,0 +1,79 @@ +'use client'; + +import { useTranslation } from '@/hooks/useTranslation'; +import type { TranslationKey } from '@/i18n'; +import type { MediaJobItem } from '@/types'; + +interface BatchExecutionItemProps { + item: MediaJobItem; +} + +export function BatchExecutionItem({ item }: BatchExecutionItemProps) { + const { t } = useTranslation(); + + const statusColor = { + pending: 'text-muted-foreground', + processing: 'text-blue-500', + completed: 'text-green-500', + failed: 'text-red-500', + cancelled: 'text-muted-foreground', + }[item.status]; + + const statusLabel = { + pending: t('batchImageGen.itemPending' as TranslationKey), + processing: t('batchImageGen.itemProcessing' as TranslationKey), + completed: t('batchImageGen.itemCompleted' as TranslationKey), + failed: t('batchImageGen.itemFailed' as TranslationKey), + cancelled: t('batchImageGen.cancel' as TranslationKey), + }[item.status]; + + return ( +
+ {/* Index */} + + #{item.idx + 1} + + + {/* Status indicator */} +
+ {item.status === 'processing' ? ( +
+ ) : item.status === 'completed' ? ( +
+ + + +
+ ) : item.status === 'failed' ? ( +
+ + + +
+ ) : ( +
+ )} +
+ + {/* Content */} +
+

{item.prompt}

+
+ {item.aspect_ratio} + {item.image_size} + {statusLabel} + {item.error && ( + {item.error} + )} +
+
+ + {/* Retry count */} + {item.retry_count > 0 && ( + + retry {item.retry_count} + + )} +
+ ); +} diff --git a/src/components/chat/batch-image-gen/BatchPlanInlinePreview.tsx b/src/components/chat/batch-image-gen/BatchPlanInlinePreview.tsx new file mode 100644 index 0000000..e89e62e --- /dev/null +++ b/src/components/chat/batch-image-gen/BatchPlanInlinePreview.tsx @@ -0,0 +1,127 @@ +'use client'; + +import { useState } from 'react'; +import { useBatchImageGen } from '@/hooks/useBatchImageGen'; +import { usePanel } from '@/hooks/usePanel'; +import { useTranslation } from '@/hooks/useTranslation'; +import type { TranslationKey } from '@/i18n'; +import type { PlannerOutput, PlannerItem } from '@/types'; +import { BatchPlanRow } from './BatchPlanRow'; +import { BatchExecutionDashboard } from './BatchExecutionDashboard'; +import { BatchContextSync } from './BatchContextSync'; + +interface BatchPlanInlinePreviewProps { + plan: PlannerOutput; + messageId: string; +} + +export function BatchPlanInlinePreview({ plan: initialPlan, messageId }: BatchPlanInlinePreviewProps) { + const batchImageGen = useBatchImageGen(); + const { sessionId } = usePanel(); + const { t } = useTranslation(); + + // Local editable state — initialized from the parsed plan + const [localPlan, setLocalPlan] = useState(initialPlan); + const [executed, setExecuted] = useState(false); + + // Once execution starts, delegate to the global batch state + const isExecuting = executed && ( + batchImageGen.state.phase === 'executing' || + batchImageGen.state.phase === 'completed' || + batchImageGen.state.phase === 'syncing' + ); + + const handleUpdateItem = (index: number, updates: Partial) => { + setLocalPlan(prev => { + const newItems = [...prev.items]; + newItems[index] = { ...newItems[index], ...updates }; + return { ...prev, items: newItems }; + }); + }; + + const handleAddItem = () => { + setLocalPlan(prev => ({ + ...prev, + items: [...prev.items, { prompt: '', aspectRatio: '1:1', resolution: '1K', tags: [], sourceRefs: [] }], + })); + }; + + const handleRemoveItem = (index: number) => { + setLocalPlan(prev => ({ + ...prev, + items: prev.items.filter((_, i) => i !== index), + })); + }; + + const handleExecute = async () => { + if (localPlan.items.length === 0) return; + setExecuted(true); + // Inject the (potentially edited) plan into batch state and start execution + await batchImageGen.injectPlanAndExecute(localPlan, sessionId || undefined); + // Now actually execute the job + await batchImageGen.executeJob(sessionId || undefined); + }; + + // Show execution dashboard once execution has started + if (isExecuting) { + return ( +
+ + +
+ ); + } + + return ( +
+ {/* Header */} +
+
+

+ {t('batchImageGen.planPreviewTitle' as TranslationKey)} +

+ + {localPlan.items.length} {localPlan.items.length === 1 ? 'item' : 'items'} + +
+ {localPlan.summary && ( +

{localPlan.summary}

+ )} +
+ + {/* Items */} +
+ {localPlan.items.map((item, i) => ( + + ))} +
+ + {/* Footer Actions */} +
+ +
+ +
+
+ ); +} diff --git a/src/components/chat/batch-image-gen/BatchPlanRow.tsx b/src/components/chat/batch-image-gen/BatchPlanRow.tsx new file mode 100644 index 0000000..62930cd --- /dev/null +++ b/src/components/chat/batch-image-gen/BatchPlanRow.tsx @@ -0,0 +1,91 @@ +'use client'; + +import { useTranslation } from '@/hooks/useTranslation'; +import type { TranslationKey } from '@/i18n'; +import type { PlannerItem } from '@/types'; + +const ASPECT_RATIOS = ['1:1', '16:9', '9:16', '3:2', '2:3', '4:3', '3:4', '4:5', '5:4', '21:9']; +const RESOLUTIONS = ['1K', '2K', '4K']; + +interface BatchPlanRowProps { + item: PlannerItem; + index: number; + onUpdate: (index: number, updates: Partial) => void; + onRemove: (index: number) => void; + disabled?: boolean; +} + +export function BatchPlanRow({ item, index, onUpdate, onRemove, disabled }: BatchPlanRowProps) { + const { t } = useTranslation(); + + return ( +
+
+ + #{index + 1} + +
+ {/* Prompt */} +