diff --git a/src/api/extract.ts b/src/api/extract.ts index dfc92c8..7bc66fc 100644 --- a/src/api/extract.ts +++ b/src/api/extract.ts @@ -9,146 +9,188 @@ import { assetService } from '../service/asset' const app = new Hono() -const extractSchema = z.object({ +const analyzeSchema = z.object({ image: z.instanceof(File), }) -const route = app.post('/', zValidator('form', extractSchema), async (c) => { - const { image } = c.req.valid('form') - const file = image as File - - return streamSSE(c, async (stream) => { - try { - // 1. Analyze - await stream.writeSSE({ - event: 'status', - data: JSON.stringify({ status: 'analyzing', message: 'Analyzing character assets...' }), - }) - - let analysis - try { - analysis = await extractionService.analyzeImage(file) - } catch (err: unknown) { - throw new Error(`Analysis failed: ${err instanceof Error ? err.message : String(err)}`, { - cause: err, - }) - } - - if (!analysis.assets || analysis.assets.length === 0) { - await stream.writeSSE({ - event: 'complete', - data: JSON.stringify({ assets: [] }), - }) - return - } - - // 2. Generate Texture Sheet - await stream.writeSSE({ - event: 'status', - data: JSON.stringify({ status: 'generating', message: 'Generating texture sheet...' }), - }) - - let sheetBase64: string - try { - sheetBase64 = await extractionService.generateTextureSheet(file, analysis.assets) - } catch (err: unknown) { - throw new Error(`Generation failed: ${err instanceof Error ? err.message : String(err)}`, { - cause: err, - }) - } - - // 2.5 Emit texture sheet to client - const dimensions = extractionService.getGridDimensions(analysis.assets.length) - await stream.writeSSE({ - event: 'texture_generated', - data: JSON.stringify({ - image: sheetBase64, - grid: dimensions, - }), - }) - - // 3. Crop Assets (Splitting) - await stream.writeSSE({ - event: 'status', - data: JSON.stringify({ status: 'splitting', message: 'Splitting texture sheet...' }), - }) - - let crops - try { - crops = await extractionService.cropAssets(sheetBase64, analysis.assets) - } catch (err: unknown) { - throw new Error(`Splitting failed: ${err instanceof Error ? err.message : String(err)}`, { - cause: err, - }) - } - - // 4. Refine & Save - await stream.writeSSE({ - event: 'status', - data: JSON.stringify({ status: 'refining', message: 'Refining and saving assets...' }), - }) - - const refinedAssets = [] - - for (const crop of crops) { - // Find matching asset from analysis to get category/description - // Since crop.name comes from analysis.assets, this should always find it. - const originalMeta = analysis.assets.find((a) => a.item_name === crop.name) || { - item_name: crop.name, - description: 'Extracted asset', - category: 'Others', - sub_category: 'Others', - } - - // Refine image - const refinedBase64 = await extractionService.refineAsset(crop.base64) - - // Save to file system - const savedFile = await fileService.saveBase64Image(refinedBase64) - - // Create Asset record (AssetService now has createAssetRecord) - const asset = await assetService.createAssetRecord({ - name: originalMeta.item_name, - type: 'image/webp', - path: savedFile.filename, - }) - - // Create Equipment record linked to Asset and using string categories - const equipment = await equipmentService.createEquipment({ - name: originalMeta.item_name, - description: originalMeta.description, - imageId: asset.id, - category: originalMeta.category, - subCategory: originalMeta.sub_category, - }) - - const refinedAsset = { - ...equipment, - imageUrl: `/files/${savedFile.filename}`, // Frontend needs URL - } - - refinedAssets.push(refinedAsset) - - // Stream event for progressive display - await stream.writeSSE({ - event: 'asset_refined', - data: JSON.stringify({ asset: refinedAsset }), - }) - } - - await stream.writeSSE({ - event: 'complete', - data: JSON.stringify({ assets: refinedAssets }), - }) - } catch (e: unknown) { - console.error(e) - const error = e instanceof Error ? e : new Error(String(e)) - await stream.writeSSE({ - event: 'error', - data: JSON.stringify({ message: error.message || 'Extraction failed' }), - }) - } - }) +const refineSchema = z.object({ + assets: z.array( + z.object({ + name: z.string(), + description: z.string(), + category: z.string(), + subCategory: z.string().optional(), + base64: z.string(), + }), + ), }) +const route = app + .post('/analyze', zValidator('form', analyzeSchema), async (c) => { + const { image } = c.req.valid('form') + const file = image as File + + return streamSSE(c, async (stream) => { + try { + // 1. Analyze + await stream.writeSSE({ + event: 'status', + data: JSON.stringify({ status: 'analyzing', message: 'Analyzing character assets...' }), + }) + + let analysis + try { + analysis = await extractionService.analyzeImage(file) + } catch (err: unknown) { + throw new Error(`Analysis failed: ${err instanceof Error ? err.message : String(err)}`, { + cause: err, + }) + } + + if (!analysis.assets || analysis.assets.length === 0) { + await stream.writeSSE({ + event: 'complete', + data: JSON.stringify({ assets: [] }), + }) + return + } + + // 2. Generate Texture Sheet + await stream.writeSSE({ + event: 'status', + data: JSON.stringify({ status: 'generating', message: 'Generating texture sheet...' }), + }) + + let sheetBase64: string + try { + sheetBase64 = await extractionService.generateTextureSheet(file, analysis.assets) + } catch (err: unknown) { + throw new Error( + `Generation failed: ${err instanceof Error ? err.message : String(err)}`, + { + cause: err, + }, + ) + } + + // 2.5 Emit texture sheet to client + const dimensions = extractionService.getGridDimensions(analysis.assets.length) + await stream.writeSSE({ + event: 'texture_generated', + data: JSON.stringify({ + image: sheetBase64, + grid: dimensions, + }), + }) + + // 3. Crop Assets (Splitting) + await stream.writeSSE({ + event: 'status', + data: JSON.stringify({ status: 'splitting', message: 'Splitting texture sheet...' }), + }) + + let crops + try { + crops = await extractionService.cropAssets(sheetBase64, analysis.assets) + } catch (err: unknown) { + throw new Error(`Splitting failed: ${err instanceof Error ? err.message : String(err)}`, { + cause: err, + }) + } + + // 4. Combine crops with metadata and return + const candidates = crops.map((crop) => { + const originalMeta = analysis.assets.find((a) => a.item_name === crop.name) || { + item_name: crop.name, + description: 'Extracted asset', + category: 'Others', + sub_category: 'Others', + } + return { + name: originalMeta.item_name, + description: originalMeta.description, + category: originalMeta.category, + subCategory: originalMeta.sub_category, + base64: crop.base64, + } + }) + + await stream.writeSSE({ + event: 'complete', + data: JSON.stringify({ assets: candidates }), + }) + } catch (e: unknown) { + console.error(e) + const error = e instanceof Error ? e : new Error(String(e)) + await stream.writeSSE({ + event: 'error', + data: JSON.stringify({ message: error.message || 'Analysis failed' }), + }) + } + }) + }) + .post('/refine', zValidator('json', refineSchema), async (c) => { + const { assets } = c.req.valid('json') + + return streamSSE(c, async (stream) => { + try { + await stream.writeSSE({ + event: 'status', + data: JSON.stringify({ status: 'refining', message: 'Refining and saving assets...' }), + }) + + const refinedAssets = [] + + for (const asset of assets) { + // Refine image + const refinedBase64 = await extractionService.refineAsset(asset.base64) + + // Save to file system + const savedFile = await fileService.saveBase64Image(refinedBase64) + + // Create Asset record + const assetRecord = await assetService.createAssetRecord({ + name: asset.name, + type: 'image/webp', + path: savedFile.filename, + }) + + // Create Equipment record + const equipment = await equipmentService.createEquipment({ + name: asset.name, + description: asset.description, + imageId: assetRecord.id, + category: asset.category, + subCategory: asset.subCategory, + }) + + const refinedAsset = { + ...equipment, + imageUrl: `/files/${savedFile.filename}`, + } + + refinedAssets.push(refinedAsset) + + // Stream event for progressive display + await stream.writeSSE({ + event: 'asset_refined', + data: JSON.stringify({ asset: refinedAsset }), + }) + } + + await stream.writeSSE({ + event: 'complete', + data: JSON.stringify({ assets: refinedAssets }), + }) + } catch (e: unknown) { + console.error(e) + const error = e instanceof Error ? e : new Error(String(e)) + await stream.writeSSE({ + event: 'error', + data: JSON.stringify({ message: error.message || 'Refinement failed' }), + }) + } + }) + }) + export default route diff --git a/src/ui/components/ExtractorDialog.tsx b/src/ui/components/ExtractorDialog.tsx index 5a6576c..6bfbebc 100644 --- a/src/ui/components/ExtractorDialog.tsx +++ b/src/ui/components/ExtractorDialog.tsx @@ -1,8 +1,9 @@ -import { AlertCircle, Loader2 } from 'lucide-react' +import { AlertCircle, Check, Loader2 } from 'lucide-react' import React, { useEffect, useState } from 'react' import { client } from '../client' -import { ExtractedAsset, ExtractionStatus } from '../types' +import { CandidateAsset, ExtractedAsset, ExtractionStatus } from '../types' import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from './ui/dialog' +import { cn } from '../lib/utils' interface ExtractorDialogProps { open: boolean @@ -25,7 +26,14 @@ export const ExtractorDialog: React.FC = ({ const [status, setStatus] = useState('idle') const [statusMessage, setStatusMessage] = useState('') const [error, setError] = useState(null) + + // Intermediate candidates + const [candidates, setCandidates] = useState([]) + const [selectedIndices, setSelectedIndices] = useState([]) + + // Final results const [results, setResults] = useState([]) + const [textureSheet, setTextureSheet] = useState(null) const [gridDimensions, setGridDimensions] = useState(null) @@ -33,6 +41,8 @@ export const ExtractorDialog: React.FC = ({ setFile(f) setPreview(URL.createObjectURL(f)) setResults([]) + setCandidates([]) + setSelectedIndices([]) setError(null) setStatus('idle') setTextureSheet(null) @@ -60,18 +70,19 @@ export const ExtractorDialog: React.FC = ({ return () => window.removeEventListener('paste', handlePaste) }, [open]) - const handleExtract = async () => { + const handleAnalyze = async () => { if (!file) return + setCandidates([]) setResults([]) setError(null) setStatus('analyzing') - setStatusMessage('Starting extraction...') + setStatusMessage('Starting analysis...') setTextureSheet(null) setGridDimensions(null) try { - const response = await client.extract.$post({ + const response = await client.extract.analyze.$post({ form: { image: file, }, @@ -107,11 +118,80 @@ export const ExtractorDialog: React.FC = ({ } else if (event === 'texture_generated') { setTextureSheet(data.image) setGridDimensions(data.grid) + } else if (event === 'complete') { + setStatus('selection') + setStatusMessage('Select items to refine') + const assets = data.assets as CandidateAsset[] + setCandidates(assets) + // Select all by default + setSelectedIndices(assets.map((_, i) => i)) + } else if (event === 'error') { + throw new Error(data.message) + } + } + } + } + } + } catch (e: unknown) { + console.error(e) + const err = e instanceof Error ? e : new Error(String(e)) + setError(err.message || 'Analysis failed') + setStatus('error') + } + } + + const handleRefine = async () => { + if (selectedIndices.length === 0) { + setError('Please select at least one item to refine.') + return + } + + const selectedCandidates = candidates.filter((_, i) => selectedIndices.includes(i)) + + setStatus('refining') + setStatusMessage('Refining selected assets...') + setResults([]) + setError(null) + + try { + const response = await client.extract.refine.$post({ + json: { + assets: selectedCandidates, + }, + }) + + if (!response.ok) throw new Error('Server error') + if (!response.body) throw new Error('No response body') + + const reader = response.body.getReader() + const decoder = new TextDecoder() + let buffer = '' + + while (true) { + const { done, value } = await reader.read() + if (done) break + + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split('\n\n') + buffer = lines.pop() || '' + + for (const line of lines) { + if (line.startsWith('event: ')) { + const eventMatch = line.match(/^event: (.+)$/m) + const dataMatch = line.match(/^data: (.+)$/m) + + if (eventMatch && dataMatch) { + const event = eventMatch[1].trim() + const data = JSON.parse(dataMatch[1]) + + if (event === 'status') { + setStatus(data.status) + setStatusMessage(data.message) } else if (event === 'asset_refined') { setResults((prev) => [...prev, data.asset]) } else if (event === 'complete') { setStatus('complete') - setStatusMessage('Extraction complete!') + setStatusMessage('Refinement complete!') setResults(data.assets) onSuccess(data.assets) } else if (event === 'error') { @@ -124,16 +204,24 @@ export const ExtractorDialog: React.FC = ({ } catch (e: unknown) { console.error(e) const err = e instanceof Error ? e : new Error(String(e)) - setError(err.message || 'Extraction failed') + setError(err.message || 'Refinement failed') setStatus('error') } } + const toggleSelection = (index: number) => { + setSelectedIndices((prev) => + prev.includes(index) ? prev.filter((i) => i !== index) : [...prev, index], + ) + } + const handleReset = () => { setFile(null) if (preview) URL.revokeObjectURL(preview) setPreview(null) + setCandidates([]) setResults([]) + setSelectedIndices([]) setStatus('idle') setError(null) setTextureSheet(null) @@ -275,7 +363,7 @@ export const ExtractorDialog: React.FC = ({ )} ) : ( -
- -

{statusMessage}

-

- Processing... -

+
+ {/* Progress Status */} + {status !== 'selection' && ( +
+ +

+ {statusMessage} +

+

+ Processing... +

+
+ )} + + {/* Selection Step Actions */} + {status === 'selection' && ( +
+

+ Review & Select +

+

+ Select the items you want to keep. They will be refined and added to + your collection. +

+ +
+ )}
)} @@ -314,16 +430,107 @@ export const ExtractorDialog: React.FC = ({
- {/* Results Grid (Appears progressively) */} + {/* Selection Grid */} + {status === 'selection' && candidates.length > 0 && ( +
+
+
+

+ ✦ Candidates Found +

+

+ Select items to refine +

+
+
+ + | + +
+
+ +
+ {candidates.map((item, i) => { + const isSelected = selectedIndices.includes(i) + return ( +
toggleSelection(i)} + className={cn( + 'group relative rounded-lg border-2 cursor-pointer transition-all duration-300', + isSelected + ? 'bg-stone-50 border-amber-500 shadow-md scale-[1.02]' + : 'bg-white border-stone-100 opacity-60 hover:opacity-100 hover:border-stone-300', + )} + > + {/* Selection Checkbox Overlay */} +
+ +
+ +
+ {/* Checkerboard bg */} +
+ +
+
+
+ {item.name} +
+
+ {item.category} +
+
+
+ ) + })} +
+
+ )} + + {/* Final Results Grid */} {(results.length > 0 || status === 'complete') && (

- ✦ Loot Acquired + ✦ Loot Acquired

- {results.length} items found & stashed + {results.length} items refined & stashed

@@ -332,7 +539,7 @@ export const ExtractorDialog: React.FC = ({ {results.map((item, i) => (
{/* Checkerboard bg */} @@ -378,7 +585,7 @@ export const ExtractorDialog: React.FC = ({
{item.category}
-
+
))} diff --git a/src/ui/components/VibeCard.tsx b/src/ui/components/VibeCard.tsx index 2cfb74c..0af7a2c 100644 --- a/src/ui/components/VibeCard.tsx +++ b/src/ui/components/VibeCard.tsx @@ -3,10 +3,10 @@ import React from 'react' import { cn } from '../lib/utils' import { Button } from './ui/button' import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuTrigger, + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, } from './ui/dropdown-menu' export interface VibeCardAction { diff --git a/src/ui/types.ts b/src/ui/types.ts index 7a4e3cf..35f4eb1 100644 --- a/src/ui/types.ts +++ b/src/ui/types.ts @@ -7,11 +7,20 @@ export interface ExtractedAsset { imageUrl: string } +export interface CandidateAsset { + name: string + description: string + category: string + subCategory?: string + base64: string +} + export type ExtractionStatus = | 'idle' | 'analyzing' | 'generating' | 'splitting' + | 'selection' | 'refining' | 'complete' | 'error'