feat: add confirmation step to equipment extraction

Split the equipment extraction process into two phases: 1. Analyze & Split: Generates texture sheet and crops assets. 2. Refine & Save: Allows user to select which assets to keep before refining and saving. - Refactored `src/api/extract.ts` to expose `/analyze` and `/refine` endpoints. - Updated `src/ui/components/ExtractorDialog.tsx` to handle the multi-step flow and selection UI. - Added `CandidateAsset` type and updated `ExtractionStatus` in `src/ui/types.ts`.
2026-02-16 02:39:37 +00:00
1 parent d8cc081023
commit b2f6869563
@@ -9,146 +9,188 @@ import { assetService } from '../service/asset'

 const app = new Hono()

-const extractSchema = z.object({
+const analyzeSchema = z.object({
  image: z.instanceof(File),
 })

-const route = app.post('/', zValidator('form', extractSchema), async (c) => {
-  const { image } = c.req.valid('form')
-  const file = image as File
-
-  return streamSSE(c, async (stream) => {
-    try {
-      // 1. Analyze
-      await stream.writeSSE({
-        event: 'status',
-        data: JSON.stringify({ status: 'analyzing', message: 'Analyzing character assets...' }),
-      })
-
-      let analysis
-      try {
-        analysis = await extractionService.analyzeImage(file)
-      } catch (err: unknown) {
-        throw new Error(`Analysis failed: ${err instanceof Error ? err.message : String(err)}`, {
-          cause: err,
-        })
-      }
-
-      if (!analysis.assets || analysis.assets.length === 0) {
-        await stream.writeSSE({
-          event: 'complete',
-          data: JSON.stringify({ assets: [] }),
-        })
-        return
-      }
-
-      // 2. Generate Texture Sheet
-      await stream.writeSSE({
-        event: 'status',
-        data: JSON.stringify({ status: 'generating', message: 'Generating texture sheet...' }),
-      })
-
-      let sheetBase64: string
-      try {
-        sheetBase64 = await extractionService.generateTextureSheet(file, analysis.assets)
-      } catch (err: unknown) {
-        throw new Error(`Generation failed: ${err instanceof Error ? err.message : String(err)}`, {
-          cause: err,
-        })
-      }
-
-      // 2.5 Emit texture sheet to client
-      const dimensions = extractionService.getGridDimensions(analysis.assets.length)
-      await stream.writeSSE({
-        event: 'texture_generated',
-        data: JSON.stringify({
-          image: sheetBase64,
-          grid: dimensions,
-        }),
-      })
-
-      // 3. Crop Assets (Splitting)
-      await stream.writeSSE({
-        event: 'status',
-        data: JSON.stringify({ status: 'splitting', message: 'Splitting texture sheet...' }),
-      })
-
-      let crops
-      try {
-        crops = await extractionService.cropAssets(sheetBase64, analysis.assets)
-      } catch (err: unknown) {
-        throw new Error(`Splitting failed: ${err instanceof Error ? err.message : String(err)}`, {
-          cause: err,
-        })
-      }
-
-      // 4. Refine & Save
-      await stream.writeSSE({
-        event: 'status',
-        data: JSON.stringify({ status: 'refining', message: 'Refining and saving assets...' }),
-      })
-
-      const refinedAssets = []
-
-      for (const crop of crops) {
-        // Find matching asset from analysis to get category/description
-        // Since crop.name comes from analysis.assets, this should always find it.
-        const originalMeta = analysis.assets.find((a) => a.item_name === crop.name) || {
-          item_name: crop.name,
-          description: 'Extracted asset',
-          category: 'Others',
-          sub_category: 'Others',
-        }
-
-        // Refine image
-        const refinedBase64 = await extractionService.refineAsset(crop.base64)
-
-        // Save to file system
-        const savedFile = await fileService.saveBase64Image(refinedBase64)
-
-        // Create Asset record (AssetService now has createAssetRecord)
-        const asset = await assetService.createAssetRecord({
-          name: originalMeta.item_name,
-          type: 'image/webp',
-          path: savedFile.filename,
-        })
-
-        // Create Equipment record linked to Asset and using string categories
-        const equipment = await equipmentService.createEquipment({
-          name: originalMeta.item_name,
-          description: originalMeta.description,
-          imageId: asset.id,
-          category: originalMeta.category,
-          subCategory: originalMeta.sub_category,
-        })
-
-        const refinedAsset = {
-          ...equipment,
-          imageUrl: `/files/${savedFile.filename}`, // Frontend needs URL
-        }
-
-        refinedAssets.push(refinedAsset)
-
-        // Stream event for progressive display
-        await stream.writeSSE({
-          event: 'asset_refined',
-          data: JSON.stringify({ asset: refinedAsset }),
-        })
-      }
-
-      await stream.writeSSE({
-        event: 'complete',
-        data: JSON.stringify({ assets: refinedAssets }),
-      })
-    } catch (e: unknown) {
-      console.error(e)
-      const error = e instanceof Error ? e : new Error(String(e))
-      await stream.writeSSE({
-        event: 'error',
-        data: JSON.stringify({ message: error.message || 'Extraction failed' }),
-      })
-    }
-  })
+const refineSchema = z.object({
+  assets: z.array(
+    z.object({
+      name: z.string(),
+      description: z.string(),
+      category: z.string(),
+      subCategory: z.string().optional(),
+      base64: z.string(),
+    }),
+  ),
 })

+const route = app
+  .post('/analyze', zValidator('form', analyzeSchema), async (c) => {
+    const { image } = c.req.valid('form')
+    const file = image as File
+
+    return streamSSE(c, async (stream) => {
+      try {
+        // 1. Analyze
+        await stream.writeSSE({
+          event: 'status',
+          data: JSON.stringify({ status: 'analyzing', message: 'Analyzing character assets...' }),
+        })
+
+        let analysis
+        try {
+          analysis = await extractionService.analyzeImage(file)
+        } catch (err: unknown) {
+          throw new Error(`Analysis failed: ${err instanceof Error ? err.message : String(err)}`, {
+            cause: err,
+          })
+        }
+
+        if (!analysis.assets || analysis.assets.length === 0) {
+          await stream.writeSSE({
+            event: 'complete',
+            data: JSON.stringify({ assets: [] }),
+          })
+          return
+        }
+
+        // 2. Generate Texture Sheet
+        await stream.writeSSE({
+          event: 'status',
+          data: JSON.stringify({ status: 'generating', message: 'Generating texture sheet...' }),
+        })
+
+        let sheetBase64: string
+        try {
+          sheetBase64 = await extractionService.generateTextureSheet(file, analysis.assets)
+        } catch (err: unknown) {
+          throw new Error(
+            `Generation failed: ${err instanceof Error ? err.message : String(err)}`,
+            {
+              cause: err,
+            },
+          )
+        }
+
+        // 2.5 Emit texture sheet to client
+        const dimensions = extractionService.getGridDimensions(analysis.assets.length)
+        await stream.writeSSE({
+          event: 'texture_generated',
+          data: JSON.stringify({
+            image: sheetBase64,
+            grid: dimensions,
+          }),
+        })
+
+        // 3. Crop Assets (Splitting)
+        await stream.writeSSE({
+          event: 'status',
+          data: JSON.stringify({ status: 'splitting', message: 'Splitting texture sheet...' }),
+        })
+
+        let crops
+        try {
+          crops = await extractionService.cropAssets(sheetBase64, analysis.assets)
+        } catch (err: unknown) {
+          throw new Error(`Splitting failed: ${err instanceof Error ? err.message : String(err)}`, {
+            cause: err,
+          })
+        }
+
+        // 4. Combine crops with metadata and return
+        const candidates = crops.map((crop) => {
+          const originalMeta = analysis.assets.find((a) => a.item_name === crop.name) || {
+            item_name: crop.name,
+            description: 'Extracted asset',
+            category: 'Others',
+            sub_category: 'Others',
+          }
+          return {
+            name: originalMeta.item_name,
+            description: originalMeta.description,
+            category: originalMeta.category,
+            subCategory: originalMeta.sub_category,
+            base64: crop.base64,
+          }
+        })
+
+        await stream.writeSSE({
+          event: 'complete',
+          data: JSON.stringify({ assets: candidates }),
+        })
+      } catch (e: unknown) {
+        console.error(e)
+        const error = e instanceof Error ? e : new Error(String(e))
+        await stream.writeSSE({
+          event: 'error',
+          data: JSON.stringify({ message: error.message || 'Analysis failed' }),
+        })
+      }
+    })
+  })
+  .post('/refine', zValidator('json', refineSchema), async (c) => {
+    const { assets } = c.req.valid('json')
+
+    return streamSSE(c, async (stream) => {
+      try {
+        await stream.writeSSE({
+          event: 'status',
+          data: JSON.stringify({ status: 'refining', message: 'Refining and saving assets...' }),
+        })
+
+        const refinedAssets = []
+
+        for (const asset of assets) {
+          // Refine image
+          const refinedBase64 = await extractionService.refineAsset(asset.base64)
+
+          // Save to file system
+          const savedFile = await fileService.saveBase64Image(refinedBase64)
+
+          // Create Asset record
+          const assetRecord = await assetService.createAssetRecord({
+            name: asset.name,
+            type: 'image/webp',
+            path: savedFile.filename,
+          })
+
+          // Create Equipment record
+          const equipment = await equipmentService.createEquipment({
+            name: asset.name,
+            description: asset.description,
+            imageId: assetRecord.id,
+            category: asset.category,
+            subCategory: asset.subCategory,
+          })
+
+          const refinedAsset = {
+            ...equipment,
+            imageUrl: `/files/${savedFile.filename}`,
+          }
+
+          refinedAssets.push(refinedAsset)
+
+          // Stream event for progressive display
+          await stream.writeSSE({
+            event: 'asset_refined',
+            data: JSON.stringify({ asset: refinedAsset }),
+          })
+        }
+
+        await stream.writeSSE({
+          event: 'complete',
+          data: JSON.stringify({ assets: refinedAssets }),
+        })
+      } catch (e: unknown) {
+        console.error(e)
+        const error = e instanceof Error ? e : new Error(String(e))
+        await stream.writeSSE({
+          event: 'error',
+          data: JSON.stringify({ message: error.message || 'Refinement failed' }),
+        })
+      }
+    })
+  })
+
 export default route
@@ -1,8 +1,9 @@
-import { AlertCircle, Loader2 } from 'lucide-react'
+import { AlertCircle, Check, Loader2 } from 'lucide-react'
 import React, { useEffect, useState } from 'react'
 import { client } from '../client'
-import { ExtractedAsset, ExtractionStatus } from '../types'
+import { CandidateAsset, ExtractedAsset, ExtractionStatus } from '../types'
 import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from './ui/dialog'
+import { cn } from '../lib/utils'

 interface ExtractorDialogProps {
  open: boolean
@@ -25,7 +26,14 @@ export const ExtractorDialog: React.FC<ExtractorDialogProps> = ({
  const [status, setStatus] = useState<ExtractionStatus>('idle')
  const [statusMessage, setStatusMessage] = useState<string>('')
  const [error, setError] = useState<string | null>(null)
+
+  // Intermediate candidates
+  const [candidates, setCandidates] = useState<CandidateAsset[]>([])
+  const [selectedIndices, setSelectedIndices] = useState<number[]>([])
+
+  // Final results
  const [results, setResults] = useState<ExtractedAsset[]>([])
+
  const [textureSheet, setTextureSheet] = useState<string | null>(null)
  const [gridDimensions, setGridDimensions] = useState<GridDimensions | null>(null)

@@ -33,6 +41,8 @@ export const ExtractorDialog: React.FC<ExtractorDialogProps> = ({
    setFile(f)
    setPreview(URL.createObjectURL(f))
    setResults([])
+    setCandidates([])
+    setSelectedIndices([])
    setError(null)
    setStatus('idle')
    setTextureSheet(null)
@@ -60,18 +70,19 @@ export const ExtractorDialog: React.FC<ExtractorDialogProps> = ({
    return () => window.removeEventListener('paste', handlePaste)
  }, [open])

-  const handleExtract = async () => {
+  const handleAnalyze = async () => {
    if (!file) return

+    setCandidates([])
    setResults([])
    setError(null)
    setStatus('analyzing')
-    setStatusMessage('Starting extraction...')
+    setStatusMessage('Starting analysis...')
    setTextureSheet(null)
    setGridDimensions(null)

    try {
-      const response = await client.extract.$post({
+      const response = await client.extract.analyze.$post({
        form: {
          image: file,
        },
@@ -107,11 +118,80 @@ export const ExtractorDialog: React.FC<ExtractorDialogProps> = ({
              } else if (event === 'texture_generated') {
                setTextureSheet(data.image)
                setGridDimensions(data.grid)
+              } else if (event === 'complete') {
+                setStatus('selection')
+                setStatusMessage('Select items to refine')
+                const assets = data.assets as CandidateAsset[]
+                setCandidates(assets)
+                // Select all by default
+                setSelectedIndices(assets.map((_, i) => i))
+              } else if (event === 'error') {
+                throw new Error(data.message)
+              }
+            }
+          }
+        }
+      }
+    } catch (e: unknown) {
+      console.error(e)
+      const err = e instanceof Error ? e : new Error(String(e))
+      setError(err.message || 'Analysis failed')
+      setStatus('error')
+    }
+  }
+
+  const handleRefine = async () => {
+    if (selectedIndices.length === 0) {
+      setError('Please select at least one item to refine.')
+      return
+    }
+
+    const selectedCandidates = candidates.filter((_, i) => selectedIndices.includes(i))
+
+    setStatus('refining')
+    setStatusMessage('Refining selected assets...')
+    setResults([])
+    setError(null)
+
+    try {
+      const response = await client.extract.refine.$post({
+        json: {
+          assets: selectedCandidates,
+        },
+      })
+
+      if (!response.ok) throw new Error('Server error')
+      if (!response.body) throw new Error('No response body')
+
+      const reader = response.body.getReader()
+      const decoder = new TextDecoder()
+      let buffer = ''
+
+      while (true) {
+        const { done, value } = await reader.read()
+        if (done) break
+
+        buffer += decoder.decode(value, { stream: true })
+        const lines = buffer.split('\n\n')
+        buffer = lines.pop() || ''
+
+        for (const line of lines) {
+          if (line.startsWith('event: ')) {
+            const eventMatch = line.match(/^event: (.+)$/m)
+            const dataMatch = line.match(/^data: (.+)$/m)
+
+            if (eventMatch && dataMatch) {
+              const event = eventMatch[1].trim()
+              const data = JSON.parse(dataMatch[1])
+
+              if (event === 'status') {
+                setStatus(data.status)
+                setStatusMessage(data.message)
              } else if (event === 'asset_refined') {
                setResults((prev) => [...prev, data.asset])
              } else if (event === 'complete') {
                setStatus('complete')
-                setStatusMessage('Extraction complete!')
+                setStatusMessage('Refinement complete!')
                setResults(data.assets)
                onSuccess(data.assets)
              } else if (event === 'error') {
@@ -124,16 +204,24 @@ export const ExtractorDialog: React.FC<ExtractorDialogProps> = ({
    } catch (e: unknown) {
      console.error(e)
      const err = e instanceof Error ? e : new Error(String(e))
-      setError(err.message || 'Extraction failed')
+      setError(err.message || 'Refinement failed')
      setStatus('error')
    }
  }

+  const toggleSelection = (index: number) => {
+    setSelectedIndices((prev) =>
+      prev.includes(index) ? prev.filter((i) => i !== index) : [...prev, index],
+    )
+  }
+
  const handleReset = () => {
    setFile(null)
    if (preview) URL.revokeObjectURL(preview)
    setPreview(null)
+    setCandidates([])
    setResults([])
+    setSelectedIndices([])
    setStatus('idle')
    setError(null)
    setTextureSheet(null)
@@ -275,7 +363,7 @@ export const ExtractorDialog: React.FC<ExtractorDialogProps> = ({
                          </div>
                        )}
                        <button
-                          onClick={handleExtract}
+                          onClick={handleAnalyze}
                          className="w-full py-4 text-white font-black uppercase tracking-widest text-lg rounded-xl shadow-lg hover:shadow-xl hover:-translate-y-1 transition-all bg-gradient-to-r from-amber-500 to-orange-600 flex items-center justify-center gap-3"
                        >
                          <svg
@@ -296,12 +384,40 @@ export const ExtractorDialog: React.FC<ExtractorDialogProps> = ({
                        </button>
                      </div>
                    ) : (
-                      <div className="w-full py-8 px-6 bg-stone-50 rounded-xl border border-stone-100 text-center">
-                        <Loader2 className="w-10 h-10 text-amber-500 animate-spin mx-auto mb-4" />
-                        <h4 className="text-lg font-bold text-stone-800 mb-1">{statusMessage}</h4>
-                        <p className="text-xs text-stone-400 font-mono uppercase tracking-widest">
-                          Processing...
-                        </p>
+                      <div className="w-full flex flex-col items-center">
+                        {/* Progress Status */}
+                        {status !== 'selection' && (
+                          <div className="w-full py-8 px-6 bg-stone-50 rounded-xl border border-stone-100 text-center mb-6">
+                            <Loader2 className="w-10 h-10 text-amber-500 animate-spin mx-auto mb-4" />
+                            <h4 className="text-lg font-bold text-stone-800 mb-1">
+                              {statusMessage}
+                            </h4>
+                            <p className="text-xs text-stone-400 font-mono uppercase tracking-widest">
+                              Processing...
+                            </p>
+                          </div>
+                        )}
+
+                        {/* Selection Step Actions */}
+                        {status === 'selection' && (
+                          <div className="w-full text-center md:text-left">
+                            <h4 className="text-lg font-bold text-stone-800 mb-2">
+                              Review & Select
+                            </h4>
+                            <p className="text-stone-500 text-sm mb-6">
+                              Select the items you want to keep. They will be refined and added to
+                              your collection.
+                            </p>
+                            <button
+                              onClick={handleRefine}
+                              disabled={selectedIndices.length === 0}
+                              className="w-full py-4 text-white font-black uppercase tracking-widest text-lg rounded-xl shadow-lg hover:shadow-xl hover:-translate-y-1 transition-all bg-gradient-to-r from-green-500 to-emerald-600 flex items-center justify-center gap-3 disabled:opacity-50 disabled:cursor-not-allowed"
+                            >
+                              <Check className="w-6 h-6" />
+                              Refine ({selectedIndices.length}) Items
+                            </button>
+                          </div>
+                        )}
                      </div>
                    )}

@@ -314,16 +430,107 @@ export const ExtractorDialog: React.FC<ExtractorDialogProps> = ({
                </div>
              </div>

-              {/* Results Grid (Appears progressively) */}
+              {/* Selection Grid */}
+              {status === 'selection' && candidates.length > 0 && (
+                <div className="bg-white p-6 rounded-2xl border border-stone-200 shadow-xl animate-fade-in-up delay-100">
+                  <div className="flex justify-between items-center mb-6">
+                    <div>
+                      <h2 className="text-xl font-black text-stone-800 flex items-center gap-2 uppercase tracking-tighter">
+                        <span className="text-amber-500">✦</span> Candidates Found
+                      </h2>
+                      <p className="text-stone-500 text-xs font-bold tracking-[0.2em] uppercase mt-1">
+                        Select items to refine
+                      </p>
+                    </div>
+                    <div className="flex gap-2">
+                      <button
+                        onClick={() => setSelectedIndices(candidates.map((_, i) => i))}
+                        className="text-xs font-bold text-cyan-600 hover:text-cyan-800 uppercase tracking-wider"
+                      >
+                        Select All
+                      </button>
+                      <span className="text-stone-300">|</span>
+                      <button
+                        onClick={() => setSelectedIndices([])}
+                        className="text-xs font-bold text-stone-400 hover:text-stone-600 uppercase tracking-wider"
+                      >
+                        Select None
+                      </button>
+                    </div>
+                  </div>
+
+                  <div className="grid grid-cols-2 sm:grid-cols-3 md:grid-cols-4 gap-4 max-h-[400px] overflow-y-auto custom-scrollbar pr-2">
+                    {candidates.map((item, i) => {
+                      const isSelected = selectedIndices.includes(i)
+                      return (
+                        <div
+                          key={i}
+                          onClick={() => toggleSelection(i)}
+                          className={cn(
+                            'group relative rounded-lg border-2 cursor-pointer transition-all duration-300',
+                            isSelected
+                              ? 'bg-stone-50 border-amber-500 shadow-md scale-[1.02]'
+                              : 'bg-white border-stone-100 opacity-60 hover:opacity-100 hover:border-stone-300',
+                          )}
+                        >
+                          {/* Selection Checkbox Overlay */}
+                          <div
+                            className={cn(
+                              'absolute top-2 right-2 z-10 w-6 h-6 rounded-full border-2 flex items-center justify-center transition-colors',
+                              isSelected
+                                ? 'bg-amber-500 border-amber-500 text-white'
+                                : 'bg-white/80 border-stone-300 text-transparent',
+                            )}
+                          >
+                            <Check className="w-3.5 h-3.5" strokeWidth={3} />
+                          </div>
+
+                          <div className="aspect-square p-4 flex items-center justify-center relative overflow-hidden rounded-t-lg">
+                            {/* Checkerboard bg */}
+                            <div
+                              className="absolute inset-0 opacity-5"
+                              style={{
+                                backgroundImage: 'radial-gradient(#000 1px, transparent 1px)',
+                                backgroundSize: '8px 8px',
+                              }}
+                            ></div>
+                            <img
+                              src={item.base64}
+                              className="max-w-full max-h-full object-contain drop-shadow-md"
+                            />
+                          </div>
+                          <div
+                            className={cn(
+                              'p-2 border-t rounded-b-lg transition-colors',
+                              isSelected
+                                ? 'bg-white border-amber-100'
+                                : 'bg-stone-50 border-stone-100',
+                            )}
+                          >
+                            <div className="text-xs font-bold text-stone-700 truncate">
+                              {item.name}
+                            </div>
+                            <div className="text-[10px] text-stone-400 truncate uppercase">
+                              {item.category}
+                            </div>
+                          </div>
+                        </div>
+                      )
+                    })}
+                  </div>
+                </div>
+              )}
+
+              {/* Final Results Grid */}
              {(results.length > 0 || status === 'complete') && (
                <div className="bg-white p-6 rounded-2xl border border-stone-200 shadow-xl animate-fade-in-up delay-100">
                  <div className="flex justify-between items-center mb-6">
                    <div>
                      <h2 className="text-xl font-black text-stone-800 flex items-center gap-2 uppercase tracking-tighter">
-                        <span className="text-amber-500">✦</span> Loot Acquired
+                        <span className="text-green-500">✦</span> Loot Acquired
                      </h2>
                      <p className="text-stone-500 text-xs font-bold tracking-[0.2em] uppercase mt-1">
-                        {results.length} items found & stashed
+                        {results.length} items refined & stashed
                      </p>
                    </div>
                  </div>
@@ -332,7 +539,7 @@ export const ExtractorDialog: React.FC<ExtractorDialogProps> = ({
                    {results.map((item, i) => (
                      <div
                        key={i}
-                        className="group relative bg-stone-50 rounded-lg border-2 border-stone-200 hover:border-amber-400 hover:shadow-lg transition-all duration-300"
+                        className="group relative bg-stone-50 rounded-lg border-2 border-stone-200 hover:border-green-400 hover:shadow-lg transition-all duration-300"
                      >
                        <div className="aspect-square p-4 flex items-center justify-center relative overflow-hidden rounded-t-lg">
                          {/* Checkerboard bg */}
@@ -378,7 +585,7 @@ export const ExtractorDialog: React.FC<ExtractorDialogProps> = ({
                          <div className="text-[10px] text-stone-400 truncate uppercase">
                            {item.category}
                          </div>
-                          <div className="w-6 h-1 bg-amber-400 rounded-full mt-2"></div>
+                          <div className="w-6 h-1 bg-green-400 rounded-full mt-2"></div>
                        </div>
                      </div>
                    ))}
@@ -3,10 +3,10 @@ import React from 'react'
 import { cn } from '../lib/utils'
 import { Button } from './ui/button'
 import {
-    DropdownMenu,
-    DropdownMenuContent,
-    DropdownMenuItem,
-    DropdownMenuTrigger,
+  DropdownMenu,
+  DropdownMenuContent,
+  DropdownMenuItem,
+  DropdownMenuTrigger,
 } from './ui/dropdown-menu'

 export interface VibeCardAction {
@@ -7,11 +7,20 @@ export interface ExtractedAsset {
  imageUrl: string
 }

+export interface CandidateAsset {
+  name: string
+  description: string
+  category: string
+  subCategory?: string
+  base64: string
+}
+
 export type ExtractionStatus =
  | 'idle'
  | 'analyzing'
  | 'generating'
  | 'splitting'
+  | 'selection'
  | 'refining'
  | 'complete'
  | 'error'