From 4615a075b1e3c53de2e97c85bc55e5aa2e533a96 Mon Sep 17 00:00:00 2001 From: yiling ji Date: Sat, 7 Feb 2026 21:05:33 +0800 Subject: [PATCH] add demo ui --- ui/ui-demo/.gitignore | 24 ++ ui/ui-demo/App.tsx | 173 ++++++++++ ui/ui-demo/README.md | 20 ++ ui/ui-demo/components/AssetList.tsx | 89 +++++ ui/ui-demo/components/Extractor.tsx | 277 ++++++++++++++++ ui/ui-demo/components/LoadingOverlay.tsx | 71 ++++ ui/ui-demo/components/Modifier.tsx | 197 +++++++++++ ui/ui-demo/components/Stash.tsx | 74 +++++ ui/ui-demo/index.html | 64 ++++ ui/ui-demo/index.tsx | 15 + ui/ui-demo/metadata.json | 5 + ui/ui-demo/package.json | 22 ++ ui/ui-demo/services/geminiService.ts | 398 +++++++++++++++++++++++ ui/ui-demo/tsconfig.json | 29 ++ ui/ui-demo/types.ts | 53 +++ ui/ui-demo/vite.config.ts | 23 ++ 16 files changed, 1534 insertions(+) create mode 100644 ui/ui-demo/.gitignore create mode 100644 ui/ui-demo/App.tsx create mode 100644 ui/ui-demo/README.md create mode 100644 ui/ui-demo/components/AssetList.tsx create mode 100644 ui/ui-demo/components/Extractor.tsx create mode 100644 ui/ui-demo/components/LoadingOverlay.tsx create mode 100644 ui/ui-demo/components/Modifier.tsx create mode 100644 ui/ui-demo/components/Stash.tsx create mode 100644 ui/ui-demo/index.html create mode 100644 ui/ui-demo/index.tsx create mode 100644 ui/ui-demo/metadata.json create mode 100644 ui/ui-demo/package.json create mode 100644 ui/ui-demo/services/geminiService.ts create mode 100644 ui/ui-demo/tsconfig.json create mode 100644 ui/ui-demo/types.ts create mode 100644 ui/ui-demo/vite.config.ts diff --git a/ui/ui-demo/.gitignore b/ui/ui-demo/.gitignore new file mode 100644 index 0000000..a547bf3 --- /dev/null +++ b/ui/ui-demo/.gitignore @@ -0,0 +1,24 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/ui/ui-demo/App.tsx b/ui/ui-demo/App.tsx new file mode 100644 index 0000000..e7d6c2c --- /dev/null +++ b/ui/ui-demo/App.tsx @@ -0,0 +1,173 @@ +import React, { useState, useEffect } from 'react'; +import { ExtractionStatus, StashItem, CroppedAsset } from './types'; +import { LoadingOverlay } from './components/LoadingOverlay'; +import { Extractor } from './components/Extractor'; +import { Stash } from './components/Stash'; +import { Modifier } from './components/Modifier'; + +export default function App() { + const [view, setView] = useState<'extractor' | 'stash' | 'modifier'>('extractor'); + const [status, setStatus] = useState('idle'); + const [loadingCurrent, setLoadingCurrent] = useState(0); + const [loadingTotal, setLoadingTotal] = useState(0); + const [error, setError] = useState(null); + const [stash, setStash] = useState([]); + const [apiKeySelected, setApiKeySelected] = useState(false); + const [notification, setNotification] = useState(null); + + useEffect(() => { + checkApiKey(); + loadStash(); + }, []); + + const checkApiKey = async () => { + const aistudio = (window as any).aistudio; + if (aistudio?.hasSelectedApiKey) { + const hasKey = await aistudio.hasSelectedApiKey(); + setApiKeySelected(hasKey); + } else { + setApiKeySelected(true); + } + }; + + const handleSelectKey = async () => { + const aistudio = (window as any).aistudio; + if (aistudio?.openSelectKey) { + await aistudio.openSelectKey(); + setApiKeySelected(true); + } + }; + + const loadStash = () => { + try { + // Changed key to reflect new app name + const saved = localStorage.getItem('forgery_stash'); + if (saved) setStash(JSON.parse(saved)); + } catch (e) { + console.error("Failed to load stash", e); + } + }; + + const saveStash = (items: StashItem[]) => { + setStash(items); + localStorage.setItem('forgery_stash', JSON.stringify(items)); + }; + + const handleAddToStash = (assets: CroppedAsset[]) => { + const newItems: StashItem[] = assets.map(asset => ({ + id: Math.random().toString(36).substr(2, 9), + item_name: asset.item_name, + imageUrl: asset.refinedImageUrl || asset.imageUrl, + createdAt: Date.now() + })); + saveStash([...newItems, ...stash]); + showNotification("Loot successfully stashed!"); + }; + + const showNotification = (msg: string) => { + setNotification(msg); + setTimeout(() => setNotification(null), 3000); + }; + + const handleStatusChange = (newStatus: ExtractionStatus, current?: number, total?: number) => { + setStatus(newStatus); + if (current !== undefined) setLoadingCurrent(current); + if (total !== undefined) setLoadingTotal(total); + }; + + const NavButton = ({ target, label, color, icon }: any) => ( + + ); + + return ( +
+ {/* Notifications */} + {notification && ( +
+
+ {notification} +
+ )} + + {/* Global Error */} + {error && ( +
+ {error} + +
+ )} + + {/* Loading */} + {(status !== 'idle' && status !== 'complete' && status !== 'error') && ( + + )} + +
+
+
+ +
+
+

Forgery

+

Digital Asset Synthesis

+
+
+ +
+ + + + + {!apiKeySelected && (window as any).aistudio && ( + + )} +
+
+ +
+ {view === 'extractor' && ( + {}} + apiKeyValid={apiKeySelected} + onConnectKey={handleSelectKey} + onSaveToStash={handleAddToStash} + /> + )} + {view === 'stash' && ( + saveStash(stash.filter(i => i.id !== id))} + onClear={() => saveStash([])} + onNavigateToExtractor={() => setView('extractor')} + /> + )} + {view === 'modifier' && ( + setView('extractor')} + /> + )} +
+
+ ); +} \ No newline at end of file diff --git a/ui/ui-demo/README.md b/ui/ui-demo/README.md new file mode 100644 index 0000000..3d0d7bf --- /dev/null +++ b/ui/ui-demo/README.md @@ -0,0 +1,20 @@ +
+GHBanner +
+ +# Run and deploy your AI Studio app + +This contains everything you need to run your app locally. + +View your app in AI Studio: https://ai.studio/apps/drive/19Xb6Drl-US_jUxV3S8ruwCrcP6C8ZUHh + +## Run Locally + +**Prerequisites:** Node.js + + +1. Install dependencies: + `npm install` +2. Set the `GEMINI_API_KEY` in [.env.local](.env.local) to your Gemini API key +3. Run the app: + `npm run dev` diff --git a/ui/ui-demo/components/AssetList.tsx b/ui/ui-demo/components/AssetList.tsx new file mode 100644 index 0000000..987fc45 --- /dev/null +++ b/ui/ui-demo/components/AssetList.tsx @@ -0,0 +1,89 @@ +import React from 'react'; +import { Asset, HEX_COLORS } from '../types'; + +interface AssetListProps { + assets: Asset[]; + onConfirm: () => void; + onCancel: () => void; + selectedModel: string; + onModelChange: (model: string) => void; +} + +export const AssetList: React.FC = ({ assets, onConfirm, onCancel, selectedModel, onModelChange }) => { + return ( +
+
+
+

Identified Assets

+

Review the items detected for extraction.

+
+ + {assets.length} Items Found + +
+ +
+ {assets.map((asset, idx) => ( +
+
+
+

{asset.item_name}

+

{asset.description}

+
+ Target Key: {asset.background_color} +
+
+
+ ))} +
+ + {/* Model Selection */} +
+

Extraction Model

+
+
+ ); +}; \ No newline at end of file diff --git a/ui/ui-demo/components/Extractor.tsx b/ui/ui-demo/components/Extractor.tsx new file mode 100644 index 0000000..faa5528 --- /dev/null +++ b/ui/ui-demo/components/Extractor.tsx @@ -0,0 +1,277 @@ +import React, { useState, useEffect } from 'react'; +import { analyzeImageForAssets, generateTextureSheet, detectAssetBoundingBoxes, cropAssetsFromSheet, refineAssetImage } from '../services/geminiService'; +import { AnalysisResponse, CroppedAsset, ExtractionStatus } from '../types'; + +interface ExtractorProps { + onStatusChange: (status: ExtractionStatus, index?: number, total?: number) => void; + onError: (msg: string | null) => void; + onAssetsExtracted: (assets: CroppedAsset[]) => void; + apiKeyValid: boolean; + onConnectKey: () => void; + onSaveToStash: (assets: CroppedAsset[]) => void; +} + +export const Extractor: React.FC = ({ + onStatusChange, + onError, + onAssetsExtracted, + apiKeyValid, + onConnectKey, + onSaveToStash +}) => { + const [files, setFiles] = useState([]); + const [previews, setPreviews] = useState([]); + const [results, setResults] = useState([]); + const [selectedSourceIdx, setSelectedSourceIdx] = useState(null); + + // Handle Paste Events + useEffect(() => { + const handlePaste = (e: ClipboardEvent) => { + if (e.clipboardData && e.clipboardData.items) { + const items = e.clipboardData.items; + const newFiles: File[] = []; + for (let i = 0; i < items.length; i++) { + if (items[i].type.indexOf("image") !== -1) { + const blob = items[i].getAsFile(); + if (blob) { + newFiles.push(blob); + } + } + } + + if (newFiles.length > 0) { + addFiles(newFiles); + } + } + }; + + window.addEventListener('paste', handlePaste); + return () => { + window.removeEventListener('paste', handlePaste); + }; + }, []); // Empty dependency array means this runs once on mount and cleans up on unmount + + const handleFileChange = (e: React.ChangeEvent) => { + if (e.target.files) addFiles(Array.from(e.target.files)); + }; + + const addFiles = (newFiles: File[]) => { + setFiles(prev => [...prev, ...newFiles]); + setPreviews(prev => [...prev, ...newFiles.map(f => URL.createObjectURL(f))]); + setResults([]); // Clear results on new upload + onError(null); + }; + + const removeFile = (idx: number) => { + const newFiles = [...files]; + const newPreviews = [...previews]; + URL.revokeObjectURL(newPreviews[idx]); + newFiles.splice(idx, 1); + newPreviews.splice(idx, 1); + setFiles(newFiles); + setPreviews(newPreviews); + }; + + const handleExtract = async () => { + if (files.length === 0) return; + if (!apiKeyValid) { + onConnectKey(); + return; + } + + setResults([]); + onError(null); + const allAssets: CroppedAsset[] = []; + + try { + for (let i = 0; i < files.length; i++) { + onStatusChange('analyzing', i + 1, files.length); + + // 1. Analyze + let analysis: AnalysisResponse; + try { + analysis = await analyzeImageForAssets(files[i]); + } catch (e) { + console.error(`Analysis failed for ${files[i].name}`, e); + continue; + } + + if (analysis.assets.length === 0) continue; + + // 2. Generate Sheet + onStatusChange('generating', i + 1, files.length); + const sheetUrl = await generateTextureSheet(files[i], analysis.assets); + + // 3. Slice + onStatusChange('splitting', i + 1, files.length); + const base64 = sheetUrl.split(',')[1]; + const boxes = await detectAssetBoundingBoxes(base64); + const cropped = await cropAssetsFromSheet(sheetUrl, boxes.assets); + + // 4. Refine + onStatusChange('refining', i + 1, files.length); + const refined = await Promise.all(cropped.map(async (a) => { + try { + const refinedUrl = await refineAssetImage(a.imageUrl); + return { ...a, refinedImageUrl: refinedUrl, sourceIndex: i }; + } catch { + return { ...a, sourceIndex: i }; + } + })); + + allAssets.push(...refined); + } + + setResults(allAssets); + onAssetsExtracted(allAssets); + onStatusChange('complete'); + if (allAssets.length > 0) setSelectedSourceIdx(0); + + } catch (e: any) { + console.error(e); + onError(e.message || "Extraction failed"); + onStatusChange('error'); + } + }; + + const handleReset = () => { + setFiles([]); + previews.forEach(p => URL.revokeObjectURL(p)); + setPreviews([]); + setResults([]); + onStatusChange('idle'); + }; + + if (results.length > 0) { + const filtered = selectedSourceIdx !== null + ? results.filter(r => r.sourceIndex === selectedSourceIdx) + : results; + + return ( +
+ {/* Source Selection Strip */} +
+

Source References

+
+ {previews.map((src, idx) => ( + + ))} +
+
+ + {/* Results Grid */} +
+
+
+

+ ✦ LOOT ACQUIRED +

+

{filtered.length} items found

+
+
+ + +
+
+ +
+ {filtered.map((item, i) => ( +
+
+ {/* Checkerboard bg */} +
+ + + {/* Download Overlay */} + + + +
+
+
{item.item_name}
+
+
+
+ ))} +
+
+
+ ); + } + + // Initial State + return ( +
+ {files.length === 0 ? ( +
+
+ +
+

UPLOAD SOURCES

+

Drop character images to identify and extract equipment sprites.

+
+ Paste (Ctrl+V) Supported +
+
+ +
+
+ ) : ( +
+
+
+

Staging Area ({files.length})

+ Drag, Drop, or Paste more +
+ +
+ +
+ {previews.map((src, i) => ( +
+ + +
+ ))} +
+ + +

POWERED BY GEMINI 2.5 FLASH

+
+ )} +
+ ); +}; \ No newline at end of file diff --git a/ui/ui-demo/components/LoadingOverlay.tsx b/ui/ui-demo/components/LoadingOverlay.tsx new file mode 100644 index 0000000..febff12 --- /dev/null +++ b/ui/ui-demo/components/LoadingOverlay.tsx @@ -0,0 +1,71 @@ +import React from 'react'; + +interface LoadingOverlayProps { + status: string; + current?: number; + total?: number; +} + +export const LoadingOverlay: React.FC = ({ status, current, total }) => { + let title = 'PROCESSING'; + let description = 'Please wait...'; + + switch (status) { + case 'analyzing': + title = 'SCANNING ASSETS'; + description = 'Identifying equipment geometry...'; + break; + case 'generating': + title = 'FORGING TEXTURES'; + description = 'Creating ghost mannequin sprites...'; + break; + case 'splitting': + title = 'CUTTING SPRITES'; + description = 'Slicing texture sheet boundaries...'; + break; + case 'refining': + title = 'POLISHING'; + description = 'Refining edges and normalizing...'; + break; + case 'modifying': + title = 'SYNTHESIZING'; + description = 'Equipping character with new gear...'; + break; + } + + return ( +
+
+ {/* Top Decoration */} +
+ +
+
+
+
+ +

+ {title} +

+

+ {description} +

+ + {total && total > 1 && ( +
+
+ Progress + {current} / {total} +
+
+
+
+
+ )} +
+
+ ); +}; \ No newline at end of file diff --git a/ui/ui-demo/components/Modifier.tsx b/ui/ui-demo/components/Modifier.tsx new file mode 100644 index 0000000..cc251c9 --- /dev/null +++ b/ui/ui-demo/components/Modifier.tsx @@ -0,0 +1,197 @@ +import React, { useState } from 'react'; +import { generateModifiedCharacter } from '../services/geminiService'; +import { StashItem, ExtractionStatus } from '../types'; + +interface ModifierProps { + stash: StashItem[]; + onStatusChange: (status: ExtractionStatus) => void; + onError: (msg: string | null) => void; + apiKeyValid: boolean; + onConnectKey: () => void; + onNavigateToExtractor: () => void; +} + +export const Modifier: React.FC = ({ + stash, + onStatusChange, + onError, + apiKeyValid, + onConnectKey, + onNavigateToExtractor +}) => { + const [baseFile, setBaseFile] = useState(null); + const [basePreview, setBasePreview] = useState(null); + const [selectedItemIds, setSelectedItemIds] = useState>(new Set()); + const [resultUrl, setResultUrl] = useState(null); + const [selectedModel, setSelectedModel] = useState('gemini-3-pro-image-preview'); + + const handleBaseChange = (e: React.ChangeEvent) => { + if (e.target.files && e.target.files[0]) { + const f = e.target.files[0]; + setBaseFile(f); + setBasePreview(URL.createObjectURL(f)); + setResultUrl(null); + onError(null); + } + }; + + const toggleItem = (id: string) => { + const next = new Set(selectedItemIds); + if (next.has(id)) next.delete(id); + else next.add(id); + setSelectedItemIds(next); + }; + + const handleForge = async () => { + if (!baseFile || selectedItemIds.size === 0) return; + if (!apiKeyValid) { + onConnectKey(); + return; + } + + onStatusChange('modifying'); + setResultUrl(null); + onError(null); + + try { + const items = stash.filter(i => selectedItemIds.has(i.id)); + const url = await generateModifiedCharacter(baseFile, items, selectedModel); + setResultUrl(url); + onStatusChange('idle'); + } catch (e: any) { + console.error(e); + onError(e.message || "Modification failed"); + onStatusChange('error'); + } + }; + + return ( +
+
+

+ ⚒ SMITHY +

+

Equip your characters with extracted assets.

+
+ +
+ {/* LEFT: Base Character */} +
+

Base Character

+ +
+ {basePreview ? ( + <> + + + + ) : ( + + )} +
+ + {/* Model Selector */} +
+

Forge Model

+
+ + +
+
+
+ + {/* CENTER: Stash Selector */} +
+
+

Available Equipment

+ {selectedItemIds.size} Selected +
+ +
+ {stash.length === 0 ? ( +
+

No equipment in stash.

+ +
+ ) : ( +
+ {stash.map(item => { + const isSelected = selectedItemIds.has(item.id); + return ( +
toggleItem(item.id)} + className={`cursor-pointer rounded-lg border-2 p-2 transition-all hover:shadow-md ${isSelected ? 'border-purple-500 bg-purple-50' : 'border-stone-200 bg-white hover:border-stone-300'}`} + > +
+ +
+

{item.item_name}

+
+ ) + })} +
+ )} +
+ +
+ +
+
+
+ + {/* RESULT */} + {resultUrl && ( +
+

Masterpiece Created

+ +
+ )} +
+ ); +}; \ No newline at end of file diff --git a/ui/ui-demo/components/Stash.tsx b/ui/ui-demo/components/Stash.tsx new file mode 100644 index 0000000..f448fe7 --- /dev/null +++ b/ui/ui-demo/components/Stash.tsx @@ -0,0 +1,74 @@ +import React from 'react'; +import { StashItem } from '../types'; + +interface StashProps { + items: StashItem[]; + onRemove: (id: string) => void; + onClear: () => void; + onNavigateToExtractor: () => void; +} + +export const Stash: React.FC = ({ items, onRemove, onClear, onNavigateToExtractor }) => { + return ( +
+
+
+

+ ❖ INVENTORY +

+

Manage your collection of extracted assets.

+
+ {items.length > 0 && ( + + )} +
+ + {items.length === 0 ? ( +
+
+ +
+

Inventory Empty

+

Extract assets to populate your stash.

+ +
+ ) : ( +
+ {items.map((item) => ( +
+
+
+ + +
+ + Download + + +
+
+
+

{item.item_name}

+

{new Date(item.createdAt).toLocaleDateString()}

+
+
+ ))} +
+ )} +
+ ); +}; \ No newline at end of file diff --git a/ui/ui-demo/index.html b/ui/ui-demo/index.html new file mode 100644 index 0000000..f5fb26a --- /dev/null +++ b/ui/ui-demo/index.html @@ -0,0 +1,64 @@ + + + + + + Forgery + + + + + + +
+ + + \ No newline at end of file diff --git a/ui/ui-demo/index.tsx b/ui/ui-demo/index.tsx new file mode 100644 index 0000000..6ca5361 --- /dev/null +++ b/ui/ui-demo/index.tsx @@ -0,0 +1,15 @@ +import React from 'react'; +import ReactDOM from 'react-dom/client'; +import App from './App'; + +const rootElement = document.getElementById('root'); +if (!rootElement) { + throw new Error("Could not find root element to mount to"); +} + +const root = ReactDOM.createRoot(rootElement); +root.render( + + + +); \ No newline at end of file diff --git a/ui/ui-demo/metadata.json b/ui/ui-demo/metadata.json new file mode 100644 index 0000000..1bdc092 --- /dev/null +++ b/ui/ui-demo/metadata.json @@ -0,0 +1,5 @@ +{ + "name": "Forgery", + "description": "Extract character assets and forge new characters using Gemini models.", + "requestFramePermissions": [] +} \ No newline at end of file diff --git a/ui/ui-demo/package.json b/ui/ui-demo/package.json new file mode 100644 index 0000000..bf6c276 --- /dev/null +++ b/ui/ui-demo/package.json @@ -0,0 +1,22 @@ +{ + "name": "forgery", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "dependencies": { + "react": "^19.2.4", + "@google/genai": "^1.40.0", + "react-dom": "^19.2.4" + }, + "devDependencies": { + "@types/node": "^22.14.0", + "@vitejs/plugin-react": "^5.0.0", + "typescript": "~5.8.2", + "vite": "^6.2.0" + } +} diff --git a/ui/ui-demo/services/geminiService.ts b/ui/ui-demo/services/geminiService.ts new file mode 100644 index 0000000..3c6e7be --- /dev/null +++ b/ui/ui-demo/services/geminiService.ts @@ -0,0 +1,398 @@ +import { GoogleGenAI, Type } from "@google/genai"; +import { AnalysisResponse, Asset, BoundingBoxResponse, CroppedAsset, DetectedAssetBox, HEX_COLORS, StashItem } from "../types"; + +// Helper to convert File to Base64 +const fileToGenericBase64 = async (file: File): Promise => { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + const result = reader.result as string; + // Remove data URL prefix (e.g., "data:image/png;base64,") + const base64 = result.split(',')[1]; + resolve(base64); + }; + reader.onerror = (error) => reject(error); + reader.readAsDataURL(file); + }); +}; + +export const analyzeImageForAssets = async (file: File): Promise => { + // Use a smart model for analysis (Gemini 3 Pro or Flash) to get accurate JSON + const ai = new GoogleGenAI({ apiKey: process.env.API_KEY }); + const base64Data = await fileToGenericBase64(file); + + const prompt = ` +Task: Character Asset Identification for Extraction + +Instructions: +1. Analyze the character in the image and identify all distinct, extractable equipment and clothing items. +2. Filter for items that are clearly visible: tops, bottoms, headwear, footwear, specialized armor (arm guards, leg guards), and accessories (scarves, belts, capes). +3. For each identified item, assign one of the following high-contrast background colors for future segmentation: Red, Yellow, Green, White, Blue, Black, Magenta, Cyan, Orange, Gray. Avoid repeating colors if possible. +4. Limit to a maximum of 9 most prominent assets. + +Output Format (Strict JSON): +Return ONLY a JSON object with a list of assets. +`; + + const response = await ai.models.generateContent({ + model: 'gemini-3-pro-preview', + contents: { + parts: [ + { inlineData: { mimeType: file.type, data: base64Data } }, + { text: prompt } + ] + }, + config: { + responseMimeType: "application/json", + responseSchema: { + type: Type.OBJECT, + properties: { + assets: { + type: Type.ARRAY, + items: { + type: Type.OBJECT, + properties: { + item_name: { type: Type.STRING }, + background_color: { type: Type.STRING }, + description: { type: Type.STRING }, + }, + required: ['item_name', 'background_color', 'description'], + }, + }, + }, + required: ['assets'] + }, + }, + }); + + const text = response.text; + if (!text) throw new Error("No response from analysis model"); + + let cleanText = text.trim(); + if (cleanText.startsWith('```')) { + cleanText = cleanText.replace(/^```(json)?/, '').replace(/```$/, '').trim(); + } + + const parsed = JSON.parse(cleanText); + if (!parsed.assets) parsed.assets = []; + + return parsed as AnalysisResponse; +}; + +export const generateTextureSheet = async (originalFile: File, assets: Asset[]): Promise => { + // Use Gemini 2.5 Flash Image ("Banana") for generation as requested + const ai = new GoogleGenAI({ apiKey: process.env.API_KEY }); + const base64Data = await fileToGenericBase64(originalFile); + + // Construct the mapping string dynamically + const mappingString = assets.map(asset => { + const colorName = asset.background_color; + // Fallback to black if hex not found + const hex = HEX_COLORS[colorName] || '#000000'; + return `- ${asset.item_name}: ${colorName} Background (${hex}). ${asset.description}`; + }).join('\n'); + + const prompt = ` +Task: Character Asset Extraction & Grid Generation + +Instructions: +Analyze the character in the provided image and extract the following ${assets.length} specific assets into a new, single image organized in a grid layout (e.g., 2x3, 3x3 depending on count). + +Core Constraints: +1. Isolated Assets: Each item must be placed within its own distinct square tile. No overlapping. +2. Ghost Mannequin Style: Extract ONLY the items. Remove the character's body entirely. Clothing and armor must appear as empty, 3D shells as if worn by an invisible person. +3. Solid Chroma Key Backgrounds: Each tile must use the specific solid background color assigned below. Ensure zero shadows, highlights, or gradients on the background to facilitate clean digital segmentation. +4. Output Format: Generate a square texture sheet. Ignore the original image aspect ratio for the final output canvas. + +Item & Background Mapping: +${mappingString} + +Output Requirement: High-resolution texture sheet, flat lay presentation, sharp edges, and uniform lighting across all assets. +`; + + const response = await ai.models.generateContent({ + model: 'gemini-2.5-flash-image', + contents: { + parts: [ + { inlineData: { mimeType: originalFile.type, data: base64Data } }, + { text: prompt } + ] + }, + config: { + imageConfig: { + aspectRatio: '1:1', + // imageSize not supported for gemini-2.5-flash-image + } + } + }); + + // Extract image + let imageUrl = ''; + if (response.candidates && response.candidates[0].content.parts) { + for (const part of response.candidates[0].content.parts) { + if (part.inlineData) { + imageUrl = `data:${part.inlineData.mimeType};base64,${part.inlineData.data}`; + break; + } + } + } + + if (!imageUrl) { + throw new Error("No image generated."); + } + + return imageUrl; +}; + +export const detectAssetBoundingBoxes = async (textureSheetBase64: string): Promise => { + const ai = new GoogleGenAI({ apiKey: process.env.API_KEY }); + + const prompt = ` +Task: Spatial Asset Detection & Normalized Grid Mapping + +Instructions: +1. Analyze the provided image consisting of character assets arranged in a grid. +2. Identify the bounding box for each distinct asset container (the colored square background tiles). +3. Return the coordinates for each box using a normalized scale of 0.0 to 1.0. + - [ymin, xmin, ymax, xmax] + - (0, 0) is the top-left corner; (1.0, 1.0) is the bottom-right corner. +4. Ensure the box coordinates strictly encompass the outer boundary of each colored tile. + +Output Format (Strict JSON): +Return ONLY a JSON object. Provide the grid dimensions and a list of detected assets with their coordinates. +`; + + const response = await ai.models.generateContent({ + model: 'gemini-3-pro-preview', + contents: { + parts: [ + { inlineData: { mimeType: 'image/png', data: textureSheetBase64 } }, + { text: prompt } + ] + }, + config: { + responseMimeType: "application/json", + responseSchema: { + type: Type.OBJECT, + properties: { + grid_dimensions: { + type: Type.OBJECT, + properties: { + rows: { type: Type.INTEGER }, + cols: { type: Type.INTEGER } + } + }, + assets: { + type: Type.ARRAY, + items: { + type: Type.OBJECT, + properties: { + item_name: { type: Type.STRING }, + coordinates: { + type: Type.ARRAY, + items: { type: Type.NUMBER }, + description: "[ymin, xmin, ymax, xmax]" + }, + bg_color_detected: { type: Type.STRING } + } + } + } + }, + required: ['assets', 'grid_dimensions'] + } + } + }); + + const text = response.text; + if (!text) throw new Error("No bounding box response from model"); + + let cleanText = text.trim(); + if (cleanText.startsWith('```')) { + cleanText = cleanText.replace(/^```(json)?/, '').replace(/```$/, '').trim(); + } + + let parsed: any; + try { + parsed = JSON.parse(cleanText); + } catch (e) { + console.error("Failed to parse JSON:", cleanText); + throw new Error("Invalid JSON response from detection model"); + } + + // Ensure assets array exists + if (!parsed.assets || !Array.isArray(parsed.assets)) { + parsed.assets = []; + } + + return parsed as BoundingBoxResponse; +}; + +export const cropAssetsFromSheet = async (sheetDataUrl: string, boxes: DetectedAssetBox[]): Promise[]> => { + return new Promise((resolve, reject) => { + const img = new Image(); + img.crossOrigin = "anonymous"; + img.onload = () => { + try { + const cropped: Omit[] = []; + + if (boxes && Array.isArray(boxes)) { + boxes.forEach(box => { + if (!box.coordinates || box.coordinates.length !== 4) return; + + const [ymin, xmin, ymax, xmax] = box.coordinates; + const canvas = document.createElement('canvas'); + const ctx = canvas.getContext('2d'); + + if (!ctx) return; + + // Convert normalized coords to pixels + const srcX = xmin * img.width; + const srcY = ymin * img.height; + const srcW = (xmax - xmin) * img.width; + const srcH = (ymax - ymin) * img.height; + + canvas.width = srcW; + canvas.height = srcH; + + ctx.drawImage(img, srcX, srcY, srcW, srcH, 0, 0, srcW, srcH); + + cropped.push({ + item_name: box.item_name || "Unknown Asset", + imageUrl: canvas.toDataURL('image/png') + }); + }); + } + + resolve(cropped); + } catch (e) { + reject(e); + } + }; + img.onerror = (e) => reject(e); + img.src = sheetDataUrl; + }); +}; + +export const refineAssetImage = async (base64Data: string): Promise => { + const ai = new GoogleGenAI({ apiKey: process.env.API_KEY }); + + const prompt = ` +Task: Asset Refinement + +Instructions: +1. Process the provided asset image. +2. Remove the existing colored background and replace it with a pure white background (#FFFFFF). +3. Upscale the image and enhance details for high-quality game asset presentation. +4. Ensure the object is centered and fully visible. +5. Return only the image. + `; + + // Remove header if present + const cleanBase64 = base64Data.includes(',') ? base64Data.split(',')[1] : base64Data; + + const response = await ai.models.generateContent({ + model: 'gemini-2.5-flash-image', + contents: { + parts: [ + { inlineData: { mimeType: 'image/png', data: cleanBase64 } }, + { text: prompt } + ] + } + }); + + // Extract image + let imageUrl = ''; + if (response.candidates && response.candidates[0].content.parts) { + for (const part of response.candidates[0].content.parts) { + if (part.inlineData) { + imageUrl = `data:${part.inlineData.mimeType};base64,${part.inlineData.data}`; + break; + } + } + } + + if (!imageUrl) { + throw new Error("No refined image generated."); + } + + return imageUrl; +}; + +export const generateModifiedCharacter = async ( + characterFile: File, + assets: StashItem[], + model: string = 'gemini-3-pro-image-preview' +): Promise => { + const ai = new GoogleGenAI({ apiKey: process.env.API_KEY }); + + // Base Character + const charBase64 = await fileToGenericBase64(characterFile); + + const parts: any[] = []; + + parts.push({ + inlineData: { mimeType: characterFile.type, data: charBase64 } + }); + parts.push({ text: "Base Character Reference" }); + + // Assets + for (const asset of assets) { + // imageUrl is likely a data URL "data:image/png;base64,....." + const matches = asset.imageUrl.match(/^data:(.+);base64,(.+)$/); + if (matches && matches.length === 3) { + const mimeType = matches[1]; + const data = matches[2]; + parts.push({ + inlineData: { mimeType, data } + }); + parts.push({ text: `Equipment to equip: ${asset.item_name}` }); + } + } + + const prompt = ` + Task: Character Synthesis and Equipment Modification. + + 1. Reference the 'Base Character' for pose, body type, facial features, and general art style. + 2. Reference the 'Equipment to equip' images. + 3. Generate a new image of the Base Character wearing the provided items. + 4. The items should replace existing clothing/armor in that slot (e.g. new helmet replaces old hat, armor replaces shirt). + 5. Seamlessly blend the items into the character's style. + 6. Maintain the exact pose and angle of the Base Character. + 7. High quality, detailed, game art style. + `; + + parts.push({ text: prompt }); + + const imageConfig: any = { + aspectRatio: '1:1', + }; + + if (model === 'gemini-3-pro-image-preview') { + imageConfig.imageSize = '2K'; + } + + const response = await ai.models.generateContent({ + model: model, + contents: { parts }, + config: { + imageConfig: imageConfig + } + }); + + // Extract image + let imageUrl = ''; + if (response.candidates && response.candidates[0].content.parts) { + for (const part of response.candidates[0].content.parts) { + if (part.inlineData) { + imageUrl = `data:${part.inlineData.mimeType};base64,${part.inlineData.data}`; + break; + } + } + } + + if (!imageUrl) { + throw new Error("No image generated."); + } + + return imageUrl; +}; \ No newline at end of file diff --git a/ui/ui-demo/tsconfig.json b/ui/ui-demo/tsconfig.json new file mode 100644 index 0000000..2c6eed5 --- /dev/null +++ b/ui/ui-demo/tsconfig.json @@ -0,0 +1,29 @@ +{ + "compilerOptions": { + "target": "ES2022", + "experimentalDecorators": true, + "useDefineForClassFields": false, + "module": "ESNext", + "lib": [ + "ES2022", + "DOM", + "DOM.Iterable" + ], + "skipLibCheck": true, + "types": [ + "node" + ], + "moduleResolution": "bundler", + "isolatedModules": true, + "moduleDetection": "force", + "allowJs": true, + "jsx": "react-jsx", + "paths": { + "@/*": [ + "./*" + ] + }, + "allowImportingTsExtensions": true, + "noEmit": true + } +} \ No newline at end of file diff --git a/ui/ui-demo/types.ts b/ui/ui-demo/types.ts new file mode 100644 index 0000000..65ca5e8 --- /dev/null +++ b/ui/ui-demo/types.ts @@ -0,0 +1,53 @@ +export interface Asset { + item_name: string; + background_color: string; + description: string; +} + +export interface AnalysisResponse { + assets: Asset[]; +} + +export type ExtractionStatus = 'idle' | 'analyzing' | 'reviewing' | 'generating' | 'sheet_ready' | 'splitting' | 'refining' | 'complete' | 'error' | 'modifying'; + +export interface ColorMap { + [key: string]: string; +} + +export const HEX_COLORS: ColorMap = { + Red: '#FF0000', + Yellow: '#FFFF00', + Green: '#00FF00', + White: '#FFFFFF', + Blue: '#0000FF', + Black: '#000000', + Magenta: '#FF00FF', + Cyan: '#00FFFF', + Orange: '#FFA500', + Gray: '#808080', +}; + +export interface DetectedAssetBox { + item_name: string; + coordinates: [number, number, number, number]; // ymin, xmin, ymax, xmax + bg_color_detected: string; +} + +export interface BoundingBoxResponse { + grid_dimensions: { rows: number; cols: number }; + assets: DetectedAssetBox[]; +} + +export interface CroppedAsset { + item_name: string; + imageUrl: string; + refinedImageUrl?: string; + sourceIndex: number; +} + +export interface StashItem { + id: string; + item_name: string; + imageUrl: string; + createdAt: number; +} \ No newline at end of file diff --git a/ui/ui-demo/vite.config.ts b/ui/ui-demo/vite.config.ts new file mode 100644 index 0000000..ee5fb8d --- /dev/null +++ b/ui/ui-demo/vite.config.ts @@ -0,0 +1,23 @@ +import path from 'path'; +import { defineConfig, loadEnv } from 'vite'; +import react from '@vitejs/plugin-react'; + +export default defineConfig(({ mode }) => { + const env = loadEnv(mode, '.', ''); + return { + server: { + port: 3000, + host: '0.0.0.0', + }, + plugins: [react()], + define: { + 'process.env.API_KEY': JSON.stringify(env.GEMINI_API_KEY), + 'process.env.GEMINI_API_KEY': JSON.stringify(env.GEMINI_API_KEY) + }, + resolve: { + alias: { + '@': path.resolve(__dirname, '.'), + } + } + }; +});