From 9758a7785e3107924396cd6b34dfaa5ac815ab94 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Thu, 19 Feb 2026 15:42:59 +0800 Subject: [PATCH] Use OpenAI Image Edit API for generation (#50) * feat(ai): use OpenAI Image Edit API instead of Responses API Replaces the deprecated/unofficial Responses API usage for image generation with the standard `client.images.edit` API. This change supports multi-image inputs and follows the user's provided example for `gpt-image-1.5` model compatibility. - Updates `generateImageOpenAI` in `src/service/ai.ts` to use `client.images.edit`. - Handles conversion of text prompts and inline/reference images to the required format using `toFile`. - Updates `src/service/ai.test.ts` to mock the new API call and verify correct behavior. * feat(ai): use OpenAI Image Edit API instead of Responses API Replaces the deprecated/unofficial Responses API usage for image generation with the standard `client.images.edit` API. This change supports multi-image inputs and follows the user's provided example for `gpt-image-1.5` model compatibility. - Updates `generateImageOpenAI` in `src/service/ai.ts` to use `client.images.edit`. - Removes `response_format` parameter as it's not supported/needed for GPT image models (which default to base64). - Handles conversion of text prompts and inline/reference images to the required format using `toFile`. - Updates `src/service/ai.test.ts` to mock the new API call and verify correct behavior. --------- Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com> --- src/service/ai.test.ts | 43 ++++++++++++++++++++--------------- src/service/ai.ts | 51 ++++++++++++++++++------------------------ 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/src/service/ai.test.ts b/src/service/ai.test.ts index 285522a..71e1366 100644 --- a/src/service/ai.test.ts +++ b/src/service/ai.test.ts @@ -18,9 +18,7 @@ const mockOpenAIClient = { }, images: { generate: mock(), - }, - responses: { - create: mock(), + edit: mock(), }, } @@ -33,11 +31,14 @@ mock.module('openai', () => { } }, // Mock helper function + toFile: mock((data: unknown) => Promise.resolve({ data })), + // eslint-disable-next-line @typescript-eslint/no-explicit-any zodResponseFormat: (schema: any) => ({ type: 'json_schema', json_schema: { schema } }), } }) mock.module('openai/helpers/zod', () => ({ + // eslint-disable-next-line @typescript-eslint/no-explicit-any zodResponseFormat: (schema: any) => ({ type: 'json_schema', json_schema: { schema } }), })) @@ -69,7 +70,7 @@ describe('UnifiedAIService', () => { mockSettingService.get.mockReset() mockOpenAIClient.chat.completions.create.mockReset() mockOpenAIClient.images.generate.mockReset() - mockOpenAIClient.responses.create.mockReset() + mockOpenAIClient.images.edit.mockReset() mockGoogleGenAIClient.models.generateContent.mockReset() }) @@ -118,32 +119,38 @@ describe('UnifiedAIService', () => { expect(mockOpenAIClient.chat.completions.create).not.toHaveBeenCalled() }) - it('should use OpenAI Responses API for image generation when configured', async () => { + it('should use OpenAI Image Edit API for image generation when configured', async () => { mockSettingService.get.mockImplementation(async (key: string) => { if (key === 'openai_api_key') return 'test_key' - if (key === 'step_generation_model') return 'openai:gpt-5' + if (key === 'step_generation_model') return 'openai:gpt-image-1.5' return null }) - // Setup OpenAI response for image generation - mockOpenAIClient.responses.create.mockResolvedValue({ - output: [ + // Setup OpenAI response for image edit + mockOpenAIClient.images.edit.mockResolvedValue({ + data: [ { - type: 'image_generation_call', - result: 'base64image', + b64_json: 'base64image', }, ], }) - const parts = [{ text: 'test prompt' }] + // Create a dummy file part + const parts = [ + { text: 'test prompt' }, + { inlineData: { mimeType: 'image/png', data: 'imagedata' } }, + ] + const result = await aiService.generateImage(parts, [], 'step_generation_model') expect(result).toBe('data:image/png;base64,base64image') - expect(mockOpenAIClient.responses.create).toHaveBeenCalled() - const callArgs = mockOpenAIClient.responses.create.mock.calls[0][0] - expect(callArgs.model).toBe('gpt-5') - expect(callArgs.tools).toEqual([{ type: 'image_generation' }]) - expect(callArgs.input).toEqual([{ type: 'input_text', text: 'test prompt' }]) + expect(mockOpenAIClient.images.edit).toHaveBeenCalled() + const callArgs = mockOpenAIClient.images.edit.mock.calls[0][0] + expect(callArgs.model).toBe('gpt-image-1.5') + expect(callArgs.prompt).toBe('test prompt') + // We expect image to be an array of files (which are mocked objects from toFile) + expect(Array.isArray(callArgs.image)).toBe(true) + expect(callArgs.image.length).toBe(1) }) it('should use Google for image generation when configured', async () => { @@ -175,6 +182,6 @@ describe('UnifiedAIService', () => { expect(result).toBe('data:image/png;base64,base64image') expect(mockGoogleGenAIClient.models.generateContent).toHaveBeenCalled() - expect(mockOpenAIClient.responses.create).not.toHaveBeenCalled() + expect(mockOpenAIClient.images.edit).not.toHaveBeenCalled() }) }) diff --git a/src/service/ai.ts b/src/service/ai.ts index b60ada8..6adff6e 100644 --- a/src/service/ai.ts +++ b/src/service/ai.ts @@ -1,5 +1,5 @@ import { GoogleGenAI } from '@google/genai' -import OpenAI from 'openai' +import OpenAI, { toFile } from 'openai' import { zodResponseFormat } from 'openai/helpers/zod' import { z } from 'zod' import { settingService } from './setting' @@ -176,6 +176,7 @@ export class UnifiedAIService implements AIService { const result = await client.models.generateContent({ model: model, + // eslint-disable-next-line @typescript-eslint/no-explicit-any contents: { parts: parts as any[] }, // Type assertion due to slight mismatch or library version config: config, }) @@ -236,48 +237,39 @@ export class UnifiedAIService implements AIService { private async generateImageOpenAI(model: string, parts: AIPart[]): Promise { const client = await this.getOpenAIClient() - - // Construct input for Responses API + const promptParts: string[] = [] // eslint-disable-next-line @typescript-eslint/no-explicit-any - const input: any[] = [] + const images: any[] = [] for (const part of parts) { if (part.text) { - input.push({ - type: 'input_text', - text: part.text, - }) + promptParts.push(part.text) } else if (part.inlineData) { - input.push({ - type: 'input_image', - image_url: `data:${part.inlineData.mimeType};base64,${part.inlineData.data}`, + const buffer = Buffer.from(part.inlineData.data, 'base64') + const file = await toFile(buffer, 'image.png', { + type: part.inlineData.mimeType, }) + images.push(file) } } - // Use Responses API with image_generation tool - // @ts-ignore - response API might not be fully typed in the SDK version installed yet - const response = await client.responses.create({ + const prompt = promptParts.join('\n') + + // @ts-ignore - The OpenAI type definitions might not be fully updated for this specific usage + const response = await client.images.edit({ model: model, - input: input, - tools: [{ type: 'image_generation' }], + image: images, // Pass array of images as requested + prompt: prompt, }) - // Parse output to find image generation result - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const imageData = response.output - .filter((output: any) => output.type === 'image_generation_call') - .map((output: any) => output.result) - - if (imageData.length > 0) { - const base64 = imageData[0] - // Assuming png by default or we could inspect headers, but base64 usually lacks mime prefix from this API - // The previous code appended `data:image/png;base64,` manually if needed. - // DALL-E usually returns just the base64 data. - return `data:image/png;base64,${base64}` + if (response.data && response.data.length > 0) { + const b64Json = response.data[0].b64_json + if (b64Json) { + return `data:image/png;base64,${b64Json}` + } } - throw new Error('No image generated from OpenAI Responses API') + throw new Error('No image generated from OpenAI Image Edit API') } private async generateImageGoogle(model: string, parts: AIPart[]): Promise { @@ -294,6 +286,7 @@ export class UnifiedAIService implements AIService { const result = await client.models.generateContent({ model: model, + // eslint-disable-next-line @typescript-eslint/no-explicit-any contents: { parts: parts as any[] }, // Type assertion })