import type { ProxyResourceOwner } from '../middleware/auth.js'; import { getProxyFileByPublicIdForOwner, LOCAL_PROXY_FILE_ID_PREFIX } from './proxyFileStore.js'; import { ensureBase64DataUrl } from '../transformers/shared/inputFile.js'; import { summarizeConversationFileInputsInOpenAiBody } from '../proxy-core/capabilities/conversationFileCapabilities.js'; function isRecord(value: unknown): value is Record { return !!value && typeof value === 'object' && !Array.isArray(value); } function asTrimmedString(value: unknown): string { return typeof value === 'string' ? value.trim() : ''; } function cloneJsonValue(value: T): T { if (Array.isArray(value)) { return value.map((item) => cloneJsonValue(item)) as T; } if (isRecord(value)) { return Object.fromEntries( Object.entries(value).map(([key, item]) => [key, cloneJsonValue(item)]), ) as T; } return value; } function inferMimeTypeFromFilename(filename: string): string { const normalized = filename.toLowerCase(); if (normalized.endsWith('.pdf')) return 'application/pdf'; if (normalized.endsWith('.txt')) return 'text/plain'; if (normalized.endsWith('.md') || normalized.endsWith('.markdown')) return 'text/markdown'; if (normalized.endsWith('.json')) return 'application/json'; if (normalized.endsWith('.png')) return 'image/png'; if (normalized.endsWith('.jpg') || normalized.endsWith('.jpeg')) return 'image/jpeg'; if (normalized.endsWith('.gif')) return 'image/gif'; if (normalized.endsWith('.webp')) return 'image/webp'; if (normalized.endsWith('.wav')) return 'audio/wav'; if (normalized.endsWith('.mp3')) return 'audio/mpeg'; if (normalized.endsWith('.m4a')) return 'audio/mp4'; if (normalized.endsWith('.ogg')) return 'audio/ogg'; if (normalized.endsWith('.webm')) return 'audio/webm'; return 'application/octet-stream'; } function isSupportedMimeType(mimeType: string): boolean { return mimeType === 'application/pdf' || mimeType === 'text/plain' || mimeType === 'text/markdown' || mimeType === 'application/json' || mimeType.startsWith('image/') || mimeType.startsWith('audio/'); } function audioFormatFromMimeType(mimeType: string, filename: string): string { const normalized = mimeType.toLowerCase(); if (normalized === 'audio/mpeg') return 'mp3'; if (normalized === 'audio/wav' || normalized === 'audio/x-wav') return 'wav'; if (normalized === 'audio/mp4' || normalized === 'audio/x-m4a') return 'm4a'; if (normalized === 'audio/ogg') return 'ogg'; if (normalized === 'audio/webm') return 'webm'; const extension = filename.split('.').pop()?.trim().toLowerCase(); return extension || 'mp3'; } function isLocalProxyFileId(fileId: string): boolean { return fileId.startsWith(LOCAL_PROXY_FILE_ID_PREFIX); } type InputFileLike = { fileId?: string; filename?: string; fileData?: string; mimeType?: string; }; export class ProxyInputFileResolutionError extends Error { statusCode: number; payload: unknown; constructor(statusCode: number, message: string, type = 'invalid_request_error') { super(message); this.name = 'ProxyInputFileResolutionError'; this.statusCode = statusCode; this.payload = { error: { message, type, }, }; } } function normalizeInputFileLike(item: Record): InputFileLike | null { const type = asTrimmedString(item.type).toLowerCase(); if (type !== 'file' && type !== 'input_file') return null; const source = type === 'file' && isRecord(item.file) ? item.file : item; const fileId = asTrimmedString(source.file_id ?? item.file_id); const filename = asTrimmedString(source.filename ?? item.filename); const fileData = asTrimmedString(source.file_data ?? item.file_data); const mimeType = asTrimmedString(source.mime_type ?? source.mimeType ?? item.mime_type ?? item.mimeType); if (!fileId && !fileData) return null; return { ...(fileId ? { fileId } : {}), ...(filename ? { filename } : {}), ...(fileData ? { fileData } : {}), ...(mimeType ? { mimeType } : {}), }; } async function resolveInputFileLike(fileLike: InputFileLike, owner: ProxyResourceOwner): Promise<{ fileId?: string; filename: string; fileData: string; mimeType: string; }> { if (fileLike.fileData) { const filename = fileLike.filename || 'upload.bin'; const mimeType = fileLike.mimeType || inferMimeTypeFromFilename(filename); if (!isSupportedMimeType(mimeType)) { throw new ProxyInputFileResolutionError(400, `unsupported file mime type: ${mimeType}`); } return { ...(fileLike.fileId ? { fileId: fileLike.fileId } : {}), filename, fileData: fileLike.fileData, mimeType, }; } if (fileLike.fileId) { const stored = await getProxyFileByPublicIdForOwner(fileLike.fileId, owner); if (!stored) { throw new ProxyInputFileResolutionError(404, `file not found: ${fileLike.fileId}`, 'not_found_error'); } const mimeType = fileLike.mimeType || stored.mimeType || inferMimeTypeFromFilename(stored.filename); if (!isSupportedMimeType(mimeType)) { throw new ProxyInputFileResolutionError(400, `unsupported file mime type: ${mimeType}`); } return { fileId: fileLike.fileId, filename: fileLike.filename || stored.filename, fileData: stored.contentBase64, mimeType, }; } const filename = fileLike.filename || 'upload.bin'; const mimeType = fileLike.mimeType || inferMimeTypeFromFilename(filename); if (!fileLike.fileData) { throw new ProxyInputFileResolutionError(400, `file_data is required for inline file block: ${filename}`); } if (!isSupportedMimeType(mimeType)) { throw new ProxyInputFileResolutionError(400, `unsupported file mime type: ${mimeType}`); } return { filename, fileData: fileLike.fileData, mimeType, }; } function shouldResolveInlineFileLike(fileLike: InputFileLike): boolean { if (fileLike.fileData) return true; if (!fileLike.fileId) return false; return isLocalProxyFileId(fileLike.fileId); } function toOpenAiResolvedBlock(file: { fileId?: string; filename: string; fileData: string; mimeType: string }): Record { if (file.mimeType.startsWith('image/')) { return { type: 'image_url', image_url: `data:${file.mimeType};base64,${file.fileData}`, }; } if (file.mimeType.startsWith('audio/')) { return { type: 'input_audio', input_audio: { data: file.fileData, format: audioFormatFromMimeType(file.mimeType, file.filename), }, }; } return { type: 'file', file: { ...(file.fileId ? { file_id: file.fileId } : {}), filename: file.filename, file_data: file.fileData, mime_type: file.mimeType, }, }; } function toResponsesResolvedBlock(file: { fileId?: string; filename: string; fileData: string; mimeType: string }): Record { if (file.mimeType.startsWith('image/')) { return { type: 'input_image', image_url: `data:${file.mimeType};base64,${file.fileData}`, }; } if (file.mimeType.startsWith('audio/')) { return { type: 'input_audio', input_audio: { data: file.fileData, format: audioFormatFromMimeType(file.mimeType, file.filename), }, }; } return { type: 'input_file', ...(!file.fileData && file.fileId ? { file_id: file.fileId } : {}), filename: file.filename, file_data: ensureBase64DataUrl(file.fileData, file.mimeType), }; } async function resolveOpenAiMessageContent(content: unknown, owner: ProxyResourceOwner): Promise { if (!Array.isArray(content)) return content; return Promise.all(content.map(async (item) => { if (!isRecord(item)) return cloneJsonValue(item); const fileLike = normalizeInputFileLike(item); if (!fileLike || !shouldResolveInlineFileLike(fileLike)) return cloneJsonValue(item); return toOpenAiResolvedBlock(await resolveInputFileLike(fileLike, owner)); })); } async function resolveResponsesMessageContent(content: unknown, owner: ProxyResourceOwner): Promise { if (!Array.isArray(content)) return content; return Promise.all(content.map(async (item) => { if (!isRecord(item)) return cloneJsonValue(item); const fileLike = normalizeInputFileLike(item); if (!fileLike || !shouldResolveInlineFileLike(fileLike)) return cloneJsonValue(item); return toResponsesResolvedBlock(await resolveInputFileLike(fileLike, owner)); })); } export async function resolveOpenAiBodyInputFiles( body: Record, owner: ProxyResourceOwner, ): Promise> { const next = cloneJsonValue(body); if (!Array.isArray(next.messages)) return next; next.messages = await Promise.all(next.messages.map(async (message) => { if (!isRecord(message)) return cloneJsonValue(message); if (Array.isArray(message.content)) { return { ...message, content: await resolveOpenAiMessageContent(message.content, owner), }; } if (isRecord(message.content)) { const fileLike = normalizeInputFileLike(message.content); if (!fileLike) return cloneJsonValue(message); return { ...message, content: [toOpenAiResolvedBlock(await resolveInputFileLike(fileLike, owner))], }; } return cloneJsonValue(message); })); return next; } export async function resolveResponsesBodyInputFiles( body: Record, owner: ProxyResourceOwner, ): Promise> { const next = cloneJsonValue(body); const resolveResponsesInputItem = async (item: unknown): Promise => { if (!isRecord(item)) return cloneJsonValue(item); const fileLike = normalizeInputFileLike(item); if (fileLike && shouldResolveInlineFileLike(fileLike)) { return toResponsesResolvedBlock(await resolveInputFileLike(fileLike, owner)); } if (Array.isArray(item.content)) { return { ...item, content: await resolveResponsesMessageContent(item.content, owner), }; } if (isRecord(item.content)) { const nestedFileLike = normalizeInputFileLike(item.content); if (!nestedFileLike) return cloneJsonValue(item); return { ...item, content: [toResponsesResolvedBlock(await resolveInputFileLike(nestedFileLike, owner))], }; } return cloneJsonValue(item); }; if (Array.isArray(next.input)) { next.input = await Promise.all(next.input.map((item) => resolveResponsesInputItem(item))); return next; } if (isRecord(next.input)) { next.input = await resolveResponsesInputItem(next.input); } return next; } export function hasNonImageFileInputInOpenAiBody(body: Record): boolean { return summarizeConversationFileInputsInOpenAiBody(body).hasDocument; }