diff --git a/src/server/transformers/anthropic/messages/index.test.ts b/src/server/transformers/anthropic/messages/index.test.ts index 4fe1088..6fdd37f 100644 --- a/src/server/transformers/anthropic/messages/index.test.ts +++ b/src/server/transformers/anthropic/messages/index.test.ts @@ -34,7 +34,7 @@ describe('anthropicMessagesTransformer protocol contract', () => { }); }); - it('parses native document blocks into canonical file parts', () => { + it('preserves native image and document blocks when parsing into canonical envelopes', () => { const result = anthropicMessagesTransformer.parseRequest({ model: 'claude-sonnet-4-5', max_tokens: 256, @@ -42,15 +42,23 @@ describe('anthropicMessagesTransformer protocol contract', () => { { role: 'user', content: [ - { type: 'text', text: 'summarize this pdf' }, + { type: 'text', text: 'inspect both attachments' }, + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'QUFBQQ==', + }, + }, { type: 'document', - title: 'brief.pdf', source: { type: 'base64', media_type: 'application/pdf', - data: 'JVBERi0xLjQK', + data: 'JVBERi0x', }, + title: 'brief.pdf', }, ], }, @@ -58,20 +66,23 @@ describe('anthropicMessagesTransformer protocol contract', () => { }); expect(result.error).toBeUndefined(); - expect(result.value?.messages).toEqual([ - { - role: 'user', - parts: [ - { type: 'text', text: 'summarize this pdf' }, - { - type: 'file', - filename: 'brief.pdf', - mimeType: 'application/pdf', - fileData: 'JVBERi0xLjQK', - }, - ], - }, - ]); + expect(result.value).toMatchObject({ + messages: [ + { + role: 'user', + parts: [ + { type: 'text', text: 'inspect both attachments' }, + { type: 'image', url: 'data:image/png;base64,QUFBQQ==' }, + { + type: 'file', + fileData: 'JVBERi0x', + filename: 'brief.pdf', + mimeType: 'application/pdf', + }, + ], + }, + ], + }); }); it('builds native messages requests from canonical envelopes', () => { diff --git a/src/server/transformers/gemini/generate-content/compatibility.ts b/src/server/transformers/gemini/generate-content/compatibility.ts index cf93027..ab1b0d9 100644 --- a/src/server/transformers/gemini/generate-content/compatibility.ts +++ b/src/server/transformers/gemini/generate-content/compatibility.ts @@ -1,3 +1,4 @@ +import { toOpenAiChatFileBlock } from '../../shared/inputFile.js'; import type { NormalizedFinalResponse } from '../../shared/normalized.js'; import { extractReasoningMetadataFromGeminiRequest } from './convert.js'; @@ -34,24 +35,58 @@ function buildDataUrl(part: GeminiRecord): string | null { return `data:${mimeType};base64,${data}`; } -function buildOpenAiFileBlock(input: { - fileData?: string; - fileUrl?: string; - mimeType?: string; -}): Record | null { - const fileData = asTrimmedString(input.fileData); - const fileUrl = asTrimmedString(input.fileUrl); - const mimeType = asTrimmedString(input.mimeType); - if (!fileData && !fileUrl) return null; +function buildInlineData( + part: GeminiRecord, +): { mimeType: string; data: string } | null { + const inlineData = isRecord(part.inlineData) ? part.inlineData : null; + if (!inlineData) return null; + const mimeType = asTrimmedString(inlineData.mime_type ?? inlineData.mimeType) || 'application/octet-stream'; + const data = asTrimmedString(inlineData.data); + if (!data) return null; + return { mimeType, data }; +} - const file: Record = {}; - if (fileData) file.file_data = fileData; - if (fileUrl && !fileData) file.file_url = fileUrl; - if (mimeType) file.mime_type = mimeType; - return { - type: 'file', - file, - }; +function buildFileDataSource( + part: GeminiRecord, +): { fileUri: string; mimeType: string | null } | null { + const fileData = isRecord(part.fileData) ? part.fileData : null; + if (!fileData) return null; + const fileUri = asTrimmedString(fileData.fileUri ?? fileData.file_uri); + if (!fileUri) return null; + const mimeType = asTrimmedString(fileData.mimeType ?? fileData.mime_type) || null; + return { fileUri, mimeType }; +} + +function toOpenAiBlockFromGeminiPart(part: GeminiRecord): Record | null { + const inlineData = buildInlineData(part); + if (inlineData) { + const normalizedMimeType = inlineData.mimeType.toLowerCase(); + if (normalizedMimeType.startsWith('image/')) { + return { + type: 'image_url', + image_url: { url: buildDataUrl(part)! }, + }; + } + + return toOpenAiChatFileBlock({ + fileData: inlineData.data, + mimeType: inlineData.mimeType, + }); + } + + const fileData = buildFileDataSource(part); + if (!fileData) return null; + if (fileData.mimeType?.toLowerCase().startsWith('image/')) { + return { + type: 'image_url', + image_url: { url: fileData.fileUri }, + }; + } + + return toOpenAiChatFileBlock({ + fileUrl: fileData.fileUri, + mimeType: fileData.mimeType, + }); } function toOpenAiContent(contentParts: GeminiRecord[]): string | Array> { @@ -65,41 +100,9 @@ function toOpenAiContent(contentParts: GeminiRecord[]): string | Array { }); }); + it('compatibility preserves inline document parts as OpenAI file blocks', () => { + const body = geminiGenerateContentTransformer.compatibility.buildOpenAiBodyFromGeminiRequest({ + modelName: 'gpt-5.4', + stream: false, + body: { + contents: [ + { + role: 'user', + parts: [ + { text: 'summarize this file' }, + { + inlineData: { + mimeType: 'application/pdf', + data: 'JVBERi0x', + }, + }, + ], + }, + ], + }, + }); + + expect(body).toEqual({ + model: 'gpt-5.4', + stream: false, + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'summarize this file' }, + { + type: 'file', + file: { + file_data: 'JVBERi0x', + mime_type: 'application/pdf', + }, + }, + ], + }, + ], + }); + }); + it('preserves native Gemini request fields through normalization', () => { const body = geminiGenerateContentTransformer.inbound.normalizeRequest({ contents: [ diff --git a/src/server/transformers/shared/chatFormatsCore.ts b/src/server/transformers/shared/chatFormatsCore.ts index 4eb5710..e3f0401 100644 --- a/src/server/transformers/shared/chatFormatsCore.ts +++ b/src/server/transformers/shared/chatFormatsCore.ts @@ -1,6 +1,7 @@ import { decodeAnthropicReasoningSignature, } from './reasoningTransport.js'; +import { toOpenAiChatFileBlock } from './inputFile.js'; import { consumeThinkTaggedText, createThinkTagParserState, @@ -86,6 +87,10 @@ function isNonEmptyString(value: unknown): value is string { return typeof value === 'string' && value.trim().length > 0; } +function asTrimmedString(value: unknown): string { + return typeof value === 'string' ? value.trim() : ''; +} + function pickFiniteNumber(value: unknown): number | undefined { return typeof value === 'number' && Number.isFinite(value) ? value : undefined; } @@ -548,6 +553,111 @@ function collectToolCallsFromResponsesPayload(payload: Record): return toolCalls; } +function formatAnthropicBase64DataUrl(mimeType: string, data: string): string { + return `data:${mimeType};base64,${data}`; +} + +function parseAnthropicBase64Source( + source: unknown, +): { mimeType: string; data: string } | null { + if (!isRecord(source)) return null; + const sourceType = asTrimmedString(source.type).toLowerCase(); + if (sourceType !== 'base64') return null; + const mimeType = ( + asTrimmedString(source.media_type) + || asTrimmedString(source.mime_type) + || asTrimmedString(source.mediaType) + || asTrimmedString(source.mimeType) + || 'application/octet-stream' + ); + const data = asTrimmedString(source.data); + if (!data) return null; + return { mimeType, data }; +} + +function parseAnthropicUrlSource( + source: unknown, +): { url: string; mimeType: string | null } | null { + if (!isRecord(source)) return null; + const sourceType = asTrimmedString(source.type).toLowerCase(); + if (sourceType !== 'url') return null; + const url = asTrimmedString(source.url); + if (!url) return null; + const mimeType = ( + asTrimmedString(source.media_type) + || asTrimmedString(source.mime_type) + || asTrimmedString(source.mediaType) + || asTrimmedString(source.mimeType) + || null + ); + return { url, mimeType }; +} + +function toOpenAiContentBlockFromClaudeBlock( + block: Record, +): Record | null { + const blockType = asTrimmedString(block.type).toLowerCase(); + + if (blockType === 'text') { + const text = asTrimmedString(block.text); + return text ? { type: 'text', text } : null; + } + + if (blockType !== 'image' && blockType !== 'document') { + return null; + } + + const title = asTrimmedString(block.title); + const base64Source = parseAnthropicBase64Source(block.source); + const urlSource = parseAnthropicUrlSource(block.source); + const mimeType = (base64Source?.mimeType || urlSource?.mimeType || '').toLowerCase(); + const treatAsImage = blockType === 'image' || mimeType.startsWith('image/'); + + if (treatAsImage) { + const imageUrl = base64Source + ? formatAnthropicBase64DataUrl(base64Source.mimeType, base64Source.data) + : (urlSource?.url || ''); + return imageUrl + ? { + type: 'image_url', + image_url: { url: imageUrl }, + } + : null; + } + + if (base64Source) { + return toOpenAiChatFileBlock({ + fileData: base64Source.data, + filename: title || undefined, + mimeType: base64Source.mimeType, + }); + } + + if (urlSource) { + return toOpenAiChatFileBlock({ + fileUrl: urlSource.url, + filename: title || undefined, + mimeType: urlSource.mimeType, + }); + } + + return null; +} + +function collapseOpenAiContentBlocks( + blocks: Array>, +): string | Array> | null { + if (blocks.length <= 0) return null; + const textOnly = blocks.every((block) => block.type === 'text' && typeof block.text === 'string'); + if (!textOnly) return blocks; + + const text = blocks + .map((block) => (typeof block.text === 'string' ? block.text : '')) + .join('\n\n') + .trim(); + return text || null; +} + function convertClaudeRequestToOpenAiBody(body: Record): { model: string; stream: boolean; @@ -609,10 +719,10 @@ function convertClaudeRequestToOpenAiBody(body: Record): { const toolCalls: Array> = []; const flushContentAsMessage = () => { - const contentValue = buildOpenAiMessageContent(contentBlocks); + const contentPayload = collapseOpenAiContentBlocks(contentBlocks); contentBlocks.length = 0; - if (contentValue === undefined || contentValue === '') return; - messages.push({ role: mappedRole, content: contentValue }); + if (contentPayload === null) return; + messages.push({ role: mappedRole, content: contentPayload }); }; for (const block of content) { @@ -648,22 +758,33 @@ function convertClaudeRequestToOpenAiBody(body: Record): { continue; } - const contentBlock = convertClaudeContentBlockToOpenAi(block); - if (contentBlock) contentBlocks.push(contentBlock); + const contentBlock = toOpenAiContentBlockFromClaudeBlock(block); + if (contentBlock) { + contentBlocks.push(contentBlock); + continue; + } + + const text = parseClaudeMessageContent(block); + if (text) { + contentBlocks.push({ + type: 'text', + text, + }); + } } - const contentValue = buildOpenAiMessageContent(contentBlocks); + const merged = collapseOpenAiContentBlocks(contentBlocks); if (toolCalls.length > 0) { const assistantMessage: Record = { role: 'assistant', tool_calls: toolCalls, }; - assistantMessage.content = contentValue ?? ''; + assistantMessage.content = merged || ''; messages.push(assistantMessage); - } else if (contentValue !== undefined && contentValue !== '') { + } else if (merged !== null) { messages.push({ role: mappedRole, - content: contentValue, + content: merged, }); } } diff --git a/src/web/pages/ModelTester.tsx b/src/web/pages/ModelTester.tsx index 086bf15..759e3c4 100644 --- a/src/web/pages/ModelTester.tsx +++ b/src/web/pages/ModelTester.tsx @@ -45,7 +45,12 @@ import { type TestTargetFormat, type TestChatPayload, } from './helpers/modelTesterSession.js'; -import { resolveConversationFileCapability } from './helpers/conversationFileCapabilities.js'; +import { + buildConversationFileAccept, + buildConversationFileHint, + isConversationUploadedFileSupported, + resolveConversationFileCapability, +} from './helpers/conversationFileCapabilities.js'; import ModernSelect from '../components/ModernSelect.js'; import { useAnimatedVisibility } from '../components/useAnimatedVisibility.js'; import { tr } from '../i18n.js'; @@ -77,7 +82,6 @@ type ConversationFileState = UploadState & { }; const POLL_INTERVAL_MS = 1200; -const CONVERSATION_FILE_ACCEPT = '.pdf,.txt,.md,.markdown,.json,image/*,audio/*'; const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); const createConversationFileLocalId = () => @@ -690,6 +694,19 @@ export default function ModelTester() { const restoredSessionRef = useRef>(null); const streamAbortRef = useRef(null); const streamStopRequestedRef = useRef(false); + const conversationFileCapability = useMemo( + () => resolveConversationFileCapability(inputs.protocol), + [inputs.protocol], + ); + const conversationFileSupported = conversationFileCapability.supported; + const conversationFileAccept = useMemo( + () => buildConversationFileAccept(conversationFileCapability), + [conversationFileCapability], + ); + const conversationFileHint = useMemo( + () => buildConversationFileHint(conversationFileCapability), + [conversationFileCapability], + ); const pushDebug = useCallback((level: DebugTimelineEntry['level'], text: string) => { const now = new Date().toISOString(); @@ -881,18 +898,35 @@ export default function ModelTester() { name: file.name, mimeType: file.type || 'application/octet-stream', dataUrl: await readFileAsDataUrl(file), - fileId: null, - status: 'pending' as const, - errorMessage: null, - }))); - setConversationFiles((prev) => [...prev, ...nextFiles]); - pushDebug('info', `已添加 ${nextFiles.length} 个会话附件。`); + fileId: null, + status: 'pending' as const, + errorMessage: null, + }))); + const acceptedFiles = nextFiles.filter((file) => isConversationUploadedFileSupported( + conversationFileCapability, + { filename: file.name, mimeType: file.mimeType }, + )); + const rejectedFiles = nextFiles.filter((file) => !isConversationUploadedFileSupported( + conversationFileCapability, + { filename: file.name, mimeType: file.mimeType }, + )); + + if (acceptedFiles.length > 0) { + setConversationFiles((prev) => [...prev, ...acceptedFiles]); + pushDebug('info', `已添加 ${acceptedFiles.length} 个会话附件。`); + } + + if (rejectedFiles.length > 0) { + const message = `当前协议不支持这些会话附件：${rejectedFiles.map((file) => file.name).join('、')}。${conversationFileHint}`; + setError(message); + pushDebug('warn', message); + } } catch (readError: any) { const message = readError?.message || '读取附件失败'; setError(message); pushDebug('error', message); } - }, [pushDebug]); + }, [conversationFileCapability, conversationFileHint, pushDebug]); const removeConversationFile = useCallback((localId: string) => { if (sending) return; @@ -966,6 +1000,20 @@ export default function ModelTester() { data: item.dataUrl, })), [conversationFiles]); + const ensureSupportedConversationFiles = useCallback((files: ConversationUploadedFile[]): boolean => { + const unsupported = files.filter((file) => !isConversationUploadedFileSupported(conversationFileCapability, file)); + if (unsupported.length <= 0) return true; + + const names = unsupported.map((file, index) => { + const filename = typeof file.filename === 'string' ? file.filename.trim() : ''; + return filename || `附件${index + 1}`; + }); + const message = `当前协议不支持这些会话附件：${names.join('、')}。${conversationFileHint}`; + setError(message); + pushDebug('warn', message); + return false; + }, [conversationFileCapability, conversationFileHint, pushDebug]); + const buildConversationMessagesWithSystem = useCallback((baseMessages: ChatMessage[]) => { if (!inputs.systemPrompt.trim()) return baseMessages; return [ @@ -1335,11 +1383,6 @@ export default function ModelTester() { () => filteredModels.map((item) => ({ value: item, label: item })), [filteredModels], ); - const conversationFileCapability = useMemo( - () => resolveConversationFileCapability(inputs.protocol), - [inputs.protocol], - ); - const conversationFileSupported = conversationFileCapability.supported; const canSend = useMemo(() => { if (sending || pendingJobId || !inputs.model) return false; if (inputs.mode !== 'conversation') { @@ -1762,6 +1805,9 @@ export default function ModelTester() { baseMessages: ChatMessage[], files: ConversationUploadedFile[] = [], ) => { + if (!ensureSupportedConversationFiles(files)) { + return; + } const userMessage = createConversationUserMessage(prompt, files); const loadingAssistant = createLoadingAssistantMessage(); const nextMessages = [...baseMessages, userMessage, loadingAssistant]; @@ -1780,7 +1826,7 @@ export default function ModelTester() { } await dispatchPayload(nextMessages, payload, { syncedCustomBody }); - }, [buildConversationProxyEnvelope, buildPayloadWithMessages, createConversationUserMessage, customRequestMode, dispatchPayload, dispatchProxyEnvelope, inputs.protocol, pushDebug]); + }, [buildConversationProxyEnvelope, buildPayloadWithMessages, createConversationUserMessage, customRequestMode, dispatchPayload, dispatchProxyEnvelope, ensureSupportedConversationFiles, inputs.protocol, pushDebug]); const sendModeRequest = useCallback(async () => { const envelope = buildModeProxyEnvelope(); @@ -1817,8 +1863,13 @@ export default function ModelTester() { if (!customRequestMode && conversationFileSupported && conversationFiles.length > 0) { setSending(true); try { + const draftFiles = inlineConversationFiles(); + if (!ensureSupportedConversationFiles(draftFiles)) { + setSending(false); + return; + } const uploadedFiles = conversationFileCapability.documentMode === 'inline_only' - ? inlineConversationFiles() + ? draftFiles : await uploadConversationFiles(); setInput(''); setConversationFiles([]); @@ -1857,7 +1908,7 @@ export default function ModelTester() { { stream: inputs.stream, jobMode: !inputs.stream }, ), ); - }, [canSend, conversationFileCapability, conversationFileSupported, conversationFiles.length, customRequestBody, customRequestMode, dispatchPayload, inlineConversationFiles, input, inputs.mode, messages, pushDebug, sendModeRequest, sendWithPrompt, uploadConversationFiles]); + }, [canSend, conversationFileCapability, conversationFileSupported, conversationFiles.length, customRequestBody, customRequestMode, dispatchPayload, ensureSupportedConversationFiles, inlineConversationFiles, input, inputs.mode, messages, pushDebug, sendModeRequest, sendWithPrompt, uploadConversationFiles]); const retryPending = useCallback(async () => { if (sending || pendingJobId || !pendingPayload) return; @@ -2793,7 +2844,7 @@ export default function ModelTester() { ref={conversationFileInputRef} type="file" multiple - accept={CONVERSATION_FILE_ACCEPT} + accept={conversationFileAccept} style={{ display: 'none' }} onChange={(event) => { void handleConversationFilesChange(event.target.files); @@ -2815,9 +2866,7 @@ export default function ModelTester() { ? '自定义请求模式不会自动上传这些附件；关闭自定义模式后可走标准 /v1/files 链路。' : !conversationFileSupported ? (conversationFileCapability.reason || '当前协议暂不支持会话附件注入。') - : conversationFileCapability.documentMode === 'inline_only' - ? '支持 PDF / TXT / Markdown / JSON / 图片 / 音频；发送时会以内联文档方式注入。' - : '支持 PDF / TXT / Markdown / JSON / 图片 / 音频；发送前会先上传到 /v1/files。'} + : conversationFileHint} {conversationFiles.length > 0 && ( diff --git a/src/web/pages/helpers/conversationFileCapabilities.test.ts b/src/web/pages/helpers/conversationFileCapabilities.test.ts index e5cd273..3f6d460 100644 --- a/src/web/pages/helpers/conversationFileCapabilities.test.ts +++ b/src/web/pages/helpers/conversationFileCapabilities.test.ts @@ -1,31 +1,79 @@ import { describe, expect, it } from 'vitest'; -import { resolveConversationFileCapability } from './conversationFileCapabilities.js'; +import { + buildConversationFileAccept, + buildConversationFileHint, + isConversationUploadedFileSupported, + resolveConversationFileCapability, +} from './conversationFileCapabilities.js'; describe('resolveConversationFileCapability', () => { it('keeps OpenAI and Responses on native file references', () => { expect(resolveConversationFileCapability('openai')).toEqual({ supported: true, + imageMode: 'native', + audioMode: 'native', documentMode: 'native', reason: '', }); expect(resolveConversationFileCapability('responses')).toEqual({ supported: true, + imageMode: 'native', + audioMode: 'native', documentMode: 'native', reason: '', }); }); - it('marks Claude and Gemini picker flows as inline-only document transports', () => { + it('maps Claude and Gemini media support separately', () => { expect(resolveConversationFileCapability('claude')).toEqual({ supported: true, + imageMode: 'native', + audioMode: 'unsupported', documentMode: 'inline_only', reason: '当前界面的会话附件会以内联文档方式发送。', }); expect(resolveConversationFileCapability('gemini')).toEqual({ supported: true, + imageMode: 'native', + audioMode: 'native', documentMode: 'inline_only', reason: '当前界面的会话附件会以内联文档方式发送。', }); }); + + it('builds the accept list from the supported file kinds', () => { + expect(buildConversationFileAccept(resolveConversationFileCapability('claude'))).toBe( + '.pdf,.txt,.md,.markdown,.json,image/*', + ); + expect(buildConversationFileAccept(resolveConversationFileCapability('gemini'))).toBe( + '.pdf,.txt,.md,.markdown,.json,image/*,audio/*', + ); + }); + + it('describes the actual per-kind transport behavior', () => { + expect(buildConversationFileHint(resolveConversationFileCapability('claude'))).toBe( + '支持 PDF / TXT / Markdown / JSON / 图片；文档会以内联数据注入，图片会按图片部件发送。', + ); + expect(buildConversationFileHint(resolveConversationFileCapability('gemini'))).toBe( + '支持 PDF / TXT / Markdown / JSON / 图片 / 音频；文档会以内联数据注入，图片会按图片部件发送，音频会按音频部件发送。', + ); + }); + + it('rejects unsupported file kinds before send', () => { + const claudeCapability = resolveConversationFileCapability('claude'); + + expect(isConversationUploadedFileSupported(claudeCapability, { + filename: 'paper.pdf', + mimeType: 'application/pdf', + })).toBe(true); + expect(isConversationUploadedFileSupported(claudeCapability, { + filename: 'chart.png', + mimeType: 'image/png', + })).toBe(true); + expect(isConversationUploadedFileSupported(claudeCapability, { + filename: 'voice.mp3', + mimeType: 'audio/mpeg', + })).toBe(false); + }); }); diff --git a/src/web/pages/helpers/conversationFileCapabilities.ts b/src/web/pages/helpers/conversationFileCapabilities.ts index e176de2..c17c201 100644 --- a/src/web/pages/helpers/conversationFileCapabilities.ts +++ b/src/web/pages/helpers/conversationFileCapabilities.ts @@ -1,25 +1,101 @@ import type { PlaygroundProtocol } from './modelTesterSession.js'; +export type ConversationFileTransportMode = 'native' | 'inline_only' | 'unsupported'; + export type ConversationFileCapability = { supported: boolean; - documentMode: 'native' | 'inline_only' | 'unsupported'; + imageMode: ConversationFileTransportMode; + audioMode: ConversationFileTransportMode; + documentMode: ConversationFileTransportMode; reason: string; }; +const DOCUMENT_ACCEPT_PARTS = ['.pdf', '.txt', '.md', '.markdown', '.json'] as const; +const DOCUMENT_MIME_TYPES = new Set([ + 'application/json', + 'application/pdf', + 'text/markdown', + 'text/plain', +]); +const DOCUMENT_EXTENSIONS = ['.json', '.md', '.markdown', '.pdf', '.txt'] as const; +const IMAGE_EXTENSIONS = ['.avif', '.bmp', '.gif', '.jpeg', '.jpg', '.png', '.svg', '.webp'] as const; +const AUDIO_EXTENSIONS = ['.aac', '.flac', '.m4a', '.mp3', '.ogg', '.wav', '.weba'] as const; + +type ConversationFileDescriptor = { + filename?: string | null; + mimeType?: string | null; +}; + +type ConversationFileKind = 'document' | 'image' | 'audio' | 'unknown'; + +const isSupportedMode = (mode: ConversationFileTransportMode): boolean => mode !== 'unsupported'; + +const normalizeName = (value: string | null | undefined): string => (value || '').trim().toLowerCase(); + +function detectKindFromFilename(filename: string): ConversationFileKind { + if (!filename) return 'unknown'; + if (DOCUMENT_EXTENSIONS.some((extension) => filename.endsWith(extension))) return 'document'; + if (IMAGE_EXTENSIONS.some((extension) => filename.endsWith(extension))) return 'image'; + if (AUDIO_EXTENSIONS.some((extension) => filename.endsWith(extension))) return 'audio'; + return 'unknown'; +} + +function resolveConversationFileKind(file: ConversationFileDescriptor): ConversationFileKind { + const mimeType = normalizeName(file.mimeType); + if (mimeType.startsWith('image/')) return 'image'; + if (mimeType.startsWith('audio/')) return 'audio'; + if (DOCUMENT_MIME_TYPES.has(mimeType)) return 'document'; + return detectKindFromFilename(normalizeName(file.filename)); +} + +function buildSupportedTypeLabel(capability: ConversationFileCapability): string { + const labels: string[] = []; + if (isSupportedMode(capability.documentMode)) labels.push('PDF / TXT / Markdown / JSON'); + if (isSupportedMode(capability.imageMode)) labels.push('图片'); + if (isSupportedMode(capability.audioMode)) labels.push('音频'); + return labels.join(' / '); +} + +function buildTransportNotes(capability: ConversationFileCapability): string[] { + const notes: string[] = []; + if (capability.documentMode === 'inline_only') notes.push('文档会以内联数据注入'); + if (capability.imageMode === 'native' && capability.documentMode === 'inline_only') { + notes.push('图片会按图片部件发送'); + } + if (capability.audioMode === 'native' && capability.documentMode === 'inline_only') { + notes.push('音频会按音频部件发送'); + } + return notes; +} + export function resolveConversationFileCapability( protocol: PlaygroundProtocol, ): ConversationFileCapability { if (protocol === 'openai' || protocol === 'responses') { return { supported: true, + imageMode: 'native', + audioMode: 'native', documentMode: 'native', reason: '', }; } - if (protocol === 'claude' || protocol === 'gemini') { + if (protocol === 'claude') { return { supported: true, + imageMode: 'native', + audioMode: 'unsupported', + documentMode: 'inline_only', + reason: '当前界面的会话附件会以内联文档方式发送。', + }; + } + + if (protocol === 'gemini') { + return { + supported: true, + imageMode: 'native', + audioMode: 'native', documentMode: 'inline_only', reason: '当前界面的会话附件会以内联文档方式发送。', }; @@ -27,7 +103,53 @@ export function resolveConversationFileCapability( return { supported: false, + imageMode: 'unsupported', + audioMode: 'unsupported', documentMode: 'unsupported', reason: '当前协议暂不支持会话附件。', }; } + +export function buildConversationFileAccept(capability: ConversationFileCapability): string { + const parts: string[] = []; + if (isSupportedMode(capability.documentMode)) parts.push(...DOCUMENT_ACCEPT_PARTS); + if (isSupportedMode(capability.imageMode)) parts.push('image/*'); + if (isSupportedMode(capability.audioMode)) parts.push('audio/*'); + return parts.join(','); +} + +export function buildConversationFileHint(capability: ConversationFileCapability): string { + if (!capability.supported) { + return capability.reason || '当前协议暂不支持会话附件。'; + } + + const typeLabel = buildSupportedTypeLabel(capability); + if (!typeLabel) { + return capability.reason || '当前协议暂不支持会话附件。'; + } + + if ( + capability.documentMode === 'native' + && capability.imageMode === 'native' + && capability.audioMode === 'native' + ) { + return `支持 ${typeLabel}；发送前会先上传到 /v1/files。`; + } + + const notes = buildTransportNotes(capability); + if (notes.length <= 0) { + return `支持 ${typeLabel}。`; + } + return `支持 ${typeLabel}；${notes.join('，')}。`; +} + +export function isConversationUploadedFileSupported( + capability: ConversationFileCapability, + file: ConversationFileDescriptor, +): boolean { + const kind = resolveConversationFileKind(file); + if (kind === 'document') return isSupportedMode(capability.documentMode); + if (kind === 'image') return isSupportedMode(capability.imageMode); + if (kind === 'audio') return isSupportedMode(capability.audioMode); + return false; +} diff --git a/src/web/pages/helpers/modelTesterSession.test.ts b/src/web/pages/helpers/modelTesterSession.test.ts index a706a4e..6fa358d 100644 --- a/src/web/pages/helpers/modelTesterSession.test.ts +++ b/src/web/pages/helpers/modelTesterSession.test.ts @@ -535,6 +535,54 @@ describe('modelTesterSession', () => { }); }); + it('builds Claude conversation payloads with image blocks for inline image attachments', () => { + const payload = buildApiPayload( + [{ + id: 'u1', + role: 'user', + content: 'describe this image', + createAt: 1, + parts: [ + { + type: 'input_file', + filename: 'chart.png', + mimeType: 'image/png', + data: 'data:image/png;base64,QUFBQQ==', + }, + ], + } as ChatMessage], + { + ...DEFAULT_INPUTS, + model: 'claude-opus-4-6', + protocol: 'claude', + }, + DEFAULT_PARAMETER_ENABLED, + ); + + expect(payload.jsonBody).toEqual({ + model: 'claude-opus-4-6', + stream: false, + max_tokens: 4096, + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'describe this image' }, + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'QUFBQQ==', + }, + }, + ], + }, + ], + temperature: 0.7, + }); + }); + it('builds Gemini conversation payloads with inlineData document parts for conversation files', () => { const payload = buildApiPayload( [{ diff --git a/src/web/pages/helpers/modelTesterSession.ts b/src/web/pages/helpers/modelTesterSession.ts index 1f7fe4a..5c7f2a1 100644 --- a/src/web/pages/helpers/modelTesterSession.ts +++ b/src/web/pages/helpers/modelTesterSession.ts @@ -355,11 +355,22 @@ const toClaudeContentPart = (part: ConversationContentPart): Record