From 183860587f25486649cee746b2b7edd89d731653 Mon Sep 17 00:00:00 2001
From: 7418 <7418@openclaw.ai>
Date: Mon, 9 Feb 2026 20:51:23 +0800
Subject: [PATCH] fix: send images as multimodal vision content instead of file
 path references

Images were being saved to disk and referenced by text path in the prompt,
causing Claude to hallucinate about image contents since it never actually
saw the image data. Now images are sent as base64 content blocks via the
SDK's multimodal SDKUserMessage, enabling Claude's vision capability.
Non-image files continue to be saved to disk for Read tool access.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 package-lock.json        |  4 +--
 package.json             |  2 +-
 src/lib/claude-client.ts | 71 +++++++++++++++++++++++++++++++---------
 3 files changed, 58 insertions(+), 19 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 6f3e7c8..d00936c 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "codepilot",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "codepilot",
-      "version": "0.6.2",
+      "version": "0.6.3",
       "dependencies": {
         "@anthropic-ai/claude-agent-sdk": "^0.2.33",
         "@hugeicons/core-free-icons": "^3.1.1",
diff --git a/package.json b/package.json
index fa0000a..7046845 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "codepilot",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "private": true,
   "main": "dist-electron/main.js",
   "scripts": {
diff --git a/src/lib/claude-client.ts b/src/lib/claude-client.ts
index 24dec08..3cbd06a 100644
--- a/src/lib/claude-client.ts
+++ b/src/lib/claude-client.ts
@@ -366,24 +366,63 @@ export function streamClaude(options: ClaudeStreamOptions): ReadableStream<strin
           }
         };
 
-        // Build the prompt: save all file attachments to disk and reference them in the prompt.
-        // Claude Code's built-in Read tool can read images (converts to base64 internally),
-        // PDFs, and text files — this is more reliable than constructing SDK multimodal messages.
-        let finalPrompt: string = prompt;
+        // Build the prompt with file attachments.
+        // Images → sent as multimodal base64 content blocks (vision).
+        // Non-image files → saved to disk and referenced via Read tool.
+        let finalPrompt: string | AsyncIterable<SDKUserMessage> = prompt;
 
         if (files && files.length > 0) {
-          const workDir = workingDirectory || process.cwd();
-          const savedPaths = saveUploadedFiles(files, workDir);
-          const fileReferences = savedPaths
-            .map((p, i) => {
-              const f = files[i];
-              if (isImageFile(f.type)) {
-                return `[User attached image: ${p} (${f.name})]`;
-              }
-              return `[User attached file: ${p} (${f.name})]`;
-            })
-            .join('\n');
-          finalPrompt = `${fileReferences}\n\nPlease read the attached file(s) above using your Read tool, then respond to the user's message:\n\n${prompt}`;
+          const imageFiles = files.filter(f => isImageFile(f.type));
+          const nonImageFiles = files.filter(f => !isImageFile(f.type));
+
+          // Save non-image files to disk for Read tool access
+          let textPrompt = prompt;
+          if (nonImageFiles.length > 0) {
+            const workDir = workingDirectory || process.cwd();
+            const savedPaths = saveUploadedFiles(nonImageFiles, workDir);
+            const fileReferences = savedPaths
+              .map((p, i) => `[User attached file: ${p} (${nonImageFiles[i].name})]`)
+              .join('\n');
+            textPrompt = `${fileReferences}\n\nPlease read the attached file(s) above using your Read tool, then respond to the user's message:\n\n${prompt}`;
+          }
+
+          // If there are images, build a multimodal SDKUserMessage
+          if (imageFiles.length > 0) {
+            const contentBlocks: Array<
+              | { type: 'image'; source: { type: 'base64'; media_type: string; data: string } }
+              | { type: 'text'; text: string }
+            > = [];
+
+            for (const img of imageFiles) {
+              contentBlocks.push({
+                type: 'image',
+                source: {
+                  type: 'base64',
+                  media_type: img.type || 'image/png',
+                  data: img.data,
+                },
+              });
+            }
+
+            contentBlocks.push({ type: 'text', text: textPrompt });
+
+            const userMessage: SDKUserMessage = {
+              type: 'user',
+              message: {
+                role: 'user',
+                content: contentBlocks,
+              },
+              parent_tool_use_id: null,
+              session_id: sdkSessionId || '',
+            };
+
+            // Create a single-message async iterable
+            finalPrompt = (async function* () {
+              yield userMessage;
+            })();
+          } else {
+            finalPrompt = textPrompt;
+          }
         }
 
         const conversation = query({