From 9e1e0dbd87bd2f95bcbc05c4fae33ad8f511bdc6 Mon Sep 17 00:00:00 2001 From: yetone Date: Sat, 3 Oct 2026 10:24:03 +0800 Subject: [PATCH] =?UTF-8?q?gateway:=20an=20image=20model's=20picture=20rea?= =?UTF-8?q?ches=20a=20Gemini=20client=20(#620,=20tao-bai:=20antigravity/ge?= =?UTF-8?q?mini-3.1-flash-image=20=E8=BF=94=E5=9B=9E=20parts:=20[])=20?= =?UTF-8?q?=E2=80=94=20through=20/v1beta=20generateContent=20and=20streamG?= =?UTF-8?q?enerateContent=20the=20answer=20came=20back=20with=20no=20parts?= =?UTF-8?q?,=20though=20candidatesTokenCount=20(1567)=20said=20the=20image?= =?UTF-8?q?=20was=20made.=20The=20IR=20had=20no=20event=20for=20a=20pictur?= =?UTF-8?q?e=20the=20model=20made:=20codeAssistDecoder=20(Antigravity,=20G?= =?UTF-8?q?emini=20CLI=20and=20Factory's=20Gemini=20route)=20turned=20func?= =?UTF-8?q?tionCall,=20thought=20and=20text=20parts=20into=20events=20and?= =?UTF-8?q?=20dropped=20inlineData,=20and=20geminiParts=20/=20geminiEncode?= =?UTF-8?q?r=20had=20nothing=20to=20write=20it=20back=20as.=20A=20new=20KI?= =?UTF-8?q?mage=20event=20(Name=20the=20media=20type,=20Text=20the=20base6?= =?UTF-8?q?4)=20carries=20it;=20the=20collector=20keeps=20it=20as=20an=20I?= =?UTF-8?q?mage=20part,=20and=20the=20Gemini=20renderer=20and=20stream=20e?= =?UTF-8?q?ncoder=20send=20it=20as=20inlineData=20in=20its=20place=20among?= =?UTF-8?q?=20the=20text.=20A=20thought=20image=20(a=20draft,=20thought:?= =?UTF-8?q?=20true)=20stays=20with=20the=20reasoning=20and=20isn't=20sent,?= =?UTF-8?q?=20as=20before;=20thought=20text=20and=20signatures=20are=20unt?= =?UTF-8?q?ouched.=20Chat,=20Responses=20and=20Anthropic=20clients=20still?= =?UTF-8?q?=20don't=20get=20the=20picture=20(their=20renderers=20skip=20Im?= =?UTF-8?q?age=20parts).=20TestGeminiImageModelReturnsItsImage=20(a=20fake?= =?UTF-8?q?=20Factory=20Gemini=20upstream,=20generateContent)=20and=20Test?= =?UTF-8?q?CodeAssistStreamsImages=20(Code=20Assist=20chunks=20through=20t?= =?UTF-8?q?he=20Gemini=20stream=20encoder)=20fail=20without=20it=20with=20?= =?UTF-8?q?parts:=20[].=20Not=20tried=20against=20a=20real=20Antigravity?= =?UTF-8?q?=20account;=20the=20image=20part's=20thoughtSignature=20isn't?= =?UTF-8?q?=20returned=20to=20the=20client,=20so=20multi-turn=20image=20ed?= =?UTF-8?q?iting=20on=20Gemini=203=20image=20models=20may=20still=20need?= =?UTF-8?q?=20it.=20go=20vet=20./internal/...,=20go=20build=20-tags=20nogu?= =?UTF-8?q?i=20(darwin,=20GOOS=3Dlinux),=20GOOS=3Dwindows=20go=20build,=20?= =?UTF-8?q?go=20test=20-tags=20nogui=20-count=3D1=20./...=20under=20a=20te?= =?UTF-8?q?mp=20HOME=20pass.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/gateway/codeassist.go | 5 ++ internal/gateway/gemini.go | 7 ++ internal/gateway/gemini_image_test.go | 96 +++++++++++++++++++++++++++ internal/gateway/ir.go | 4 ++ 4 files changed, 112 insertions(+) create mode 100644 internal/gateway/gemini_image_test.go diff --git a/internal/gateway/codeassist.go b/internal/gateway/codeassist.go index 80dba831..815c28fb 100644 --- a/internal/gateway/codeassist.go +++ b/internal/gateway/codeassist.go @@ -524,6 +524,11 @@ func (d *codeAssistDecoder) decode(data string, emit func(Event)) error { if p.Signature != "" { emit(Event{Kind: KSig, Text: p.Signature}) } + case p.InlineData != nil && p.InlineData.Data != "": + // a picture an image model drew (gemini-*-image): its + // thought images, drafts on the way, stay with the + // reasoning above (#620) + emit(Event{Kind: KImage, Name: p.InlineData.MimeType, Text: p.InlineData.Data}) case p.Text != "": emit(Event{Kind: KText, Text: p.Text}) } diff --git a/internal/gateway/gemini.go b/internal/gateway/gemini.go index 5a69a7c1..f5e3936f 100644 --- a/internal/gateway/gemini.go +++ b/internal/gateway/gemini.go @@ -403,6 +403,10 @@ func geminiParts(parts []Part) []map[string]any { if p.Text != "" { out = append(out, map[string]any{"text": p.Text, "thought": true}) } + case Image: + if p.Data != "" { + out = append(out, map[string]any{"inlineData": map[string]any{"mimeType": p.MediaType, "data": p.Data}}) + } case ToolCall: id := p.ID if id == "" { @@ -487,6 +491,9 @@ func (e *geminiEncoder) event(ev Event) { case KThink: e.flushTool() e.chunk(geminiParts([]Part{{Kind: Thinking, Text: ev.Text}}), "", nil) + case KImage: + e.flushTool() + e.chunk(geminiParts([]Part{{Kind: Image, MediaType: ev.Name, Data: ev.Text}}), "", nil) case KToolStart: e.flushTool() e.tool = &Part{Kind: ToolCall, ID: ev.ID, Name: ev.Name} diff --git a/internal/gateway/gemini_image_test.go b/internal/gateway/gemini_image_test.go new file mode 100644 index 00000000..e9bc3944 --- /dev/null +++ b/internal/gateway/gemini_image_test.go @@ -0,0 +1,96 @@ +package gateway + +import ( + "encoding/json" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/yetone/magpie/internal/provider" +) + +// An image model's picture reaches a Gemini client's generateContent: the +// upstream's inlineData part comes back, and a thought image (a draft) +// stays out (#620). +func TestGeminiImageModelReturnsItsImage(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) + t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache")) + t.Setenv("XDG_DATA_HOME", filepath.Join(home, ".local", "share")) + + f := &fake{t: t, reply: `data: {"candidates":[{"content":{"parts":[` + + `{"thought":true,"inlineData":{"mimeType":"image/png","data":"ZHJhZnQ="}},` + + `{"text":"Here it is."},` + + `{"inlineData":{"mimeType":"image/png","data":"aGVsbG8="},"thoughtSignature":"c2ln"}` + + `]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":5,"candidatesTokenCount":1567}}` + "\n\n"} + up := httptest.NewServer(f) + t.Cleanup(up.Close) + t.Cleanup(provider.FactoryBaseForTest(up.URL, up.URL+"/eu")) + + auth, _ := json.Marshal(map[string]any{ + "accessToken": "tok", "refreshToken": "r", + "expiresAt": time.Now().Add(time.Hour).UnixMilli(), + "orgId": "org_D", "activeOrganizationId": "fac_D", "email": "d@example.com", + }) + dir := filepath.Dir(provider.Path()) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal([]map[string]any{{ + "agent": "factory", "user": "d@example.com", "plan": "pro", "on": true, "auth": json.RawMessage(auth), + }}) + if err := os.WriteFile(filepath.Join(dir, "logins.json"), raw, 0o600); err != nil { + t.Fatal(err) + } + + const ask = `{"contents":[{"role":"user","parts":[{"text":"draw a cat"}]}]}` + check := func(kind, body string) { + t.Helper() + if !strings.Contains(body, `"inlineData":{"data":"aGVsbG8=","mimeType":"image/png"}`) { + t.Errorf("%s: image dropped: %s", kind, body) + } + if strings.Contains(body, "ZHJhZnQ=") { + t.Errorf("%s: thought image sent: %s", kind, body) + } + if i, j := strings.Index(body, "Here it is."), strings.Index(body, "aGVsbG8="); i < 0 || j < i { + t.Errorf("%s: text and image out of order: %s", kind, body) + } + } + + // Factory's Gemini route decodes with Antigravity's decoder. + code, body := post(t, "/v1beta/models/factory/gemini-3.1-pro-preview:generateContent", ask) + if code != 200 || strings.HasPrefix(body, "data:") { + t.Fatalf("json %d %s", code, body) + } + if f.path != "/api/llm/g/v1/generate" { + t.Fatalf("upstream %s", f.path) + } + check("generateContent", body) +} + +// Antigravity and Gemini CLI wrap each chunk in {response}; streamed to a +// Gemini client, the picture goes out as an inlineData chunk. +func TestCodeAssistStreamsImages(t *testing.T) { + rec := httptest.NewRecorder() + enc := &geminiEncoder{w: newSSEWriter(rec), model: "antigravity/gemini-3.1-flash-image"} + d := &codeAssistDecoder{} + for _, line := range []string{ + `{"response":{"candidates":[{"content":{"role":"model","parts":[{"thought":true,"inlineData":{"mimeType":"image/png","data":"ZHJhZnQ="}}]}}]}}`, + `{"response":{"candidates":[{"content":{"role":"model","parts":[{"inlineData":{"mimeType":"image/jpeg","data":"aGk="}}]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":5,"candidatesTokenCount":1567}}}`, + } { + d.decode(line, enc.event) + } + enc.finish() + body := rec.Body.String() + if !strings.Contains(body, `"inlineData":{"data":"aGk=","mimeType":"image/jpeg"}`) { + t.Errorf("image dropped: %s", body) + } + if strings.Contains(body, "ZHJhZnQ=") { + t.Errorf("thought image sent: %s", body) + } +} diff --git a/internal/gateway/ir.go b/internal/gateway/ir.go index 9da0be3a..7f3d5ccf 100644 --- a/internal/gateway/ir.go +++ b/internal/gateway/ir.go @@ -159,6 +159,7 @@ const ( KUsage // Usage KError // Text KSearch // Text (the query), Hits: a web search run for the model + KImage // Name (media type), Text (base64): an image the model made ) // Event is one thing a streaming reply said. @@ -308,6 +309,9 @@ func (c *collector) add(ev Event) { case KSearch: c.closeTool() c.res.Parts = append(c.res.Parts, Part{Kind: Search, Text: ev.Text, Hits: ev.Hits}) + case KImage: + c.closeTool() + c.res.Parts = append(c.res.Parts, Part{Kind: Image, MediaType: ev.Name, Data: ev.Text}) } }