From d00d9ffe6fcbe746586e7373fb5561d694ee8776 Mon Sep 17 00:00:00 2001 From: yetone Date: Tue, 29 Sep 2026 01:20:21 +0800 Subject: [PATCH] =?UTF-8?q?gateway:=20a=20model=20that=20can't=20see=20ima?= =?UTF-8?q?ges=20is=20given=20what=20a=20model=20that=20can=20says=20of=20?= =?UTF-8?q?them=20(=E8=AE=A9Magpie=E6=94=AF=E6=8C=81=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E5=9B=BE=E5=83=8F=E8=AF=86=E5=88=AB=E6=A8=A1=E5=9E=8B)=20?= =?UTF-8?q?=E2=80=94=20until=20now=20an=20image=20sent=20to=20a=20model=20?= =?UTF-8?q?the=20catalog=20knows=20takes=20text=20only=20was=20turned=20aw?= =?UTF-8?q?ay=20in=20the=20latest=20turn=20and=20left=20out=20of=20older?= =?UTF-8?q?=20turns=20and=20tool=20results,=20so=20an=20agent=20on=20DeepS?= =?UTF-8?q?eek,=20GLM=20or=20any=20text-only=20model=20couldn't=20be=20sho?= =?UTF-8?q?wn=20a=20screenshot=20at=20all.=20Now=20each=20image=20in=20suc?= =?UTF-8?q?h=20a=20request=20is=20described=20once=20by=20the=20vision=20m?= =?UTF-8?q?odel=20and=20the=20description=20goes=20in=20its=20place,=20in?= =?UTF-8?q?=20every=20protocol=20(Chat,=20Responses,=20Anthropic,=20Gemini?= =?UTF-8?q?),=20current=20turn,=20older=20turns=20and=20tool=20results=20a?= =?UTF-8?q?like:=20"[Image,=20as=20=20describes=20it=20for=20this?= =?UTF-8?q?=20model,=20which=20can't=20see=20images:]=20=E2=80=A6=20[End?= =?UTF-8?q?=20of=20the=20image's=20description]".=20The=20describer=20is?= =?UTF-8?q?=20asked=20through=20the=20gateway=20itself=20(User-Agent=20mag?= =?UTF-8?q?pie-vision/1,=20recorded=20as=20agent=20magpie),=20with=20a=20s?= =?UTF-8?q?ystem=20prompt=20to=20transcribe=20text,=20code,=20errors=20and?= =?UTF-8?q?=20UI=20state=20exactly=20and=20not=20to=20answer=20or=20guess;?= =?UTF-8?q?=20images=20are=20described=20four=20at=20a=20time,=20and=20a?= =?UTF-8?q?=20description=20is=20kept=20(256,=20by=20model+image)=20since?= =?UTF-8?q?=20an=20agent=20sends=20a=20conversation's=20images=20again=20e?= =?UTF-8?q?very=20turn,=20a=20failure=20not=20kept.=20An=20image=20of=20th?= =?UTF-8?q?e=20latest=20turn=20that=20can't=20be=20described=20is=20a=2050?= =?UTF-8?q?2=20saying=20why;=20one=20in=20an=20older=20turn=20is=20left=20?= =?UTF-8?q?out=20as=20before.=20Settings=20=E2=86=92=20Images=20=E2=86=92?= =?UTF-8?q?=20Image=20recognition=20picks=20the=20model:=20Automatic=20(a?= =?UTF-8?q?=20small=20model=20that=20sees=20=E2=80=94=20a=20Claude=20accou?= =?UTF-8?q?nt's,=20a=20Codex=20account's,=20the=20cheapest=20of=20an=20API?= =?UTF-8?q?=20key's,=20another=20account's),=20Off=20(images=20turned=20aw?= =?UTF-8?q?ay=20as=20before),=20or=20any=20model=20or=20group=20that=20see?= =?UTF-8?q?s;=20an=20id=20that=20doesn't=20resolve=20is=20refused.=20In=20?= =?UTF-8?q?a=20routing=20group=20the=20description=20is=20made=20only=20wh?= =?UTF-8?q?en=20a=20text-only=20member=20is=20actually=20tried,=20so=20a?= =?UTF-8?q?=20rule=20that=20routes=20images=20to=20a=20member=20that=20see?= =?UTF-8?q?s=20costs=20nothing;=20a=20member=20whose=20image=20can't=20be?= =?UTF-8?q?=20described=20is=20skipped=20for=20the=20next,=20the=20last=20?= =?UTF-8?q?one=20answers=20502.=20The=20describer's=20own=20request=20is?= =?UTF-8?q?=20never=20described=20again,=20so=20a=20Vision=20set=20to=20a?= =?UTF-8?q?=20text-only=20model=20is=20turned=20away,=20not=20looped.=20Ch?= =?UTF-8?q?ecked=20in=20a=20sandbox=20HOME=20against=20a=20local=20vendor:?= =?UTF-8?q?=20an=20image=20sent=20over=20Chat=20and=20over=20Anthropic=20t?= =?UTF-8?q?o=20a=20text-only=20model=20reached=20it=20as=20the=20descripti?= =?UTF-8?q?on,=20the=20describer=20was=20asked=20once=20per=20image,=20and?= =?UTF-8?q?=20the=20Settings=20row=20and=20its=20picker=20in=20light=20and?= =?UTF-8?q?=20dark.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/gateway/gateway.go | 56 +++++- internal/gateway/image_input.go | 195 +++++++++++++----- internal/gateway/image_input_test.go | 1 + internal/gateway/rules_test.go | 18 ++ internal/gateway/search.go | 10 +- internal/gateway/vision.go | 287 +++++++++++++++++++++++++++ internal/gateway/vision_test.go | 213 ++++++++++++++++++++ internal/gui/api.go | 21 ++ internal/gui/assets/app.js | 40 +++- internal/gui/assets/i18n.js | 7 + internal/gui/assets/index.html | 2 + internal/settings/settings.go | 8 + 12 files changed, 801 insertions(+), 57 deletions(-) create mode 100644 internal/gateway/vision.go create mode 100644 internal/gateway/vision_test.go diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 3d18b470..65ffba01 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -177,6 +177,10 @@ type Server struct { // taken off it (see Relisten) lnMu sync.Mutex ln net.Listener + // the images described for models that can't see them (vision.go) + sightMu sync.Mutex + sights map[string]*sight + sightOrder []string } // New makes a gateway. @@ -693,9 +697,28 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro } } } + // Unless a model that sees describes them to it (vision.go). + seeing := sync.OnceValues(func() (string, bool) { + if describing(r.Context()) { + return "", false + } + return seer() + }) if imageInput != nil && !*imageInput { var currentImage bool - body, currentImage = textOnlyBody(from, body) + if see, ok := seeing(); ok && hasImage(from, body) { + seen, err := s.seenBody(r.Context(), from, body, see) + if err != nil { + call.Status, call.Error = 502, "image not described" + writeError(w, from, 502, fmt.Sprintf("model %q can't see images, and %s couldn't describe the image for it: %v", call.Model, see, err)) + finishCapture() + s.record(call) + return + } + body = seen + } else { + body, currentImage = textOnlyBody(from, body) + } if req, err := parse(from, body); err == nil && !currentImage { for _, msg := range req.Messages { if slices.ContainsFunc(msg.Parts, func(part Part) bool { return part.Kind == Image }) { @@ -780,7 +803,17 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro // text-only fallbacks. A failure must not send a current user image to // one of them. Historical images and tool results are omitted per // candidate below, so a text-only fallback can still answer those. - if isGroup { + // With a model to describe them, a text-only member is given the images + // described instead. + // Described only when such a member is tried: a rule that sends the + // images to a model that sees asks for no description. + var seenGroup func() ([]byte, error) + if isGroup && hasImage(from, body) { + if see, ok := seeing(); ok { + seenGroup = sync.OnceValues(func() ([]byte, error) { return s.seenBody(r.Context(), from, body, see) }) + } + } + if isGroup && seenGroup == nil { _, currentImage := textOnlyBody(from, body) if currentImage { var kept []candidate @@ -831,7 +864,24 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro } if isGroup { if in := membersImageInput([]provider.Member{{Provider: c.p, Model: c.model}}, nil); in != nil && !*in { - attemptBody, _ = textOnlyBody(from, body) // omit images in prior turns and tool results + if seenGroup != nil { + b, err := seenGroup() + if err != nil { + // only an image of the latest turn fails to be described + call.Error = "image not described" + if !last { + skipped = append(skipped, c.label()+": "+call.Error) + continue + } + see, _ := seeing() + call.Status = 502 + writeError(w, from, 502, fmt.Sprintf("model %q can't see images, and %s couldn't describe the image for it: %v", c.p.ID+"/"+c.model, see, err)) + break + } + attemptBody = b + } else { + attemptBody, _ = textOnlyBody(from, body) // omit images in prior turns and tool results + } } } picked := false // the effort asked for in place of the agent's diff --git a/internal/gateway/image_input.go b/internal/gateway/image_input.go index 26caab75..eb0f5bb9 100644 --- a/internal/gateway/image_input.go +++ b/internal/gateway/image_input.go @@ -2,18 +2,60 @@ package gateway import ( "encoding/json" + "errors" "strings" "github.com/yetone/magpie/internal/provider" ) +const imageOmitted = "[Image omitted: this model accepts text only.]" + +// errCurrentImage stops a walk at an image the latest turn gives the model. +var errCurrentImage = errors.New("an image in the latest turn") + // textOnlyBody rejects a user image in the latest turn and omits tool images // and images in older turns. It edits the wire JSON so passthrough keeps fields // the IR does not represent (such as provider-specific request options). func textOnlyBody(proto provider.Protocol, body []byte) ([]byte, bool) { + out, err := walkImages(proto, body, func(im imageAt) (json.RawMessage, error) { + if im.Current() { + return nil, errCurrentImage + } + return textBlock(proto, imageOmitted), nil + }) + if err != nil { + return body, true + } + return out, false +} + +// hasImage is whether the body has an image in any turn. +func hasImage(proto provider.Protocol, body []byte) bool { + found := errors.New("found") + _, err := walkImages(proto, body, func(imageAt) (json.RawMessage, error) { return nil, found }) + return err == found +} + +// imageAt is an image in a request, where it is. +type imageAt struct { + // Src is the image as a URL, a data: URL for one sent inline; empty for + // one only its vendor can open (a Responses file_id, a Gemini file). + Src string + // Last is set on one in the latest turn, Tool on one a tool gave back. + Last, Tool bool +} + +// Current is whether the image is one the user gives the model now, which +// the model is asked about. +func (im imageAt) Current() bool { return im.Last && !im.Tool } + +// walkImages gives every image in the body's turns to swap, and puts the +// block it returns in the image's place (nil keeps it). An error from swap +// ends the walk. The body is returned as it was when nothing was swapped. +func walkImages(proto provider.Protocol, body []byte, swap func(imageAt) (json.RawMessage, error)) ([]byte, error) { var request map[string]json.RawMessage if json.Unmarshal(body, &request) != nil { - return body, false + return body, nil } turnsKey, contentKey := "messages", "content" switch proto { @@ -24,7 +66,7 @@ func textOnlyBody(proto provider.Protocol, body []byte) ([]byte, bool) { } var turns []json.RawMessage if json.Unmarshal(request[turnsKey], &turns) != nil { - return body, false + return body, nil } changed := false for i, raw := range turns { @@ -45,33 +87,32 @@ func textOnlyBody(proto provider.Protocol, body []byte) ([]byte, bool) { json.Unmarshal(turn["role"], &kind) tool = kind == "tool" } - content, found, directImage := omitImageBlocks(proto, turn[key]) - if !found { - continue + content, swapped, err := swapImageBlocks(proto, turn[key], i == len(turns)-1, tool, swap) + if err != nil { + return body, err } - if i == len(turns)-1 && directImage && !tool { - return body, true + if !swapped { + continue } turn[key] = content turns[i], _ = json.Marshal(turn) changed = true } if !changed { - return body, false + return body, nil } request[turnsKey], _ = json.Marshal(turns) - body, _ = json.Marshal(request) - return body, false + return json.Marshal(request) } -// directImage distinguishes a pasted image from an image nested in a tool_result. -func omitImageBlocks(proto provider.Protocol, raw json.RawMessage) (json.RawMessage, bool, bool) { +// swapImageBlocks swaps the images among a turn's blocks, and those in an +// Anthropic tool_result's, which are a tool's. +func swapImageBlocks(proto provider.Protocol, raw json.RawMessage, last, tool bool, swap func(imageAt) (json.RawMessage, error)) (json.RawMessage, bool, error) { var blocks []json.RawMessage if json.Unmarshal(raw, &blocks) != nil { - return raw, false, false + return raw, false, nil } - found := false - directImage := false + swapped := false for i, rawBlock := range blocks { var block map[string]json.RawMessage if json.Unmarshal(rawBlock, &block) != nil { @@ -79,56 +120,110 @@ func omitImageBlocks(proto provider.Protocol, raw json.RawMessage) (json.RawMess } var kind string json.Unmarshal(block["type"], &kind) - isImage := false - switch proto { - case provider.Chat: - isImage = kind == "image_url" - case provider.Responses: - isImage = kind == "input_image" - case provider.Anthropic: - isImage = kind == "image" - case provider.Gemini: - var data struct { - MimeType string `json:"mimeType"` - } - json.Unmarshal(block["inlineData"], &data) - isImage = strings.HasPrefix(data.MimeType, "image/") - if len(block["fileData"]) > 0 { - var file struct { - MimeType string `json:"mimeType"` - } - if json.Unmarshal(block["fileData"], &file) != nil || file.MimeType == "" || strings.HasPrefix(file.MimeType, "image/") { - isImage = true // unknown file types may be images; fail closed - } - } - } + src, isImage := imageSrc(proto, kind, block) if isImage { - blocks[i] = imagePlaceholder(proto) - found = true - directImage = true + out, err := swap(imageAt{Src: src, Last: last, Tool: tool}) + if err != nil { + return raw, false, err + } + if out != nil { + blocks[i] = out + swapped = true + } continue } if proto == provider.Anthropic && kind == "tool_result" { - if content, nested, _ := omitImageBlocks(proto, block["content"]); nested { + content, nested, err := swapImageBlocks(proto, block["content"], last, true, swap) + if err != nil { + return raw, false, err + } + if nested { block["content"] = content blocks[i], _ = json.Marshal(block) - found = true + swapped = true } } } - if !found { - return raw, false, false + if !swapped { + return raw, false, nil } out, _ := json.Marshal(blocks) - return out, true, directImage + return out, true, nil } -func imagePlaceholder(proto provider.Protocol) json.RawMessage { +// imageSrc is whether a block is an image, and the image as a URL when it +// can be read from the block. +func imageSrc(proto provider.Protocol, kind string, block map[string]json.RawMessage) (string, bool) { + switch proto { + case provider.Chat: + if kind != "image_url" { + return "", false + } + var url string + if json.Unmarshal(block["image_url"], &url) != nil { + var obj struct { + URL string `json:"url"` + } + json.Unmarshal(block["image_url"], &obj) + url = obj.URL + } + return url, true + case provider.Responses: + if kind != "input_image" { + return "", false + } + var url string + json.Unmarshal(block["image_url"], &url) + return url, true + case provider.Anthropic: + if kind != "image" { + return "", false + } + var src struct { + Type string `json:"type"` + MediaType string `json:"media_type"` + Data string `json:"data"` + URL string `json:"url"` + } + json.Unmarshal(block["source"], &src) + switch { + case src.Type == "base64" && src.Data != "": + return "data:" + src.MediaType + ";base64," + src.Data, true + case src.Type == "url": + return src.URL, true + } + return "", true + case provider.Gemini: + var data struct { + MimeType string `json:"mimeType"` + Data string `json:"data"` + } + json.Unmarshal(block["inlineData"], &data) + if strings.HasPrefix(data.MimeType, "image/") { + return "data:" + data.MimeType + ";base64," + data.Data, true + } + if len(block["fileData"]) > 0 { + var file struct { + MimeType string `json:"mimeType"` + } + if json.Unmarshal(block["fileData"], &file) != nil || file.MimeType == "" || strings.HasPrefix(file.MimeType, "image/") { + return "", true // unknown file types may be images; fail closed + } + } + } + return "", false +} + +// textBlock is a text block of the protocol's. +func textBlock(proto provider.Protocol, text string) json.RawMessage { + var b []byte switch proto { case provider.Responses: - return json.RawMessage(`{"type":"input_text","text":"[Image omitted: this model accepts text only.]"}`) + b, _ = json.Marshal(map[string]string{"type": "input_text", "text": text}) case provider.Gemini: - return json.RawMessage(`{"text":"[Image omitted: this model accepts text only.]"}`) + b, _ = json.Marshal(map[string]string{"text": text}) + default: + b, _ = json.Marshal(map[string]string{"type": "text", "text": text}) } - return json.RawMessage(`{"type":"text","text":"[Image omitted: this model accepts text only.]"}`) + return b } diff --git a/internal/gateway/image_input_test.go b/internal/gateway/image_input_test.go index 1ac68c2d..ebca0f4f 100644 --- a/internal/gateway/image_input_test.go +++ b/internal/gateway/image_input_test.go @@ -16,6 +16,7 @@ import ( func TestKnownTextOnlyModelRejectsImagesBeforeUpstream(t *testing.T) { fresh(t) + noVision(t) var sent int up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { sent++ diff --git a/internal/gateway/rules_test.go b/internal/gateway/rules_test.go index 94d547a8..ca0b610d 100644 --- a/internal/gateway/rules_test.go +++ b/internal/gateway/rules_test.go @@ -314,6 +314,7 @@ func TestRuleUnreadyMember(t *testing.T) { func TestRuleImages(t *testing.T) { img := `{"model":"group/r","messages":[{"role":"user","content":[{"type":"text","text":"look"},{"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]}]}` s, _, b := ruled(t) + noVision(t) if code, _ := postAs(t, s, "x", img); code != 400 || b.n() != 0 { t.Fatalf("no rules: %d, b %d", code, b.n()) } @@ -521,6 +522,7 @@ func TestRuleTurnOutgrowsUnlistedMember(t *testing.T) { func TestRuleImagesUnlistedMember(t *testing.T) { img := `{"model":"group/r","messages":[{"role":"user","content":[{"type":"text","text":"look"},{"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]}]}` s, a, b := ruled(t) + noVision(t) p, err := provider.Find("a") if err != nil { t.Fatal(err) @@ -538,6 +540,7 @@ func TestRuleImagesUnlistedMember(t *testing.T) { // attached when the vision provider fails. func TestImageRuleDoesNotLeakImageToTextOnlyFallback(t *testing.T) { s, a, b := ruled(t, provider.Rule{Use: "b/big", Images: true}) + noVision(t) b.mu.Lock() b.fail = 503 b.mu.Unlock() @@ -551,6 +554,20 @@ func TestImageRuleDoesNotLeakImageToTextOnlyFallback(t *testing.T) { } } +// With the vision member down, nothing is left to describe the image: the +// text-only fallback isn't sent it, or a turn without it. +func TestImageRuleFallbackCantHaveTheImageDescribedByTheFailedMember(t *testing.T) { + s, a, b := ruled(t, provider.Rule{Use: "b/big", Images: true}) + b.mu.Lock() + b.fail = 503 + b.mu.Unlock() + img := `{"model":"group/r","messages":[{"role":"user","content":[{"type":"text","text":"look"},{"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]}]}` + code, out := postAs(t, s, "image-fallback", img) + if code != 502 || !strings.Contains(out, "couldn't describe the image") || a.n() != 0 { + t.Fatalf("%d %s, a %d", code, out, a.n()) + } +} + // When the vision member of a rule for images fails, a text-only member // answers a turn whose image is in an earlier message, without the image. func TestImageRuleFallbackOmitsOlderImages(t *testing.T) { @@ -622,6 +639,7 @@ func TestRuleTurnHoldsItsModelOnASharedAccount(t *testing.T) { // text-only first; and once the first rests, the second's goes first. func TestRuleFailsOverToTheNextRule(t *testing.T) { _, a, b := ruled(t) + noVision(t) c := &ruleUp{key: "kc"} srv := httptest.NewServer(c) t.Cleanup(srv.Close) diff --git a/internal/gateway/search.go b/internal/gateway/search.go index f60c6054..7f3d41cb 100644 --- a/internal/gateway/search.go +++ b/internal/gateway/search.go @@ -96,7 +96,7 @@ func searcher() (provider.Provider, string, bool) { if r < 0 || !p.Ready() || (top >= 0 && r >= top) { continue } - if m := smallModel(p); m != "" { + if m := smallModel(p, nil); m != "" { best, model, top = p, m, r } } @@ -104,10 +104,14 @@ func searcher() (provider.Provider, string, bool) { } // smallModel is the provider's cheapest small model of a vendor that -// searches well, or its cheapest, or its first. -func smallModel(p provider.Provider) string { +// searches well, or its cheapest, or its first; only among those keep +// holds, when it is given. +func smallModel(p provider.Provider, keep func(catalog.Model) bool) string { var ms []catalog.Model for _, m := range p.Available() { + if keep != nil && !keep(m) { + continue + } l := strings.ToLower(m.ID) // a subscription's list can have what magpie wrote into its agent if strings.HasPrefix(m.ID, provider.GroupPrefix) || (p.Account != nil && strings.Contains(m.ID, "/")) || diff --git a/internal/gateway/vision.go b/internal/gateway/vision.go new file mode 100644 index 00000000..3e837a06 --- /dev/null +++ b/internal/gateway/vision.go @@ -0,0 +1,287 @@ +package gateway + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "time" + + "github.com/yetone/magpie/internal/catalog" + "github.com/yetone/magpie/internal/provider" + "github.com/yetone/magpie/internal/settings" +) + +// A model that can't see images is given what a model that can says of +// them: each image in the request is described once by the vision model +// (the Settings' Vision, or one magpie picks) and the description goes in +// the image's place, in the current turn, older turns and tool results +// alike. With Vision off, or no model that sees, such an image is turned +// away as before (textOnlyBody). + +// VisionAgent is the User-Agent of the descriptions magpie asks for. +const VisionAgent = "magpie-vision/1" + +const visionTimeout = 2 * time.Minute + +// visionParallel is how many images of a request are described at once. +const visionParallel = 4 + +// sightsKept is how many descriptions are kept: an agent sends a +// conversation's images again every turn. +const sightsKept = 256 + +const visionSystem = `You describe images for an AI model that cannot see them. It will answer the user from your description alone, so leave nothing out that it may need. +- Transcribe all text exactly as written, keeping its layout: code, terminal output, error messages, logs, UI labels, menus, file names, numbers. +- For a screenshot of an app or page: which app or page it is, its layout, and the state of what is on it (selected, disabled, checked, highlighted, error states). +- For a chart or table: its kind, axes and labels, and every value you can read. +- For a diagram: its elements and how they are connected. +- For a photo or drawing: what it shows, with the details that matter. +Describe only what is there. Don't guess at what can't be read, say it can't be read. Don't answer questions or give advice. No preamble.` + +// describingKey marks a description magpie asks for, which must not have +// its image described again. +type describingKey struct{} + +func describing(ctx context.Context) bool { + v, _ := ctx.Value(describingKey{}).(bool) + return v +} + +// seer is the model that describes images: the Settings' Vision while it +// resolves, else AutoVision's. None when Vision is off or no model sees. +func seer() (string, bool) { + switch v := settings.Load().Vision; v { + case "off": + return "", false + case "": + default: + if _, _, ok := provider.Resolve(v); ok { + return v, true + } + } + m := AutoVision() + return m, m != "" +} + +// AutoVision is the model magpie picks to describe images when the +// Settings name none: a small model that sees of the providers set up — a +// Claude account's, a Codex account's, the cheapest of an API key's, +// another account's. "" when no model sees. +func AutoVision() string { + tier := func(p provider.Provider) int { + switch { + case p.Account != nil && p.Account.Agent == "claude": + return 0 + case p.Account != nil && p.Account.Agent == "codex": + return 1 + case p.Account == nil: + return 2 + } + return 3 + } + best, bestTier, bestCost := "", 0, 0.0 + for _, p := range provider.All() { + if !p.Ready() || p.Decides() { + continue + } + t := tier(p) + if best != "" && t > bestTier { + continue + } + m, cost := smallModelWhere(p, sees) + if m == "" { + continue + } + if best == "" || t < bestTier || cost < bestCost { + best, bestTier, bestCost = p.ID+"/"+m, t, cost + } + } + return best +} + +// sees is whether a model takes images. +func sees(m catalog.Model) bool { + return m.Images && (m.ImageInput == nil || *m.ImageInput) +} + +// sight is a description, once it is done. +type sight struct { + done chan struct{} + text string + err error +} + +// describe is what model says of the image at src, asked once for each +// image: a request that sends it again, or sends it while it is being +// described, is given the same description. +func (s *Server) describe(ctx context.Context, model, src string) (string, error) { + sum := sha256.Sum256([]byte(model + "\x00" + src)) + key := hex.EncodeToString(sum[:]) + s.sightMu.Lock() + if e, ok := s.sights[key]; ok { + s.sightMu.Unlock() + select { + case <-e.done: + return e.text, e.err + case <-ctx.Done(): + return "", ctx.Err() + } + } + if s.sights == nil { + s.sights = map[string]*sight{} + } + e := &sight{done: make(chan struct{})} + s.sights[key] = e + s.sightOrder = append(s.sightOrder, key) + for len(s.sightOrder) > sightsKept { + delete(s.sights, s.sightOrder[0]) + s.sightOrder = s.sightOrder[1:] + } + s.sightMu.Unlock() + e.text, e.err = s.askVision(ctx, model, src) + if e.err != nil { + // a failure isn't kept: the next request asks again + s.sightMu.Lock() + if s.sights[key] == e { + delete(s.sights, key) + } + s.sightMu.Unlock() + } + close(e.done) + return e.text, e.err +} + +// askVision asks model through the gateway itself, as a client would, what +// the image at src shows. +func (s *Server) askVision(ctx context.Context, model, src string) (string, error) { + ctx, cancel := context.WithTimeout(context.WithValue(ctx, describingKey{}, true), visionTimeout) + defer cancel() + req := map[string]any{ + "model": model, + "stream": false, + "messages": []map[string]any{ + {"role": "system", "content": visionSystem}, + {"role": "user", "content": []map[string]any{ + {"type": "text", "text": "Describe this image."}, + {"type": "image_url", "image_url": map[string]string{"url": src}}, + }}, + }, + "max_tokens": 4096, + } + if effort := classifyEffort(model); effort != "" { + req["reasoning_effort"] = effort + } + body, _ := json.Marshal(req) + r, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://magpie/v1/chat/completions", nil) + if err != nil { + return "", err + } + r.Header.Set("Content-Type", "application/json") + r.Header.Set("User-Agent", VisionAgent) + w := httptest.NewRecorder() + s.serve(w, r, provider.Chat, body) + if ctx.Err() != nil { + return "", fmt.Errorf("no answer in %s", visionTimeout) + } + var out struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + Error *struct { + Message string `json:"message"` + } `json:"error"` + } + if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil { + return "", fmt.Errorf("%d, not an answer", w.Code) + } + if w.Code >= 300 || out.Error != nil { + msg := http.StatusText(w.Code) + if out.Error != nil && out.Error.Message != "" { + msg = out.Error.Message + } + return "", errors.New(msg) + } + if len(out.Choices) == 0 || strings.TrimSpace(out.Choices[0].Message.Content) == "" { + return "", errors.New("no description") + } + return strings.TrimSpace(out.Choices[0].Message.Content), nil +} + +// seenBody is the body with each of its images described by model in its +// place. An image of the latest turn that can't be described is an error: +// the model would be asked about what it isn't given. One in an older turn +// or a tool's result is left out, as textOnlyBody leaves it. +func (s *Server) seenBody(ctx context.Context, proto provider.Protocol, body []byte, model string) ([]byte, error) { + var srcs []string + have := map[string]bool{} + walkImages(proto, body, func(im imageAt) (json.RawMessage, error) { + if im.Src != "" && !have[im.Src] { + have[im.Src] = true + srcs = append(srcs, im.Src) + } + return nil, nil + }) + type result struct { + text string + err error + } + results := make([]result, len(srcs)) + var wg sync.WaitGroup + slots := make(chan struct{}, visionParallel) + for i, src := range srcs { + wg.Add(1) + go func() { + defer wg.Done() + slots <- struct{}{} + defer func() { <-slots }() + text, err := s.describe(ctx, model, src) + results[i] = result{text, err} + }() + } + wg.Wait() + of := map[string]result{} + for i, src := range srcs { + of[src] = results[i] + } + return walkImages(proto, body, func(im imageAt) (json.RawMessage, error) { + if im.Src == "" { + if im.Current() { + return nil, errors.New("the image is a file only its vendor can open") + } + return textBlock(proto, imageOmitted), nil + } + r := of[im.Src] + if r.err != nil { + if im.Current() { + return nil, r.err + } + return textBlock(proto, "[Image omitted: this model accepts text only, and the image couldn't be described.]"), nil + } + // on its own lines: a translation may join it to the text around it + return textBlock(proto, "\n[Image, as "+model+" describes it for this model, which can't see images:]\n"+r.text+"\n[End of the image's description]\n"), nil + }) +} + +// smallModelWhere is smallModel among the provider's models that keep +// holds, and what it costs; "" when there is none. +func smallModelWhere(p provider.Provider, keep func(catalog.Model) bool) (string, float64) { + id := smallModel(p, keep) + if id == "" { + return "", 0 + } + for _, m := range p.Available() { + if m.ID == id && m.Price != nil { + return id, m.Price.Input + m.Price.Output + } + } + return id, 1e9 +} diff --git a/internal/gateway/vision_test.go b/internal/gateway/vision_test.go new file mode 100644 index 00000000..0f2a5549 --- /dev/null +++ b/internal/gateway/vision_test.go @@ -0,0 +1,213 @@ +package gateway + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/yetone/magpie/internal/catalog" + "github.com/yetone/magpie/internal/provider" + "github.com/yetone/magpie/internal/settings" +) + +// noVision turns describing off, for a test of images turned away. +func noVision(t *testing.T) { + t.Helper() + s := settings.Load() + s.Vision = "off" + if err := settings.Save(s); err != nil { + t.Fatal(err) + } +} + +// eyeUp is an upstream with a text-only model and one that sees: the one +// that sees describes every image as a red square, or fails when fail is +// set; what each model is sent is kept. +type eyeUp struct { + mu sync.Mutex + fail bool + bodies map[string][]string +} + +func (u *eyeUp) ServeHTTP(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + model := modelOf(body) + u.mu.Lock() + if u.bodies == nil { + u.bodies = map[string][]string{} + } + u.bodies[model] = append(u.bodies[model], string(body)) + fail := u.fail + u.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + if strings.HasSuffix(model, "eye") && fail { + w.WriteHeader(500) + io.WriteString(w, `{"error":{"message":"eye is down"}}`) + return + } + answer := "ok" + if strings.HasSuffix(model, "eye") { + answer = "A red square with the word HELLO in it." + } + if strings.Contains(string(body), `"stream":true`) { + w.Header().Set("Content-Type", "text/event-stream") + io.WriteString(w, `data: {"id":"x","choices":[{"index":0,"delta":{"role":"assistant","content":"`+answer+`"}}]}`+"\n\n") + io.WriteString(w, `data: {"id":"x","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`+"\n\n") + io.WriteString(w, "data: [DONE]\n\n") + return + } + io.WriteString(w, `{"id":"x","choices":[{"index":0,"message":{"role":"assistant","content":"`+answer+`"},"finish_reason":"stop"}]}`) +} + +func (u *eyeUp) sent(model string) []string { + u.mu.Lock() + defer u.mu.Unlock() + return append([]string(nil), u.bodies[model]...) +} + +func eyed(t *testing.T) (*Server, *eyeUp) { + t.Helper() + fresh(t) + u := &eyeUp{} + up := httptest.NewServer(u) + t.Cleanup(up.Close) + if err := provider.Save(provider.Provider{ID: "probe", Name: "Probe", Chat: up.URL + "/v1", Key: "key", Models: []string{"text", "eye"}}); err != nil { + t.Fatal(err) + } + if err := catalog.SaveLive("probe", up.URL+"/v1", []catalog.Model{ + {ID: "text", ImageInput: imageInputBool(false)}, + {ID: "eye", Images: true, ImageInput: imageInputBool(true)}, + }); err != nil { + t.Fatal(err) + } + return New(), u +} + +func TestTextOnlyModelIsGivenTheImagesDescription(t *testing.T) { + s, u := eyed(t) + cases := []struct{ path, body string }{ + {"/v1/chat/completions", `{"model":"probe/text","messages":[{"role":"user","content":[{"type":"text","text":"read"},{"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]}]}`}, + {"/v1/responses", `{"model":"probe/text","input":[{"role":"user","content":[{"type":"input_text","text":"read"},{"type":"input_image","image_url":"data:image/png;base64,aGVsbG8="}]}]}`}, + {"/v1/messages", `{"model":"probe/text","max_tokens":16,"messages":[{"role":"user","content":[{"type":"text","text":"read"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"aGVsbG8="}}]}]}`}, + {"/v1beta/models/probe/text:generateContent", `{"contents":[{"role":"user","parts":[{"text":"read"},{"inlineData":{"mimeType":"image/png","data":"aGVsbG8="}}]}]}`}, + } + for _, tc := range cases { + rec := httptest.NewRecorder() + s.Handler().ServeHTTP(rec, httptest.NewRequest("POST", tc.path, strings.NewReader(tc.body))) + if rec.Code != 200 { + t.Errorf("%s: %d %s", tc.path, rec.Code, rec.Body.String()) + } + } + got := u.sent("text") + if len(got) != len(cases) { + t.Fatalf("text model sent %d requests, want %d", len(got), len(cases)) + } + for i, b := range got { + if !strings.Contains(b, "HELLO") || !strings.Contains(b, "probe/eye describes it") || strings.Contains(b, "aGVsbG8=") { + t.Errorf("%s: text model was sent %s", cases[i].path, b) + } + } + // the same image in four requests is described once + if n := len(u.sent("eye")); n != 1 { + t.Fatalf("the image was described %d times", n) + } + if !strings.Contains(u.sent("eye")[0], "aGVsbG8=") { + t.Fatalf("the describer wasn't sent the image: %s", u.sent("eye")[0]) + } +} + +func TestVisionSettingPicksTheDescriber(t *testing.T) { + s, u := eyed(t) + // a second model that sees, which the setting names + up2 := &eyeUp{} + srv := httptest.NewServer(up2) + t.Cleanup(srv.Close) + if err := provider.Save(provider.Provider{ID: "other", Name: "Other", Chat: srv.URL + "/v1", Key: "key", Models: []string{"bigeye"}}); err != nil { + t.Fatal(err) + } + if err := catalog.SaveLive("other", srv.URL+"/v1", []catalog.Model{{ID: "bigeye", Images: true, ImageInput: imageInputBool(true)}}); err != nil { + t.Fatal(err) + } + st := settings.Load() + st.Vision = "other/bigeye" + if err := settings.Save(st); err != nil { + t.Fatal(err) + } + code, body := postAs(t, s, "", `{"model":"probe/text","messages":[{"role":"user","content":[{"type":"image_url","image_url":"https://example.com/a.png"}]}]}`) + if code != 200 { + t.Fatalf("%d %s", code, body) + } + if len(u.sent("eye")) != 0 || len(up2.sent("bigeye")) != 1 { + t.Fatalf("eye asked %d times, bigeye %d", len(u.sent("eye")), len(up2.sent("bigeye"))) + } + if !strings.Contains(u.sent("text")[0], "other/bigeye describes it") { + t.Fatalf("text model was sent %s", u.sent("text")[0]) + } +} + +func TestImageThatCantBeDescribed(t *testing.T) { + s, u := eyed(t) + u.fail = true + code, body := postAs(t, s, "", `{"model":"probe/text","messages":[{"role":"user","content":[{"type":"text","text":"read"},{"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]}]}`) + if code != 502 || !strings.Contains(body, "couldn't describe the image") || len(u.sent("text")) != 0 { + t.Fatalf("current image: %d %s; text model sent %d", code, body, len(u.sent("text"))) + } + // an older turn's image that can't be described is left out + code, body = postAs(t, s, "", `{"model":"probe/text","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,b2xk"}}]},{"role":"assistant","content":"seen"},{"role":"user","content":"and now?"}]}`) + if code != 200 { + t.Fatalf("older image: %d %s", code, body) + } + if got := u.sent("text"); len(got) != 1 || !strings.Contains(got[0], "couldn't be described") { + t.Fatalf("text model was sent %v", got) + } + // a failure isn't kept + u.mu.Lock() + u.fail = false + u.mu.Unlock() + if code, body := postAs(t, s, "", `{"model":"probe/text","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]}]}`); code != 200 { + t.Fatalf("after recovery: %d %s", code, body) + } +} + +func TestGroupsTextOnlyMemberIsGivenTheDescription(t *testing.T) { + s, u := eyed(t) + if err := provider.SaveGroup(provider.Group{Name: "Mixed", Members: []string{"probe/text", "probe/eye"}, Routing: provider.Ordered}); err != nil { + t.Fatal(err) + } + code, body := postAs(t, s, "", `{"model":"group/mixed","messages":[{"role":"user","content":[{"type":"text","text":"read"},{"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]}]}`) + if code != 200 { + t.Fatalf("%d %s", code, body) + } + if got := u.sent("text"); len(got) != 1 || !strings.Contains(got[0], "HELLO") { + t.Fatalf("the group's first member was sent %v", got) + } +} + +func TestDescriberThatCantSeeIsntAskedToDescribe(t *testing.T) { + s, u := eyed(t) + // the setting names a text-only model: its own request is turned away, + // not described by itself again and again + st := settings.Load() + st.Vision = "probe/text" + if err := settings.Save(st); err != nil { + t.Fatal(err) + } + code, body := postAs(t, s, "", `{"model":"probe/text","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]}]}`) + if code != 502 || len(u.sent("text")) != 0 { + t.Fatalf("%d %s; text model sent %d", code, body, len(u.sent("text"))) + } +} + +func TestSeerPicksAModelThatSees(t *testing.T) { + eyed(t) + if m, ok := seer(); !ok || m != "probe/eye" { + t.Fatalf("seer = %q %v", m, ok) + } + noVision(t) + if _, ok := seer(); ok { + t.Fatal("seer with vision off") + } +} diff --git a/internal/gui/api.go b/internal/gui/api.go index caa63b75..73ddbe1b 100644 --- a/internal/gui/api.go +++ b/internal/gui/api.go @@ -8,6 +8,7 @@ import ( "context" "embed" "encoding/json" + "fmt" "mime" "net/http" "os" @@ -119,6 +120,10 @@ type settingsJSON struct { ProxySource string `json:"proxySource"` // whether magpie opens at login: the system's record, not a setting Login bool `json:"login"` + // the model that describes images when Vision names none, and those + // that can be named + VisionAuto string `json:"visionAuto,omitempty"` + VisionModels []modelRef `json:"visionModels"` // where other machines reach the gateway while it is shared LANURLs []string `json:"lanURLs,omitempty"` // when the Codex warm-up last started an account's window @@ -135,6 +140,16 @@ func settingsState() settingsJSON { s.LANURLs = gateway.LANURLs() } s.CodexWarmed, s.ClaudeWarmed = latest(provider.CodexWarmed()), latest(provider.ClaudeWarmed()) + s.VisionAuto, s.VisionModels = gateway.AutoVision(), []modelRef{} + for _, e := range provider.Served() { + if e.Images && (e.ImageInput == nil || *e.ImageInput) && (e.Group != "" || e.Provider.Ready()) { + m := modelRef{ID: e.ID, Name: e.Name, Provider: e.Provider.ID, PName: e.Provider.Name, Icon: e.Provider.Icon} + if e.Group != "" { + m.Provider, m.PName = "", e.Group + } + s.VisionModels = append(s.VisionModels, m) + } + } return s } @@ -334,6 +349,12 @@ func Handler(w Windows, gw *gateway.Server) http.Handler { in.LAN, in.LANKey = cur.LAN, cur.LANKey // used or left is the Usage page's toggle as much as Settings', set on its own in.QuotaLeft = cur.QuotaLeft + if v := strings.TrimSpace(in.Vision); v != "" && v != "off" && v != cur.Vision { + if _, _, ok := provider.Resolve(v); !ok { + fail(rw, fmt.Errorf("no model %s to describe images", v)) + return + } + } if err := settings.Save(in); err != nil { fail(rw, err) return diff --git a/internal/gui/assets/app.js b/internal/gui/assets/app.js index a4cb756f..a1a63a64 100644 --- a/internal/gui/assets/app.js +++ b/internal/gui/assets/app.js @@ -5372,6 +5372,7 @@ function renderSettings() { (v) => savePrefs({ ...keep, claudeWarmAt: v })); renderTrayUsage(s, keep); renderProxy(s, keep); + renderImages(s, keep); renderRedact(s, keep); renderLAN(s); renderSync(); @@ -5716,6 +5717,43 @@ function renderWarmAt(box, sub, at, onReset, what, save) { box.append(segs([["off", t("Off")], ["on", t("On")]], at ? "on" : "off", (v) => save(v === "on" ? at || "06:00" : ""))); } +// renderImages: the model that describes images to a model that can't see +// them — the one magpie picks, one named, or none, and then such an image +// is turned away. +function renderImages(s, keep) { + const box = $("#imagesList"); + box.replaceChildren(); + const models = s.visionModels || []; + const named = (id) => { + const m = models.find((x) => x.id === id); + return m ? `${m.name || m.id} · ${m.providerName}` : id; + }; + const r = el("div", "row pref"); + const who = el("div", "who"); + const v = s.vision || ""; + who.append(el("div", "name", t("Image recognition")), el("div", "sub", + v === "off" ? t("A model that can't see images is sent none: a request with one in its latest message is turned away") + : t("When the model in use can't see images, this one describes them to it, once for each image"))); + const b = el("button", "rt-cond on"); + const icOf = (id) => models.find((x) => x.id === id)?.icon; + if (v === "off") b.append(el("span", "", t("Off"))); + else if (v) b.append(icon(icOf(v) || "generic"), el("span", "", named(v))); + else { + if (s.visionAuto) b.append(icon(icOf(s.visionAuto) || "generic")); + b.append(el("span", "", s.visionAuto ? t("Automatic") + " · " + named(s.visionAuto) : t("Automatic") + " · " + t("no model that sees"))); + } + const opt = (x) => ({ value: x.id, label: x.name || x.id, note: x.providerName, icon: x.icon, group: x.providerName, ref: x.id }); + b.onclick = (ev) => openPicker({ id: "", name: "", fields: [] }, { key: "vision", label: "model", value: v, options: [ + { value: "", label: t("Automatic"), note: s.visionAuto ? named(s.visionAuto) : t("no model that sees"), reset: true }, + { value: "off", label: t("Off"), note: t("images turned away") }, + ...models.map(opt)], + onPick: (id) => { if (id !== v) savePrefs({ ...keep, vision: id }); } }, b, ev); + const val = el("div", "val"); + val.append(b); + r.append(who, val); + box.append(r); +} + // renderRedact: what the gateway masks before a request goes to a vendor — // secrets, personal data, the user's own words — and puts back in what the // vendor answers. @@ -5842,7 +5880,7 @@ function prefsKeep(s) { return { theme: s.theme, lang: s.lang, tray: s.tray, dock: !!s.dock, dockWindow: !!s.dockWindow, proxy: s.proxy || "", redact: !!s.redact, redactPersonal: !!s.redactPersonal, redactWords: s.redactWords || [], codexWarmup: s.codexWarmup || "", claudeWarmup: s.claudeWarmup || "", codexWarmAt: s.codexWarmAt || "", claudeWarmAt: s.claudeWarmAt || "", noStats: !!s.noStats, - trayUsage: s.trayUsage || "", trayUsageEvery: s.trayUsageEvery || 3 }; + trayUsage: s.trayUsage || "", trayUsageEvery: s.trayUsageEvery || 3, vision: s.vision || "" }; } // savePrefs sends what the page was drawn with (prefsBase) and the choice diff --git a/internal/gui/assets/i18n.js b/internal/gui/assets/i18n.js index 4bc876a5..004d48e0 100644 --- a/internal/gui/assets/i18n.js +++ b/internal/gui/assets/i18n.js @@ -84,6 +84,13 @@ const I18N = { "No system proxy found; requests to vendors go direct": "未发现系统代理，对模型厂商的请求直连", "Off": "关闭", "On": "开启", + "Images": "图像", + "Image recognition": "识图模型", + "When the model in use can't see images, this one describes them to it, once for each image": "当前模型看不了图时，由它把图片描述成文字交给当前模型，每张图只描述一次", + "A model that can't see images is sent none: a request with one in its latest message is turned away": "不给看不了图的模型发图片：最新一条消息带图的请求会被拒绝", + "Automatic": "自动", + "no model that sees": "没有能看图的模型", + "images turned away": "带图的请求被拒绝", "Privacy": "隐私", "Mask secrets": "脱敏密钥", "Local network": "局域网", diff --git a/internal/gui/assets/index.html b/internal/gui/assets/index.html index 93be35a3..988f9095 100644 --- a/internal/gui/assets/index.html +++ b/internal/gui/assets/index.html @@ -186,6 +186,8 @@
Local network
+
Images
+
Privacy
Sync and backup
diff --git a/internal/settings/settings.go b/internal/settings/settings.go index 2dffa128..d52513ba 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -66,6 +66,10 @@ type Settings struct { // NoStats stops the one event a day that counts magpie's users (see // internal/stats). NoStats bool `json:"noStats,omitempty"` + // Vision is the model that describes an image to a model that can't see + // it: a model's id (provider/model, group/), "off" to turn such an + // image away, or empty for one magpie picks (see gateway.seer). + Vision string `json:"vision,omitempty"` // TrayUsage is the subscription or plan whose windows are shown beside // the tray icon, by its provider and account ("claude|a@b.c"); "" none. TrayUsage string `json:"trayUsage,omitempty"` @@ -203,6 +207,10 @@ func Save(s Settings) error { return fmt.Errorf("proxy must look like http://127.0.0.1:7890 or socks5://127.0.0.1:1080, not %q", s.Proxy) } } + s.Vision = strings.TrimSpace(s.Vision) + if s.Vision != "" && s.Vision != "off" && !strings.Contains(s.Vision, "/") { + return fmt.Errorf("the vision model must be a model's id such as openai/gpt-5-mini, or off, not %q", s.Vision) + } s.AgentOrder, s.AgentsHidden, s.AgentsShown = ids(s.AgentOrder), ids(s.AgentsHidden), ids(s.AgentsShown) if err := os.MkdirAll(Dir(), 0o755); err != nil { return err