gateway: a model that can't see images is given what a model that can says of them (让Magpie支持配置图像识别模型) — until now an image sent to a model the catalog knows takes text only was turned away in the latest turn and left out of older turns and tool results, so an agent on DeepSeek, GLM or any text-only model couldn't be shown a screenshot at all. Now each image in such a request is described once by the vision model and the description goes in its place, in every protocol (Chat, Responses, Anthropic, Gemini), current turn, older turns and tool results alike: "[Image, as <model> describes it for this model, which can't see images:] … [End of the image's description]". The describer is asked through the gateway itself (User-Agent magpie-vision/1, recorded as agent magpie), with a system prompt to transcribe text, code, errors and UI state exactly and not to answer or guess; images are described four at a time, and a description is kept (256, by model+image) since an agent sends a conversation's images again every turn, a failure not kept. An image of the latest turn that can't be described is a 502 saying why; one in an older turn is left out as before. Settings → Images → Image recognition picks the model: Automatic (a small model that sees — a Claude account's, a Codex account's, the cheapest of an API key's, another account's), Off (images turned away as before), or any model or group that sees; an id that doesn't resolve is refused. In a routing group the description is made only when a text-only member is actually tried, so a rule that routes images to a member that sees costs nothing; a member whose image can't be described is skipped for the next, the last one answers 502. The describer's own request is never described again, so a Vision set to a text-only model is turned away, not looped. Checked in a sandbox HOME against a local vendor: an image sent over Chat and over Anthropic to a text-only model reached it as the description, the describer was asked once per image, and the Settings row and its picker in light and dark.

internal/gateway/gateway.go

@@ -177,6 +177,10 @@ type Server struct {
 	// taken off it (see Relisten)
 	lnMu sync.Mutex
 	ln   net.Listener
+	// the images described for models that can't see them (vision.go)
+	sightMu    sync.Mutex
+	sights     map[string]*sight
+	sightOrder []string
 }
 
 // New makes a gateway.
@@ -693,9 +697,28 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro
 			}
 		}
 	}
+	// Unless a model that sees describes them to it (vision.go).
+	seeing := sync.OnceValues(func() (string, bool) {
+		if describing(r.Context()) {
+			return "", false
+		}
+		return seer()
+	})
 	if imageInput != nil && !*imageInput {
 		var currentImage bool
-		body, currentImage = textOnlyBody(from, body)
+		if see, ok := seeing(); ok && hasImage(from, body) {
+			seen, err := s.seenBody(r.Context(), from, body, see)
+			if err != nil {
+				call.Status, call.Error = 502, "image not described"
+				writeError(w, from, 502, fmt.Sprintf("model %q can't see images, and %s couldn't describe the image for it: %v", call.Model, see, err))
+				finishCapture()
+				s.record(call)
+				return
+			}
+			body = seen
+		} else {
+			body, currentImage = textOnlyBody(from, body)
+		}
 		if req, err := parse(from, body); err == nil && !currentImage {
 			for _, msg := range req.Messages {
 				if slices.ContainsFunc(msg.Parts, func(part Part) bool { return part.Kind == Image }) {
@@ -780,7 +803,17 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro
 	// text-only fallbacks. A failure must not send a current user image to
 	// one of them. Historical images and tool results are omitted per
 	// candidate below, so a text-only fallback can still answer those.
-	if isGroup {
+	// With a model to describe them, a text-only member is given the images
+	// described instead.
+	// Described only when such a member is tried: a rule that sends the
+	// images to a model that sees asks for no description.
+	var seenGroup func() ([]byte, error)
+	if isGroup && hasImage(from, body) {
+		if see, ok := seeing(); ok {
+			seenGroup = sync.OnceValues(func() ([]byte, error) { return s.seenBody(r.Context(), from, body, see) })
+		}
+	}
+	if isGroup && seenGroup == nil {
 		_, currentImage := textOnlyBody(from, body)
 		if currentImage {
 			var kept []candidate
@@ -831,7 +864,24 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro
 		}
 		if isGroup {
 			if in := membersImageInput([]provider.Member{{Provider: c.p, Model: c.model}}, nil); in != nil && !*in {
-				attemptBody, _ = textOnlyBody(from, body) // omit images in prior turns and tool results
+				if seenGroup != nil {
+					b, err := seenGroup()
+					if err != nil {
+						// only an image of the latest turn fails to be described
+						call.Error = "image not described"
+						if !last {
+							skipped = append(skipped, c.label()+": "+call.Error)
+							continue
+						}
+						see, _ := seeing()
+						call.Status = 502
+						writeError(w, from, 502, fmt.Sprintf("model %q can't see images, and %s couldn't describe the image for it: %v", c.p.ID+"/"+c.model, see, err))
+						break
+					}
+					attemptBody = b
+				} else {
+					attemptBody, _ = textOnlyBody(from, body) // omit images in prior turns and tool results
+				}
 			}
 		}
 		picked := false // the effort asked for in place of the agent's

internal/gateway/image_input.go

@@ -2,18 +2,60 @@ package gateway
 
 import (
 	"encoding/json"
+	"errors"
 	"strings"
 
 	"github.com/yetone/magpie/internal/provider"
 )
 
+const imageOmitted = "[Image omitted: this model accepts text only.]"
+
+// errCurrentImage stops a walk at an image the latest turn gives the model.
+var errCurrentImage = errors.New("an image in the latest turn")
+
 // textOnlyBody rejects a user image in the latest turn and omits tool images
 // and images in older turns. It edits the wire JSON so passthrough keeps fields
 // the IR does not represent (such as provider-specific request options).
 func textOnlyBody(proto provider.Protocol, body []byte) ([]byte, bool) {
+	out, err := walkImages(proto, body, func(im imageAt) (json.RawMessage, error) {
+		if im.Current() {
+			return nil, errCurrentImage
+		}
+		return textBlock(proto, imageOmitted), nil
+	})
+	if err != nil {
+		return body, true
+	}
+	return out, false
+}
+
+// hasImage is whether the body has an image in any turn.
+func hasImage(proto provider.Protocol, body []byte) bool {
+	found := errors.New("found")
+	_, err := walkImages(proto, body, func(imageAt) (json.RawMessage, error) { return nil, found })
+	return err == found
+}
+
+// imageAt is an image in a request, where it is.
+type imageAt struct {
+	// Src is the image as a URL, a data: URL for one sent inline; empty for
+	// one only its vendor can open (a Responses file_id, a Gemini file).
+	Src string
+	// Last is set on one in the latest turn, Tool on one a tool gave back.
+	Last, Tool bool
+}
+
+// Current is whether the image is one the user gives the model now, which
+// the model is asked about.
+func (im imageAt) Current() bool { return im.Last && !im.Tool }
+
+// walkImages gives every image in the body's turns to swap, and puts the
+// block it returns in the image's place (nil keeps it). An error from swap
+// ends the walk. The body is returned as it was when nothing was swapped.
+func walkImages(proto provider.Protocol, body []byte, swap func(imageAt) (json.RawMessage, error)) ([]byte, error) {
 	var request map[string]json.RawMessage
 	if json.Unmarshal(body, &request) != nil {
-		return body, false
+		return body, nil
 	}
 	turnsKey, contentKey := "messages", "content"
 	switch proto {
@@ -24,7 +66,7 @@ func textOnlyBody(proto provider.Protocol, body []byte) ([]byte, bool) {
 	}
 	var turns []json.RawMessage
 	if json.Unmarshal(request[turnsKey], &turns) != nil {
-		return body, false
+		return body, nil
 	}
 	changed := false
 	for i, raw := range turns {
@@ -45,33 +87,32 @@ func textOnlyBody(proto provider.Protocol, body []byte) ([]byte, bool) {
 			json.Unmarshal(turn["role"], &kind)
 			tool = kind == "tool"
 		}
-		content, found, directImage := omitImageBlocks(proto, turn[key])
-		if !found {
-			continue
+		content, swapped, err := swapImageBlocks(proto, turn[key], i == len(turns)-1, tool, swap)
+		if err != nil {
+			return body, err
 		}
-		if i == len(turns)-1 && directImage && !tool {
-			return body, true
+		if !swapped {
+			continue
 		}
 		turn[key] = content
 		turns[i], _ = json.Marshal(turn)
 		changed = true
 	}
 	if !changed {
-		return body, false
+		return body, nil
 	}
 	request[turnsKey], _ = json.Marshal(turns)
-	body, _ = json.Marshal(request)
-	return body, false
+	return json.Marshal(request)
 }
 
-// directImage distinguishes a pasted image from an image nested in a tool_result.
-func omitImageBlocks(proto provider.Protocol, raw json.RawMessage) (json.RawMessage, bool, bool) {
+// swapImageBlocks swaps the images among a turn's blocks, and those in an
+// Anthropic tool_result's, which are a tool's.
+func swapImageBlocks(proto provider.Protocol, raw json.RawMessage, last, tool bool, swap func(imageAt) (json.RawMessage, error)) (json.RawMessage, bool, error) {
 	var blocks []json.RawMessage
 	if json.Unmarshal(raw, &blocks) != nil {
-		return raw, false, false
+		return raw, false, nil
 	}
-	found := false
-	directImage := false
+	swapped := false
 	for i, rawBlock := range blocks {
 		var block map[string]json.RawMessage
 		if json.Unmarshal(rawBlock, &block) != nil {
@@ -79,56 +120,110 @@ func omitImageBlocks(proto provider.Protocol, raw json.RawMessage) (json.RawMess
 		}
 		var kind string
 		json.Unmarshal(block["type"], &kind)
-		isImage := false
-		switch proto {
-		case provider.Chat:
-			isImage = kind == "image_url"
-		case provider.Responses:
-			isImage = kind == "input_image"
-		case provider.Anthropic:
-			isImage = kind == "image"
-		case provider.Gemini:
-			var data struct {
-				MimeType string `json:"mimeType"`
-			}
-			json.Unmarshal(block["inlineData"], &data)
-			isImage = strings.HasPrefix(data.MimeType, "image/")
-			if len(block["fileData"]) > 0 {
-				var file struct {
-					MimeType string `json:"mimeType"`
-				}
-				if json.Unmarshal(block["fileData"], &file) != nil || file.MimeType == "" || strings.HasPrefix(file.MimeType, "image/") {
-					isImage = true // unknown file types may be images; fail closed
-				}
-			}
-		}
+		src, isImage := imageSrc(proto, kind, block)
 		if isImage {
-			blocks[i] = imagePlaceholder(proto)
-			found = true
-			directImage = true
+			out, err := swap(imageAt{Src: src, Last: last, Tool: tool})
+			if err != nil {
+				return raw, false, err
+			}
+			if out != nil {
+				blocks[i] = out
+				swapped = true
+			}
 			continue
 		}
 		if proto == provider.Anthropic && kind == "tool_result" {
-			if content, nested, _ := omitImageBlocks(proto, block["content"]); nested {
+			content, nested, err := swapImageBlocks(proto, block["content"], last, true, swap)
+			if err != nil {
+				return raw, false, err
+			}
+			if nested {
 				block["content"] = content
 				blocks[i], _ = json.Marshal(block)
-				found = true
+				swapped = true
 			}
 		}
 	}
-	if !found {
-		return raw, false, false
+	if !swapped {
+		return raw, false, nil
 	}
 	out, _ := json.Marshal(blocks)
-	return out, true, directImage
+	return out, true, nil
 }
 
-func imagePlaceholder(proto provider.Protocol) json.RawMessage {
+// imageSrc is whether a block is an image, and the image as a URL when it
+// can be read from the block.
+func imageSrc(proto provider.Protocol, kind string, block map[string]json.RawMessage) (string, bool) {
+	switch proto {
+	case provider.Chat:
+		if kind != "image_url" {
+			return "", false
+		}
+		var url string
+		if json.Unmarshal(block["image_url"], &url) != nil {
+			var obj struct {
+				URL string `json:"url"`
+			}
+			json.Unmarshal(block["image_url"], &obj)
+			url = obj.URL
+		}
+		return url, true
+	case provider.Responses:
+		if kind != "input_image" {
+			return "", false
+		}
+		var url string
+		json.Unmarshal(block["image_url"], &url)
+		return url, true
+	case provider.Anthropic:
+		if kind != "image" {
+			return "", false
+		}
+		var src struct {
+			Type      string `json:"type"`
+			MediaType string `json:"media_type"`
+			Data      string `json:"data"`
+			URL       string `json:"url"`
+		}
+		json.Unmarshal(block["source"], &src)
+		switch {
+		case src.Type == "base64" && src.Data != "":
+			return "data:" + src.MediaType + ";base64," + src.Data, true
+		case src.Type == "url":
+			return src.URL, true
+		}
+		return "", true
+	case provider.Gemini:
+		var data struct {
+			MimeType string `json:"mimeType"`
+			Data     string `json:"data"`
+		}
+		json.Unmarshal(block["inlineData"], &data)
+		if strings.HasPrefix(data.MimeType, "image/") {
+			return "data:" + data.MimeType + ";base64," + data.Data, true
+		}
+		if len(block["fileData"]) > 0 {
+			var file struct {
+				MimeType string `json:"mimeType"`
+			}
+			if json.Unmarshal(block["fileData"], &file) != nil || file.MimeType == "" || strings.HasPrefix(file.MimeType, "image/") {
+				return "", true // unknown file types may be images; fail closed
+			}
+		}
+	}
+	return "", false
+}
+
+// textBlock is a text block of the protocol's.
+func textBlock(proto provider.Protocol, text string) json.RawMessage {
+	var b []byte
 	switch proto {
 	case provider.Responses:
-		return json.RawMessage(`{"type":"input_text","text":"[Image omitted: this model accepts text only.]"}`)
+		b, _ = json.Marshal(map[string]string{"type": "input_text", "text": text})
 	case provider.Gemini:
-		return json.RawMessage(`{"text":"[Image omitted: this model accepts text only.]"}`)
+		b, _ = json.Marshal(map[string]string{"text": text})
+	default:
+		b, _ = json.Marshal(map[string]string{"type": "text", "text": text})
 	}
-	return json.RawMessage(`{"type":"text","text":"[Image omitted: this model accepts text only.]"}`)
+	return b
 }

internal/gateway/image_input_test.go

@@ -16,6 +16,7 @@ import (
 
 func TestKnownTextOnlyModelRejectsImagesBeforeUpstream(t *testing.T) {
 	fresh(t)
+	noVision(t)
 	var sent int
 	up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		sent++

internal/gateway/rules_test.go

@@ -314,6 +314,7 @@ func TestRuleUnreadyMember(t *testing.T) {
 func TestRuleImages(t *testing.T) {
 	img := `{"model":"group/r","messages":[{"role":"user","content":[{"type":"text","text":"look"},{"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]}]}`
 	s, _, b := ruled(t)
+	noVision(t)
 	if code, _ := postAs(t, s, "x", img); code != 400 || b.n() != 0 {
 		t.Fatalf("no rules: %d, b %d", code, b.n())
 	}
@@ -521,6 +522,7 @@ func TestRuleTurnOutgrowsUnlistedMember(t *testing.T) {
 func TestRuleImagesUnlistedMember(t *testing.T) {
 	img := `{"model":"group/r","messages":[{"role":"user","content":[{"type":"text","text":"look"},{"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]}]}`
 	s, a, b := ruled(t)
+	noVision(t)
 	p, err := provider.Find("a")
 	if err != nil {
 		t.Fatal(err)
@@ -538,6 +540,7 @@ func TestRuleImagesUnlistedMember(t *testing.T) {
 // attached when the vision provider fails.
 func TestImageRuleDoesNotLeakImageToTextOnlyFallback(t *testing.T) {
 	s, a, b := ruled(t, provider.Rule{Use: "b/big", Images: true})
+	noVision(t)
 	b.mu.Lock()
 	b.fail = 503
 	b.mu.Unlock()
@@ -551,6 +554,20 @@ func TestImageRuleDoesNotLeakImageToTextOnlyFallback(t *testing.T) {
 	}
 }
 
+// With the vision member down, nothing is left to describe the image: the
+// text-only fallback isn't sent it, or a turn without it.
+func TestImageRuleFallbackCantHaveTheImageDescribedByTheFailedMember(t *testing.T) {
+	s, a, b := ruled(t, provider.Rule{Use: "b/big", Images: true})
+	b.mu.Lock()
+	b.fail = 503
+	b.mu.Unlock()
+	img := `{"model":"group/r","messages":[{"role":"user","content":[{"type":"text","text":"look"},{"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]}]}`
+	code, out := postAs(t, s, "image-fallback", img)
+	if code != 502 || !strings.Contains(out, "couldn't describe the image") || a.n() != 0 {
+		t.Fatalf("%d %s, a %d", code, out, a.n())
+	}
+}
+
 // When the vision member of a rule for images fails, a text-only member
 // answers a turn whose image is in an earlier message, without the image.
 func TestImageRuleFallbackOmitsOlderImages(t *testing.T) {
@@ -622,6 +639,7 @@ func TestRuleTurnHoldsItsModelOnASharedAccount(t *testing.T) {
 // text-only first; and once the first rests, the second's goes first.
 func TestRuleFailsOverToTheNextRule(t *testing.T) {
 	_, a, b := ruled(t)
+	noVision(t)
 	c := &ruleUp{key: "kc"}
 	srv := httptest.NewServer(c)
 	t.Cleanup(srv.Close)

internal/gateway/search.go

@@ -96,7 +96,7 @@ func searcher() (provider.Provider, string, bool) {
 		if r < 0 || !p.Ready() || (top >= 0 && r >= top) {
 			continue
 		}
-		if m := smallModel(p); m != "" {
+		if m := smallModel(p, nil); m != "" {
 			best, model, top = p, m, r
 		}
 	}
@@ -104,10 +104,14 @@ func searcher() (provider.Provider, string, bool) {
 }
 
 // smallModel is the provider's cheapest small model of a vendor that
-// searches well, or its cheapest, or its first.
-func smallModel(p provider.Provider) string {
+// searches well, or its cheapest, or its first; only among those keep
+// holds, when it is given.
+func smallModel(p provider.Provider, keep func(catalog.Model) bool) string {
 	var ms []catalog.Model
 	for _, m := range p.Available() {
+		if keep != nil && !keep(m) {
+			continue
+		}
 		l := strings.ToLower(m.ID)
 		// a subscription's list can have what magpie wrote into its agent
 		if strings.HasPrefix(m.ID, provider.GroupPrefix) || (p.Account != nil && strings.Contains(m.ID, "/")) ||

internal/gateway/vision.go

@@ -0,0 +1,287 @@
+package gateway
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/yetone/magpie/internal/catalog"
+	"github.com/yetone/magpie/internal/provider"
+	"github.com/yetone/magpie/internal/settings"
+)
+
+// A model that can't see images is given what a model that can says of
+// them: each image in the request is described once by the vision model
+// (the Settings' Vision, or one magpie picks) and the description goes in
+// the image's place, in the current turn, older turns and tool results
+// alike. With Vision off, or no model that sees, such an image is turned
+// away as before (textOnlyBody).
+
+// VisionAgent is the User-Agent of the descriptions magpie asks for.
+const VisionAgent = "magpie-vision/1"
+
+const visionTimeout = 2 * time.Minute
+
+// visionParallel is how many images of a request are described at once.
+const visionParallel = 4
+
+// sightsKept is how many descriptions are kept: an agent sends a
+// conversation's images again every turn.
+const sightsKept = 256
+
+const visionSystem = `You describe images for an AI model that cannot see them. It will answer the user from your description alone, so leave nothing out that it may need.
+- Transcribe all text exactly as written, keeping its layout: code, terminal output, error messages, logs, UI labels, menus, file names, numbers.
+- For a screenshot of an app or page: which app or page it is, its layout, and the state of what is on it (selected, disabled, checked, highlighted, error states).
+- For a chart or table: its kind, axes and labels, and every value you can read.
+- For a diagram: its elements and how they are connected.
+- For a photo or drawing: what it shows, with the details that matter.
+Describe only what is there. Don't guess at what can't be read, say it can't be read. Don't answer questions or give advice. No preamble.`
+
+// describingKey marks a description magpie asks for, which must not have
+// its image described again.
+type describingKey struct{}
+
+func describing(ctx context.Context) bool {
+	v, _ := ctx.Value(describingKey{}).(bool)
+	return v
+}
+
+// seer is the model that describes images: the Settings' Vision while it
+// resolves, else AutoVision's. None when Vision is off or no model sees.
+func seer() (string, bool) {
+	switch v := settings.Load().Vision; v {
+	case "off":
+		return "", false
+	case "":
+	default:
+		if _, _, ok := provider.Resolve(v); ok {
+			return v, true
+		}
+	}
+	m := AutoVision()
+	return m, m != ""
+}
+
+// AutoVision is the model magpie picks to describe images when the
+// Settings name none: a small model that sees of the providers set up — a
+// Claude account's, a Codex account's, the cheapest of an API key's,
+// another account's. "" when no model sees.
+func AutoVision() string {
+	tier := func(p provider.Provider) int {
+		switch {
+		case p.Account != nil && p.Account.Agent == "claude":
+			return 0
+		case p.Account != nil && p.Account.Agent == "codex":
+			return 1
+		case p.Account == nil:
+			return 2
+		}
+		return 3
+	}
+	best, bestTier, bestCost := "", 0, 0.0
+	for _, p := range provider.All() {
+		if !p.Ready() || p.Decides() {
+			continue
+		}
+		t := tier(p)
+		if best != "" && t > bestTier {
+			continue
+		}
+		m, cost := smallModelWhere(p, sees)
+		if m == "" {
+			continue
+		}
+		if best == "" || t < bestTier || cost < bestCost {
+			best, bestTier, bestCost = p.ID+"/"+m, t, cost
+		}
+	}
+	return best
+}
+
+// sees is whether a model takes images.
+func sees(m catalog.Model) bool {
+	return m.Images && (m.ImageInput == nil || *m.ImageInput)
+}
+
+// sight is a description, once it is done.
+type sight struct {
+	done chan struct{}
+	text string
+	err  error
+}
+
+// describe is what model says of the image at src, asked once for each
+// image: a request that sends it again, or sends it while it is being
+// described, is given the same description.
+func (s *Server) describe(ctx context.Context, model, src string) (string, error) {
+	sum := sha256.Sum256([]byte(model + "\x00" + src))
+	key := hex.EncodeToString(sum[:])
+	s.sightMu.Lock()
+	if e, ok := s.sights[key]; ok {
+		s.sightMu.Unlock()
+		select {
+		case <-e.done:
+			return e.text, e.err
+		case <-ctx.Done():
+			return "", ctx.Err()
+		}
+	}
+	if s.sights == nil {
+		s.sights = map[string]*sight{}
+	}
+	e := &sight{done: make(chan struct{})}
+	s.sights[key] = e
+	s.sightOrder = append(s.sightOrder, key)
+	for len(s.sightOrder) > sightsKept {
+		delete(s.sights, s.sightOrder[0])
+		s.sightOrder = s.sightOrder[1:]
+	}
+	s.sightMu.Unlock()
+	e.text, e.err = s.askVision(ctx, model, src)
+	if e.err != nil {
+		// a failure isn't kept: the next request asks again
+		s.sightMu.Lock()
+		if s.sights[key] == e {
+			delete(s.sights, key)
+		}
+		s.sightMu.Unlock()
+	}
+	close(e.done)
+	return e.text, e.err
+}
+
+// askVision asks model through the gateway itself, as a client would, what
+// the image at src shows.
+func (s *Server) askVision(ctx context.Context, model, src string) (string, error) {
+	ctx, cancel := context.WithTimeout(context.WithValue(ctx, describingKey{}, true), visionTimeout)
+	defer cancel()
+	req := map[string]any{
+		"model":  model,
+		"stream": false,
+		"messages": []map[string]any{
+			{"role": "system", "content": visionSystem},
+			{"role": "user", "content": []map[string]any{
+				{"type": "text", "text": "Describe this image."},
+				{"type": "image_url", "image_url": map[string]string{"url": src}},
+			}},
+		},
+		"max_tokens": 4096,
+	}
+	if effort := classifyEffort(model); effort != "" {
+		req["reasoning_effort"] = effort
+	}
+	body, _ := json.Marshal(req)
+	r, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://magpie/v1/chat/completions", nil)
+	if err != nil {
+		return "", err
+	}
+	r.Header.Set("Content-Type", "application/json")
+	r.Header.Set("User-Agent", VisionAgent)
+	w := httptest.NewRecorder()
+	s.serve(w, r, provider.Chat, body)
+	if ctx.Err() != nil {
+		return "", fmt.Errorf("no answer in %s", visionTimeout)
+	}
+	var out struct {
+		Choices []struct {
+			Message struct {
+				Content string `json:"content"`
+			} `json:"message"`
+		} `json:"choices"`
+		Error *struct {
+			Message string `json:"message"`
+		} `json:"error"`
+	}
+	if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil {
+		return "", fmt.Errorf("%d, not an answer", w.Code)
+	}
+	if w.Code >= 300 || out.Error != nil {
+		msg := http.StatusText(w.Code)
+		if out.Error != nil && out.Error.Message != "" {
+			msg = out.Error.Message
+		}
+		return "", errors.New(msg)
+	}
+	if len(out.Choices) == 0 || strings.TrimSpace(out.Choices[0].Message.Content) == "" {
+		return "", errors.New("no description")
+	}
+	return strings.TrimSpace(out.Choices[0].Message.Content), nil
+}
+
+// seenBody is the body with each of its images described by model in its
+// place. An image of the latest turn that can't be described is an error:
+// the model would be asked about what it isn't given. One in an older turn
+// or a tool's result is left out, as textOnlyBody leaves it.
+func (s *Server) seenBody(ctx context.Context, proto provider.Protocol, body []byte, model string) ([]byte, error) {
+	var srcs []string
+	have := map[string]bool{}
+	walkImages(proto, body, func(im imageAt) (json.RawMessage, error) {
+		if im.Src != "" && !have[im.Src] {
+			have[im.Src] = true
+			srcs = append(srcs, im.Src)
+		}
+		return nil, nil
+	})
+	type result struct {
+		text string
+		err  error
+	}
+	results := make([]result, len(srcs))
+	var wg sync.WaitGroup
+	slots := make(chan struct{}, visionParallel)
+	for i, src := range srcs {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			slots <- struct{}{}
+			defer func() { <-slots }()
+			text, err := s.describe(ctx, model, src)
+			results[i] = result{text, err}
+		}()
+	}
+	wg.Wait()
+	of := map[string]result{}
+	for i, src := range srcs {
+		of[src] = results[i]
+	}
+	return walkImages(proto, body, func(im imageAt) (json.RawMessage, error) {
+		if im.Src == "" {
+			if im.Current() {
+				return nil, errors.New("the image is a file only its vendor can open")
+			}
+			return textBlock(proto, imageOmitted), nil
+		}
+		r := of[im.Src]
+		if r.err != nil {
+			if im.Current() {
+				return nil, r.err
+			}
+			return textBlock(proto, "[Image omitted: this model accepts text only, and the image couldn't be described.]"), nil
+		}
+		// on its own lines: a translation may join it to the text around it
+		return textBlock(proto, "\n[Image, as "+model+" describes it for this model, which can't see images:]\n"+r.text+"\n[End of the image's description]\n"), nil
+	})
+}
+
+// smallModelWhere is smallModel among the provider's models that keep
+// holds, and what it costs; "" when there is none.
+func smallModelWhere(p provider.Provider, keep func(catalog.Model) bool) (string, float64) {
+	id := smallModel(p, keep)
+	if id == "" {
+		return "", 0
+	}
+	for _, m := range p.Available() {
+		if m.ID == id && m.Price != nil {
+			return id, m.Price.Input + m.Price.Output
+		}
+	}
+	return id, 1e9
+}

internal/gateway/vision_test.go

@@ -0,0 +1,213 @@
+package gateway
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/yetone/magpie/internal/catalog"
+	"github.com/yetone/magpie/internal/provider"
+	"github.com/yetone/magpie/internal/settings"
+)
+
+// noVision turns describing off, for a test of images turned away.
+func noVision(t *testing.T) {
+	t.Helper()
+	s := settings.Load()
+	s.Vision = "off"
+	if err := settings.Save(s); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// eyeUp is an upstream with a text-only model and one that sees: the one
+// that sees describes every image as a red square, or fails when fail is
+// set; what each model is sent is kept.
+type eyeUp struct {
+	mu     sync.Mutex
+	fail   bool
+	bodies map[string][]string
+}
+
+func (u *eyeUp) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	body, _ := io.ReadAll(r.Body)
+	model := modelOf(body)
+	u.mu.Lock()
+	if u.bodies == nil {
+		u.bodies = map[string][]string{}
+	}
+	u.bodies[model] = append(u.bodies[model], string(body))
+	fail := u.fail
+	u.mu.Unlock()
+	w.Header().Set("Content-Type", "application/json")
+	if strings.HasSuffix(model, "eye") && fail {
+		w.WriteHeader(500)
+		io.WriteString(w, `{"error":{"message":"eye is down"}}`)
+		return
+	}
+	answer := "ok"
+	if strings.HasSuffix(model, "eye") {
+		answer = "A red square with the word HELLO in it."
+	}
+	if strings.Contains(string(body), `"stream":true`) {
+		w.Header().Set("Content-Type", "text/event-stream")
+		io.WriteString(w, `data: {"id":"x","choices":[{"index":0,"delta":{"role":"assistant","content":"`+answer+`"}}]}`+"\n\n")
+		io.WriteString(w, `data: {"id":"x","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`+"\n\n")
+		io.WriteString(w, "data: [DONE]\n\n")
+		return
+	}
+	io.WriteString(w, `{"id":"x","choices":[{"index":0,"message":{"role":"assistant","content":"`+answer+`"},"finish_reason":"stop"}]}`)
+}
+
+func (u *eyeUp) sent(model string) []string {
+	u.mu.Lock()
+	defer u.mu.Unlock()
+	return append([]string(nil), u.bodies[model]...)
+}
+
+func eyed(t *testing.T) (*Server, *eyeUp) {
+	t.Helper()
+	fresh(t)
+	u := &eyeUp{}
+	up := httptest.NewServer(u)
+	t.Cleanup(up.Close)
+	if err := provider.Save(provider.Provider{ID: "probe", Name: "Probe", Chat: up.URL + "/v1", Key: "key", Models: []string{"text", "eye"}}); err != nil {
+		t.Fatal(err)
+	}
+	if err := catalog.SaveLive("probe", up.URL+"/v1", []catalog.Model{
+		{ID: "text", ImageInput: imageInputBool(false)},
+		{ID: "eye", Images: true, ImageInput: imageInputBool(true)},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	return New(), u
+}
+
+func TestTextOnlyModelIsGivenTheImagesDescription(t *testing.T) {
+	s, u := eyed(t)
+	cases := []struct{ path, body string }{
+		{"/v1/chat/completions", `{"model":"probe/text","messages":[{"role":"user","content":[{"type":"text","text":"read"},{"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]}]}`},
+		{"/v1/responses", `{"model":"probe/text","input":[{"role":"user","content":[{"type":"input_text","text":"read"},{"type":"input_image","image_url":"data:image/png;base64,aGVsbG8="}]}]}`},
+		{"/v1/messages", `{"model":"probe/text","max_tokens":16,"messages":[{"role":"user","content":[{"type":"text","text":"read"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"aGVsbG8="}}]}]}`},
+		{"/v1beta/models/probe/text:generateContent", `{"contents":[{"role":"user","parts":[{"text":"read"},{"inlineData":{"mimeType":"image/png","data":"aGVsbG8="}}]}]}`},
+	}
+	for _, tc := range cases {
+		rec := httptest.NewRecorder()
+		s.Handler().ServeHTTP(rec, httptest.NewRequest("POST", tc.path, strings.NewReader(tc.body)))
+		if rec.Code != 200 {
+			t.Errorf("%s: %d %s", tc.path, rec.Code, rec.Body.String())
+		}
+	}
+	got := u.sent("text")
+	if len(got) != len(cases) {
+		t.Fatalf("text model sent %d requests, want %d", len(got), len(cases))
+	}
+	for i, b := range got {
+		if !strings.Contains(b, "HELLO") || !strings.Contains(b, "probe/eye describes it") || strings.Contains(b, "aGVsbG8=") {
+			t.Errorf("%s: text model was sent %s", cases[i].path, b)
+		}
+	}
+	// the same image in four requests is described once
+	if n := len(u.sent("eye")); n != 1 {
+		t.Fatalf("the image was described %d times", n)
+	}
+	if !strings.Contains(u.sent("eye")[0], "aGVsbG8=") {
+		t.Fatalf("the describer wasn't sent the image: %s", u.sent("eye")[0])
+	}
+}
+
+func TestVisionSettingPicksTheDescriber(t *testing.T) {
+	s, u := eyed(t)
+	// a second model that sees, which the setting names
+	up2 := &eyeUp{}
+	srv := httptest.NewServer(up2)
+	t.Cleanup(srv.Close)
+	if err := provider.Save(provider.Provider{ID: "other", Name: "Other", Chat: srv.URL + "/v1", Key: "key", Models: []string{"bigeye"}}); err != nil {
+		t.Fatal(err)
+	}
+	if err := catalog.SaveLive("other", srv.URL+"/v1", []catalog.Model{{ID: "bigeye", Images: true, ImageInput: imageInputBool(true)}}); err != nil {
+		t.Fatal(err)
+	}
+	st := settings.Load()
+	st.Vision = "other/bigeye"
+	if err := settings.Save(st); err != nil {
+		t.Fatal(err)
+	}
+	code, body := postAs(t, s, "", `{"model":"probe/text","messages":[{"role":"user","content":[{"type":"image_url","image_url":"https://example.com/a.png"}]}]}`)
+	if code != 200 {
+		t.Fatalf("%d %s", code, body)
+	}
+	if len(u.sent("eye")) != 0 || len(up2.sent("bigeye")) != 1 {
+		t.Fatalf("eye asked %d times, bigeye %d", len(u.sent("eye")), len(up2.sent("bigeye")))
+	}
+	if !strings.Contains(u.sent("text")[0], "other/bigeye describes it") {
+		t.Fatalf("text model was sent %s", u.sent("text")[0])
+	}
+}
+
+func TestImageThatCantBeDescribed(t *testing.T) {
+	s, u := eyed(t)
+	u.fail = true
+	code, body := postAs(t, s, "", `{"model":"probe/text","messages":[{"role":"user","content":[{"type":"text","text":"read"},{"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]}]}`)
+	if code != 502 || !strings.Contains(body, "couldn't describe the image") || len(u.sent("text")) != 0 {
+		t.Fatalf("current image: %d %s; text model sent %d", code, body, len(u.sent("text")))
+	}
+	// an older turn's image that can't be described is left out
+	code, body = postAs(t, s, "", `{"model":"probe/text","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,b2xk"}}]},{"role":"assistant","content":"seen"},{"role":"user","content":"and now?"}]}`)
+	if code != 200 {
+		t.Fatalf("older image: %d %s", code, body)
+	}
+	if got := u.sent("text"); len(got) != 1 || !strings.Contains(got[0], "couldn't be described") {
+		t.Fatalf("text model was sent %v", got)
+	}
+	// a failure isn't kept
+	u.mu.Lock()
+	u.fail = false
+	u.mu.Unlock()
+	if code, body := postAs(t, s, "", `{"model":"probe/text","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]}]}`); code != 200 {
+		t.Fatalf("after recovery: %d %s", code, body)
+	}
+}
+
+func TestGroupsTextOnlyMemberIsGivenTheDescription(t *testing.T) {
+	s, u := eyed(t)
+	if err := provider.SaveGroup(provider.Group{Name: "Mixed", Members: []string{"probe/text", "probe/eye"}, Routing: provider.Ordered}); err != nil {
+		t.Fatal(err)
+	}
+	code, body := postAs(t, s, "", `{"model":"group/mixed","messages":[{"role":"user","content":[{"type":"text","text":"read"},{"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]}]}`)
+	if code != 200 {
+		t.Fatalf("%d %s", code, body)
+	}
+	if got := u.sent("text"); len(got) != 1 || !strings.Contains(got[0], "HELLO") {
+		t.Fatalf("the group's first member was sent %v", got)
+	}
+}
+
+func TestDescriberThatCantSeeIsntAskedToDescribe(t *testing.T) {
+	s, u := eyed(t)
+	// the setting names a text-only model: its own request is turned away,
+	// not described by itself again and again
+	st := settings.Load()
+	st.Vision = "probe/text"
+	if err := settings.Save(st); err != nil {
+		t.Fatal(err)
+	}
+	code, body := postAs(t, s, "", `{"model":"probe/text","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]}]}`)
+	if code != 502 || len(u.sent("text")) != 0 {
+		t.Fatalf("%d %s; text model sent %d", code, body, len(u.sent("text")))
+	}
+}
+
+func TestSeerPicksAModelThatSees(t *testing.T) {
+	eyed(t)
+	if m, ok := seer(); !ok || m != "probe/eye" {
+		t.Fatalf("seer = %q %v", m, ok)
+	}
+	noVision(t)
+	if _, ok := seer(); ok {
+		t.Fatal("seer with vision off")
+	}
+}

internal/gui/api.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"embed"
 	"encoding/json"
+	"fmt"
 	"mime"
 	"net/http"
 	"os"
@@ -119,6 +120,10 @@ type settingsJSON struct {
 	ProxySource string `json:"proxySource"`
 	// whether magpie opens at login: the system's record, not a setting
 	Login bool `json:"login"`
+	// the model that describes images when Vision names none, and those
+	// that can be named
+	VisionAuto   string     `json:"visionAuto,omitempty"`
+	VisionModels []modelRef `json:"visionModels"`
 	// where other machines reach the gateway while it is shared
 	LANURLs []string `json:"lanURLs,omitempty"`
 	// when the Codex warm-up last started an account's window
@@ -135,6 +140,16 @@ func settingsState() settingsJSON {
 		s.LANURLs = gateway.LANURLs()
 	}
 	s.CodexWarmed, s.ClaudeWarmed = latest(provider.CodexWarmed()), latest(provider.ClaudeWarmed())
+	s.VisionAuto, s.VisionModels = gateway.AutoVision(), []modelRef{}
+	for _, e := range provider.Served() {
+		if e.Images && (e.ImageInput == nil || *e.ImageInput) && (e.Group != "" || e.Provider.Ready()) {
+			m := modelRef{ID: e.ID, Name: e.Name, Provider: e.Provider.ID, PName: e.Provider.Name, Icon: e.Provider.Icon}
+			if e.Group != "" {
+				m.Provider, m.PName = "", e.Group
+			}
+			s.VisionModels = append(s.VisionModels, m)
+		}
+	}
 	return s
 }
 
@@ -334,6 +349,12 @@ func Handler(w Windows, gw *gateway.Server) http.Handler {
 		in.LAN, in.LANKey = cur.LAN, cur.LANKey
 		// used or left is the Usage page's toggle as much as Settings', set on its own
 		in.QuotaLeft = cur.QuotaLeft
+		if v := strings.TrimSpace(in.Vision); v != "" && v != "off" && v != cur.Vision {
+			if _, _, ok := provider.Resolve(v); !ok {
+				fail(rw, fmt.Errorf("no model %s to describe images", v))
+				return
+			}
+		}
 		if err := settings.Save(in); err != nil {
 			fail(rw, err)
 			return

internal/gui/assets/app.js

@@ -5372,6 +5372,7 @@ function renderSettings() {
     (v) => savePrefs({ ...keep, claudeWarmAt: v }));
   renderTrayUsage(s, keep);
   renderProxy(s, keep);
+  renderImages(s, keep);
   renderRedact(s, keep);
   renderLAN(s);
   renderSync();
@@ -5716,6 +5717,43 @@ function renderWarmAt(box, sub, at, onReset, what, save) {
   box.append(segs([["off", t("Off")], ["on", t("On")]], at ? "on" : "off", (v) => save(v === "on" ? at || "06:00" : "")));
 }
 
+// renderImages: the model that describes images to a model that can't see
+// them — the one magpie picks, one named, or none, and then such an image
+// is turned away.
+function renderImages(s, keep) {
+  const box = $("#imagesList");
+  box.replaceChildren();
+  const models = s.visionModels || [];
+  const named = (id) => {
+    const m = models.find((x) => x.id === id);
+    return m ? `${m.name || m.id} · ${m.providerName}` : id;
+  };
+  const r = el("div", "row pref");
+  const who = el("div", "who");
+  const v = s.vision || "";
+  who.append(el("div", "name", t("Image recognition")), el("div", "sub",
+    v === "off" ? t("A model that can't see images is sent none: a request with one in its latest message is turned away")
+    : t("When the model in use can't see images, this one describes them to it, once for each image")));
+  const b = el("button", "rt-cond on");
+  const icOf = (id) => models.find((x) => x.id === id)?.icon;
+  if (v === "off") b.append(el("span", "", t("Off")));
+  else if (v) b.append(icon(icOf(v) || "generic"), el("span", "", named(v)));
+  else {
+    if (s.visionAuto) b.append(icon(icOf(s.visionAuto) || "generic"));
+    b.append(el("span", "", s.visionAuto ? t("Automatic") + " · " + named(s.visionAuto) : t("Automatic") + " · " + t("no model that sees")));
+  }
+  const opt = (x) => ({ value: x.id, label: x.name || x.id, note: x.providerName, icon: x.icon, group: x.providerName, ref: x.id });
+  b.onclick = (ev) => openPicker({ id: "", name: "", fields: [] }, { key: "vision", label: "model", value: v, options: [
+    { value: "", label: t("Automatic"), note: s.visionAuto ? named(s.visionAuto) : t("no model that sees"), reset: true },
+    { value: "off", label: t("Off"), note: t("images turned away") },
+    ...models.map(opt)],
+  onPick: (id) => { if (id !== v) savePrefs({ ...keep, vision: id }); } }, b, ev);
+  const val = el("div", "val");
+  val.append(b);
+  r.append(who, val);
+  box.append(r);
+}
+
 // renderRedact: what the gateway masks before a request goes to a vendor —
 // secrets, personal data, the user's own words — and puts back in what the
 // vendor answers.
@@ -5842,7 +5880,7 @@ function prefsKeep(s) {
   return { theme: s.theme, lang: s.lang, tray: s.tray, dock: !!s.dock, dockWindow: !!s.dockWindow, proxy: s.proxy || "",
     redact: !!s.redact, redactPersonal: !!s.redactPersonal, redactWords: s.redactWords || [], codexWarmup: s.codexWarmup || "",
     claudeWarmup: s.claudeWarmup || "", codexWarmAt: s.codexWarmAt || "", claudeWarmAt: s.claudeWarmAt || "", noStats: !!s.noStats,
-    trayUsage: s.trayUsage || "", trayUsageEvery: s.trayUsageEvery || 3 };
+    trayUsage: s.trayUsage || "", trayUsageEvery: s.trayUsageEvery || 3, vision: s.vision || "" };
 }
 
 // savePrefs sends what the page was drawn with (prefsBase) and the choice

internal/gui/assets/i18n.js

@@ -84,6 +84,13 @@ const I18N = {
     "No system proxy found; requests to vendors go direct": "未发现系统代理，对模型厂商的请求直连",
     "Off": "关闭",
     "On": "开启",
+    "Images": "图像",
+    "Image recognition": "识图模型",
+    "When the model in use can't see images, this one describes them to it, once for each image": "当前模型看不了图时，由它把图片描述成文字交给当前模型，每张图只描述一次",
+    "A model that can't see images is sent none: a request with one in its latest message is turned away": "不给看不了图的模型发图片：最新一条消息带图的请求会被拒绝",
+    "Automatic": "自动",
+    "no model that sees": "没有能看图的模型",
+    "images turned away": "带图的请求被拒绝",
     "Privacy": "隐私",
     "Mask secrets": "脱敏密钥",
     "Local network": "局域网",

internal/gui/assets/index.html

@@ -186,6 +186,8 @@
   </div>
   <div class="row-head"><span class="label" data-t>Local network</span></div>
   <div class="list prefs" id="lanList"></div>
+  <div class="row-head"><span class="label" data-t>Images</span></div>
+  <div class="list prefs" id="imagesList"></div>
   <div class="row-head"><span class="label" data-t>Privacy</span></div>
   <div class="list prefs" id="redactList"></div>
   <div class="row-head"><span class="label" data-t>Sync and backup</span></div>

internal/settings/settings.go

@@ -66,6 +66,10 @@ type Settings struct {
 	// NoStats stops the one event a day that counts magpie's users (see
 	// internal/stats).
 	NoStats bool `json:"noStats,omitempty"`
+	// Vision is the model that describes an image to a model that can't see
+	// it: a model's id (provider/model, group/<id>), "off" to turn such an
+	// image away, or empty for one magpie picks (see gateway.seer).
+	Vision string `json:"vision,omitempty"`
 	// TrayUsage is the subscription or plan whose windows are shown beside
 	// the tray icon, by its provider and account ("claude|a@b.c"); "" none.
 	TrayUsage string `json:"trayUsage,omitempty"`
@@ -203,6 +207,10 @@ func Save(s Settings) error {
 			return fmt.Errorf("proxy must look like http://127.0.0.1:7890 or socks5://127.0.0.1:1080, not %q", s.Proxy)
 		}
 	}
+	s.Vision = strings.TrimSpace(s.Vision)
+	if s.Vision != "" && s.Vision != "off" && !strings.Contains(s.Vision, "/") {
+		return fmt.Errorf("the vision model must be a model's id such as openai/gpt-5-mini, or off, not %q", s.Vision)
+	}
 	s.AgentOrder, s.AgentsHidden, s.AgentsShown = ids(s.AgentOrder), ids(s.AgentsHidden), ids(s.AgentsShown)
 	if err := os.MkdirAll(Dir(), 0o755); err != nil {
 		return err