vision: while a model that sees describes images, agents are told every model takes them (Discord, Fate: with Settings › Vision set, a text-only model in Codex still said it "does not support image input") — the gateway already described an image for a text-only model on every wire format, but the agents were told the model was text-only (Codex input_modalities ["text"], opencode modalities, dsh/omp input, droid NoImageSupport, cline/zed/qoder/workbuddy flags…), so they turned the image away before magpie was asked. provider.Described, set by the gateway to whether seer() finds a describer (Vision names one, or Automatic finds a model that sees; not with Vision off), makes magpieModels (every agent's model file) and codexListed (the ChatGPT backend's /models list and its ETag) say images for every model; Vision changed in Settings rewrites the agents' lists (catalog.Touched). TestDescribedModelsTakeImagesInAgentsLists (Codex list, input_modalities, tag) and TestDescribedModelTakesImages (agent files) fail on the old code and pass; TestDshModelLimits and TestMiniMaxCode turn Vision off to keep testing a text-only model as such. go vet, builds (darwin, GOOS=linux, GOOS=windows) and go test -tags nogui -count=1 ./... under a temp HOME pass. Not tried in the Codex App with a real account.

internal/agent/dsh_test.go

@@ -182,6 +182,7 @@ func TestDshModelLimits(t *testing.T) {
 	t.Setenv("DSH_HOME", "")
 	t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config"))
 	t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache"))
+	visionOff(t) // else v/see describes images to v/plain, which takes them then
 	if err := provider.Save(provider.Provider{ID: "v", Name: "V", Chat: "https://example.test/v1", Key: "k", Models: []string{"see", "plain"}}); err != nil {
 		t.Fatal(err)
 	}

internal/agent/minimax_test.go

@@ -119,6 +119,7 @@ func readMiniMax(t *testing.T, path string) (miniMaxFile, string) {
 // default and variant they had.
 func TestMiniMaxCode(t *testing.T) {
 	home, path := miniMaxHome(t)
+	visionOff(t) // flash is text-only only while no model describes images to it
 	a := miniMax(home)
 	if a.Path != path || !a.Detected() {
 		t.Fatalf("path %q, detected %v", a.Path, a.Detected())

internal/agent/route.go

@@ -92,8 +92,14 @@ func magpieModels(agent string) []catalog.Model {
 	var out []catalog.Model
 	shown, _ := provider.CatalogFor(agent)
 	labels := provider.Labels(shown)
+	// a model magpie describes images to takes them (provider.Described)
+	seen := provider.Described != nil && provider.Described()
 	for i, e := range shown {
-		m := catalog.Model{ID: e.ID, Name: labels[i], Provider: firstOf(e.Provider.Catalogs()), Efforts: e.Efforts, Images: e.Images, ImageInput: e.ImageInput, Context: e.Context, Output: e.Output}
+		m := catalog.Model{ID: e.ID, Name: labels[i], Provider: firstOf(e.Provider.Catalogs()), Efforts: e.Efforts, Images: e.Images || seen, ImageInput: e.ImageInput, Context: e.Context, Output: e.Output}
+		if seen && !e.Images {
+			yes := true
+			m.ImageInput = &yes
+		}
 		// APIs is the one to ask it on for the gateway to relay the request
 		// as it is; none for a group, whose members may each want another
 		if e.Group == "" {

internal/agent/vision_test.go

@@ -0,0 +1,57 @@
+package agent
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/yetone/magpie/internal/catalog"
+	"github.com/yetone/magpie/internal/provider"
+	"github.com/yetone/magpie/internal/settings"
+)
+
+// visionOff has no model describe images to one that can't see them, so an
+// agent is told which models take images as they are.
+func visionOff(t *testing.T) {
+	t.Helper()
+	s := settings.Load()
+	s.Vision = "off"
+	if err := settings.Save(s); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// An agent's model file says a text-only model takes images while a model
+// that sees describes them to it, and text only with Vision off (Discord,
+// Fate).
+func TestDescribedModelTakesImages(t *testing.T) {
+	home := t.TempDir()
+	t.Setenv("HOME", home)
+	t.Setenv("USERPROFILE", home)
+	t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config"))
+	t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache"))
+	if err := provider.Save(provider.Provider{ID: "v", Name: "V", Chat: "https://example.test/v1", Key: "k", Models: []string{"see", "plain"}}); err != nil {
+		t.Fatal(err)
+	}
+	if err := catalog.SaveLive("v", "https://example.test/v1", []catalog.Model{
+		{ID: "see", Name: "see", Images: true, ImageInput: imageInputBool(true)},
+		{ID: "plain", Name: "plain", ImageInput: imageInputBool(false)},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	plain := func() catalog.Model {
+		for _, m := range magpieModels("dsh") {
+			if m.ID == "v/plain" {
+				return m
+			}
+		}
+		t.Fatal("v/plain not listed")
+		return catalog.Model{}
+	}
+	if m := plain(); !m.Images || m.ImageInput == nil || !*m.ImageInput {
+		t.Fatalf("with v/see describing, v/plain is %+v", m)
+	}
+	visionOff(t)
+	if m := plain(); m.Images || m.ImageInput == nil || *m.ImageInput {
+		t.Fatalf("with Vision off, v/plain is %+v", m)
+	}
+}

internal/gateway/vision.go

@@ -25,6 +25,15 @@ import (
 // alike. With Vision off, or no model that sees, such an image is turned
 // away as before (textOnlyBody).
 
+// Agents are told a model that can't see takes images while one that can
+// describes them to it.
+func init() {
+	provider.Described = func() bool {
+		_, ok := seer()
+		return ok
+	}
+}
+
 // VisionAgent is the User-Agent of the descriptions magpie asks for.
 const VisionAgent = "magpie-vision/1"
 

internal/gateway/vision_agents_test.go

@@ -0,0 +1,53 @@
+package gateway
+
+import (
+	"encoding/json"
+	"slices"
+	"testing"
+
+	"github.com/yetone/magpie/internal/codexcat"
+	"github.com/yetone/magpie/internal/provider"
+)
+
+// A text-only model is offered to agents as taking images while magpie
+// describes images to it (Discord, Fate: Codex turned an image away for a
+// text model although Settings › Vision was set), and as text-only again
+// with Vision off.
+func TestDescribedModelsTakeImagesInAgentsLists(t *testing.T) {
+	eyed(t)
+	images := func() map[string]bool {
+		out := map[string]bool{}
+		for _, m := range provider.CodexListed() {
+			out[m.ID] = m.Images
+		}
+		return out
+	}
+	modalities := func(id string) []string {
+		b, _ := json.Marshal(codexcat.Entries(provider.CodexListed(), 0))
+		var es []struct {
+			Slug string   `json:"slug"`
+			In   []string `json:"input_modalities"`
+		}
+		_ = json.Unmarshal(b, &es)
+		for _, e := range es {
+			if e.Slug == id {
+				return e.In
+			}
+		}
+		return nil
+	}
+	if got := images(); !got["probe/text"] || !got["probe/eye"] {
+		t.Fatalf("with Vision on, Codex is told: %v", got)
+	}
+	if in := modalities("probe/text"); !slices.Contains(in, "image") {
+		t.Fatalf("probe/text's input_modalities = %v", in)
+	}
+	on := provider.CodexListTag()
+	noVision(t)
+	if got := images(); got["probe/text"] || !got["probe/eye"] {
+		t.Fatalf("with Vision off, Codex is told: %v", got)
+	}
+	if provider.CodexListTag() == on {
+		t.Fatal("Codex's list tag didn't change with Vision")
+	}
+}

internal/gui/api.go

@@ -791,6 +791,11 @@ func Handler(w Windows, gw *gateway.Server) http.Handler {
 			fail(rw, err)
 			return
 		}
+		// whether agents are told every model takes images follows Vision
+		// (provider.Described)
+		if strings.TrimSpace(in.Vision) != strings.TrimSpace(cur.Vision) {
+			catalog.Touched()
+		}
 		if (in.Dock != cur.Dock || in.DockWindow != cur.DockWindow) && onDock != nil {
 			onDock(in)
 		}

internal/provider/codex_models.go

@@ -412,11 +412,12 @@ func codexListed(shown []Entry, members func(id string) []Member) []catalog.Mode
 	// named among all shown: the account's own, which the backend lists,
 	// are in Codex's picker beside these
 	labels := Labels(shown)
+	seen := described()
 	for i, e := range shown {
 		if e.Group == "" && e.Provider.Account != nil && e.Provider.Account.Agent == "codex" {
 			continue
 		}
-		m := catalog.Model{ID: e.ID, Name: labels[i], Efforts: e.Efforts, Images: e.Images, Context: e.Context}
+		m := catalog.Model{ID: e.ID, Name: labels[i], Efforts: e.Efforts, Images: e.Images || seen, Context: e.Context}
 		if e.Group != "" {
 			for _, mb := range members(e.ID) {
 				if a := mb.Provider.Account; a != nil && a.Agent == "codex" && strings.HasPrefix(mb.Model, "gpt-") {

internal/provider/visible.go

@@ -45,6 +45,16 @@ func Shows(names []string, e Entry) bool {
 	return false
 }
 
+// Described is set by the gateway: whether an image sent to a model that
+// can't see is described to it by one that can (Settings › Vision). Agents
+// are then told every model takes images; told a model is text-only, they
+// turn the user's image away before magpie is asked (Codex: "does not
+// support image input").
+var Described func() bool
+
+// described is Described, false before the gateway sets it.
+func described() bool { return Described != nil && Described() }
+
 // CatalogFor is the catalog as agent is shown it, and what is kept from it
 // (none when its lists aren't narrowed): its visibility's, less the models
 // taken out of its lists one by one (HiddenModels).