diff --git a/internal/agent/dsh_test.go b/internal/agent/dsh_test.go index 38eaaa2c..ee63e397 100644 --- a/internal/agent/dsh_test.go +++ b/internal/agent/dsh_test.go @@ -182,6 +182,7 @@ func TestDshModelLimits(t *testing.T) { t.Setenv("DSH_HOME", "") t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache")) + visionOff(t) // else v/see describes images to v/plain, which takes them then if err := provider.Save(provider.Provider{ID: "v", Name: "V", Chat: "https://example.test/v1", Key: "k", Models: []string{"see", "plain"}}); err != nil { t.Fatal(err) } diff --git a/internal/agent/minimax_test.go b/internal/agent/minimax_test.go index 237931d5..7b6d7f27 100644 --- a/internal/agent/minimax_test.go +++ b/internal/agent/minimax_test.go @@ -119,6 +119,7 @@ func readMiniMax(t *testing.T, path string) (miniMaxFile, string) { // default and variant they had. func TestMiniMaxCode(t *testing.T) { home, path := miniMaxHome(t) + visionOff(t) // flash is text-only only while no model describes images to it a := miniMax(home) if a.Path != path || !a.Detected() { t.Fatalf("path %q, detected %v", a.Path, a.Detected()) diff --git a/internal/agent/route.go b/internal/agent/route.go index bdecca00..a3ba6da2 100644 --- a/internal/agent/route.go +++ b/internal/agent/route.go @@ -92,8 +92,14 @@ func magpieModels(agent string) []catalog.Model { var out []catalog.Model shown, _ := provider.CatalogFor(agent) labels := provider.Labels(shown) + // a model magpie describes images to takes them (provider.Described) + seen := provider.Described != nil && provider.Described() for i, e := range shown { - m := catalog.Model{ID: e.ID, Name: labels[i], Provider: firstOf(e.Provider.Catalogs()), Efforts: e.Efforts, Images: e.Images, ImageInput: e.ImageInput, Context: e.Context, Output: e.Output} + m := catalog.Model{ID: e.ID, Name: labels[i], Provider: firstOf(e.Provider.Catalogs()), Efforts: e.Efforts, Images: e.Images || seen, ImageInput: e.ImageInput, Context: e.Context, Output: e.Output} + if seen && !e.Images { + yes := true + m.ImageInput = &yes + } // APIs is the one to ask it on for the gateway to relay the request // as it is; none for a group, whose members may each want another if e.Group == "" { diff --git a/internal/agent/vision_test.go b/internal/agent/vision_test.go new file mode 100644 index 00000000..fc37982b --- /dev/null +++ b/internal/agent/vision_test.go @@ -0,0 +1,57 @@ +package agent + +import ( + "path/filepath" + "testing" + + "github.com/yetone/magpie/internal/catalog" + "github.com/yetone/magpie/internal/provider" + "github.com/yetone/magpie/internal/settings" +) + +// visionOff has no model describe images to one that can't see them, so an +// agent is told which models take images as they are. +func visionOff(t *testing.T) { + t.Helper() + s := settings.Load() + s.Vision = "off" + if err := settings.Save(s); err != nil { + t.Fatal(err) + } +} + +// An agent's model file says a text-only model takes images while a model +// that sees describes them to it, and text only with Vision off (Discord, +// Fate). +func TestDescribedModelTakesImages(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) + t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache")) + if err := provider.Save(provider.Provider{ID: "v", Name: "V", Chat: "https://example.test/v1", Key: "k", Models: []string{"see", "plain"}}); err != nil { + t.Fatal(err) + } + if err := catalog.SaveLive("v", "https://example.test/v1", []catalog.Model{ + {ID: "see", Name: "see", Images: true, ImageInput: imageInputBool(true)}, + {ID: "plain", Name: "plain", ImageInput: imageInputBool(false)}, + }); err != nil { + t.Fatal(err) + } + plain := func() catalog.Model { + for _, m := range magpieModels("dsh") { + if m.ID == "v/plain" { + return m + } + } + t.Fatal("v/plain not listed") + return catalog.Model{} + } + if m := plain(); !m.Images || m.ImageInput == nil || !*m.ImageInput { + t.Fatalf("with v/see describing, v/plain is %+v", m) + } + visionOff(t) + if m := plain(); m.Images || m.ImageInput == nil || *m.ImageInput { + t.Fatalf("with Vision off, v/plain is %+v", m) + } +} diff --git a/internal/gateway/vision.go b/internal/gateway/vision.go index 96622c31..655ffb45 100644 --- a/internal/gateway/vision.go +++ b/internal/gateway/vision.go @@ -25,6 +25,15 @@ import ( // alike. With Vision off, or no model that sees, such an image is turned // away as before (textOnlyBody). +// Agents are told a model that can't see takes images while one that can +// describes them to it. +func init() { + provider.Described = func() bool { + _, ok := seer() + return ok + } +} + // VisionAgent is the User-Agent of the descriptions magpie asks for. const VisionAgent = "magpie-vision/1" diff --git a/internal/gateway/vision_agents_test.go b/internal/gateway/vision_agents_test.go new file mode 100644 index 00000000..2921b138 --- /dev/null +++ b/internal/gateway/vision_agents_test.go @@ -0,0 +1,53 @@ +package gateway + +import ( + "encoding/json" + "slices" + "testing" + + "github.com/yetone/magpie/internal/codexcat" + "github.com/yetone/magpie/internal/provider" +) + +// A text-only model is offered to agents as taking images while magpie +// describes images to it (Discord, Fate: Codex turned an image away for a +// text model although Settings › Vision was set), and as text-only again +// with Vision off. +func TestDescribedModelsTakeImagesInAgentsLists(t *testing.T) { + eyed(t) + images := func() map[string]bool { + out := map[string]bool{} + for _, m := range provider.CodexListed() { + out[m.ID] = m.Images + } + return out + } + modalities := func(id string) []string { + b, _ := json.Marshal(codexcat.Entries(provider.CodexListed(), 0)) + var es []struct { + Slug string `json:"slug"` + In []string `json:"input_modalities"` + } + _ = json.Unmarshal(b, &es) + for _, e := range es { + if e.Slug == id { + return e.In + } + } + return nil + } + if got := images(); !got["probe/text"] || !got["probe/eye"] { + t.Fatalf("with Vision on, Codex is told: %v", got) + } + if in := modalities("probe/text"); !slices.Contains(in, "image") { + t.Fatalf("probe/text's input_modalities = %v", in) + } + on := provider.CodexListTag() + noVision(t) + if got := images(); got["probe/text"] || !got["probe/eye"] { + t.Fatalf("with Vision off, Codex is told: %v", got) + } + if provider.CodexListTag() == on { + t.Fatal("Codex's list tag didn't change with Vision") + } +} diff --git a/internal/gui/api.go b/internal/gui/api.go index a1297fd1..f6db422b 100644 --- a/internal/gui/api.go +++ b/internal/gui/api.go @@ -791,6 +791,11 @@ func Handler(w Windows, gw *gateway.Server) http.Handler { fail(rw, err) return } + // whether agents are told every model takes images follows Vision + // (provider.Described) + if strings.TrimSpace(in.Vision) != strings.TrimSpace(cur.Vision) { + catalog.Touched() + } if (in.Dock != cur.Dock || in.DockWindow != cur.DockWindow) && onDock != nil { onDock(in) } diff --git a/internal/provider/codex_models.go b/internal/provider/codex_models.go index 1d1eeb95..84118865 100644 --- a/internal/provider/codex_models.go +++ b/internal/provider/codex_models.go @@ -412,11 +412,12 @@ func codexListed(shown []Entry, members func(id string) []Member) []catalog.Mode // named among all shown: the account's own, which the backend lists, // are in Codex's picker beside these labels := Labels(shown) + seen := described() for i, e := range shown { if e.Group == "" && e.Provider.Account != nil && e.Provider.Account.Agent == "codex" { continue } - m := catalog.Model{ID: e.ID, Name: labels[i], Efforts: e.Efforts, Images: e.Images, Context: e.Context} + m := catalog.Model{ID: e.ID, Name: labels[i], Efforts: e.Efforts, Images: e.Images || seen, Context: e.Context} if e.Group != "" { for _, mb := range members(e.ID) { if a := mb.Provider.Account; a != nil && a.Agent == "codex" && strings.HasPrefix(mb.Model, "gpt-") { diff --git a/internal/provider/visible.go b/internal/provider/visible.go index ca0af7a0..afd315ad 100644 --- a/internal/provider/visible.go +++ b/internal/provider/visible.go @@ -45,6 +45,16 @@ func Shows(names []string, e Entry) bool { return false } +// Described is set by the gateway: whether an image sent to a model that +// can't see is described to it by one that can (Settings › Vision). Agents +// are then told every model takes images; told a model is text-only, they +// turn the user's image away before magpie is asked (Codex: "does not +// support image input"). +var Described func() bool + +// described is Described, false before the gateway sets it. +func described() bool { return Described != nil && Described() } + // CatalogFor is the catalog as agent is shown it, and what is kept from it // (none when its lists aren't narrowed): its visibility's, less the models // taken out of its lists one by one (HiddenModels).