diff --git a/docs/subsystems/gateway-routing.md b/docs/subsystems/gateway-routing.md index 572ba5a4..8b622c5a 100644 --- a/docs/subsystems/gateway-routing.md +++ b/docs/subsystems/gateway-routing.md @@ -65,6 +65,7 @@ served by a plugin are in [Provider and plugin ownership](provider-plugins.md). - Group members are `provider/model[:effort]`. A member with a fixed effort is asked for that effort, bounded by the group's levels (#671). - An agent whose config can't carry an effort has one picked in magpie (`provider.AgentEffort`, settings' `agentEfforts`; Cursor Private Inference's row, #1003). It is asked for on that agent's requests to a model that reasons (`Provider.Thinks`), in place of the level the request sent, fitted to the model's levels and bounded by the group's. Order: a member's fixed effort, then a model's `:effort` suffix, then the turn's pick (a routing rule's), then the agent's pick, then the request's own. A model that doesn't reason is sent as asked; the default (none picked) leaves every request as it came. The pick beats the request's own because Cursor sends its own default as often as a real choice. Picking Default hands the choice back to Cursor. - Gemini's OpenAI-compatible API (AI Studio's, or a proxy in front of Gemini on this computer or the LAN: `geminiCompat`) is asked for the model's thoughts in `extra_body.google.thinking_config`. A 400 naming it (`refusesThinkingConfig`) marks the provider unfit for the field (`thinking_config`), and it is asked again with `reasoning_effort` alone. Gemini 3 can't stop thinking, so reasoning turned off goes at `minimal`, or at the model's lowest level where its levels are known and have no `minimal` (`geminiOffLevel`). Google's 400 for `minimal` (`geminiMinimalRefused`; gemini-3.8-flash's "Thinking level is unsupported: THINKING_LEVEL_MINIMAL") names thinking too, but isn't the field turned away: the model is marked as refusing reasoning off (`offRefused`) and asked again at its lowest level, and so from then on, still asked for its thoughts. Vertex AI's OpenAI-compatible API marks each streamed chunk of thoughts in the delta's `extra_content.google.thought`, and such a chunk is read as thinking. +- A model's reply limit is published within the window beside it (#1438, `provider.OutputWithin`, `Entry.PublishedOutput`): `/v1/models`' `max_output_tokens` (`modelObject`), Cursor Private Inference's `capabilities.max_output_tokens` (`cursorLocalModel`), Muse Code's `limit.output` (`museModels`), the GUI's model and group rows (`providerInfo`, `providersState`), `magpie model output`, and every agent's file (`agent.maxTokens`, #338). The window and the output can come from different sources: Grok's backend lists grok-4.7's 256000 window, and models.dev fills its 500000 output (`catalog.Decorate` and `Provider.replyLimit` fill an output only where the vendor gives none). An unknown window or output is published as it is. The catalog keeps the unclamped `Entry.Output`, and a group's member is still asked for no more than that (`outputLimit`, `withMaxOutput`), so a request isn't lowered by the clamp. A limit the user set stays in their settings as they set it (`TestModelListsKeepOutputWithinTheWindow`). ## Verification diff --git a/internal/agent/route.go b/internal/agent/route.go index ef7b66eb..805c212a 100644 --- a/internal/agent/route.go +++ b/internal/agent/route.go @@ -140,12 +140,7 @@ func magpieModels(agent string) []catalog.Model { // context window it is handed with it: models.dev lists some models' output // above their window (deepseek-chat's 384000 against 128000). An unknown // window leaves the output as it is. -func maxTokens(m catalog.Model) int { - if m.Context > 0 && m.Output > m.Context { - return m.Context - } - return m.Output -} +func maxTokens(m catalog.Model) int { return provider.OutputWithin(m.Context, m.Output) } // group tags every option with a group name. func group(name string, opts []Option) []Option { diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index c6763e2a..917f7c59 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -656,8 +656,10 @@ func modelObject(e provider.Entry) map[string]any { if e.Context > 0 { m["context_window"], m["context_length"], m["max_input_tokens"] = e.Context, e.Context, e.Context } - if e.Output > 0 { - m["max_output_tokens"] = e.Output + // the reply limit, kept within that window as agents are told it + // (#1438) + if out := e.PublishedOutput(); out > 0 { + m["max_output_tokens"] = out } // for another magpie that has this one as its provider (remote-magpie): // the APIs a request for the model goes on as it is, so it sends each @@ -710,8 +712,8 @@ func cursorLocalModel(m map[string]any, e provider.Entry) { if e.Context > 0 { c["context_length"] = e.Context } - if e.Output > 0 { - c["max_output_tokens"] = e.Output + if out := e.PublishedOutput(); out > 0 { + c["max_output_tokens"] = out } if e.Images { c["supports_vision"] = true diff --git a/internal/gateway/model_output_window_test.go b/internal/gateway/model_output_window_test.go new file mode 100644 index 00000000..25735ec7 --- /dev/null +++ b/internal/gateway/model_output_window_test.go @@ -0,0 +1,125 @@ +package gateway + +import ( + "encoding/json" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/yetone/magpie/internal/catalog" + "github.com/yetone/magpie/internal/provider" +) + +// #1438 (yhong91): /v1/models published grok/grok-4.7 with a 256000 +// window, the Grok backend's own, beside max_output_tokens 500000, which +// models.dev gives x-ai's grok-4.7. Every model list the gateway serves now +// keeps the reply limit within the window beside it, as the agents' files +// have since #338; an unknown window or output keeps its shape, and a +// request is still lowered only to the model's own limit. +func TestModelListsKeepOutputWithinTheWindow(t *testing.T) { + f := &fake{ctype: "application/json", reply: `{"id":"x","object":"chat.completion","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}`} + up := setup(t, provider.Chat, f) + // models.dev's entry, as the reporter's cache has it + os.MkdirAll(filepath.Dir(catalog.CachePath()), 0o755) + if err := os.WriteFile(catalog.CachePath(), []byte(`{"xai":{"id":"xai","models":{"grok-4.7":{"id":"grok-4.7","limit":{"context":500000,"output":500000}}}}}`), 0o644); err != nil { + t.Fatal(err) + } + catalog.Reset() + t.Cleanup(catalog.Reset) + if err := provider.Save(provider.Provider{ID: "gx", Name: "GX", Key: "k", Chat: up.URL + "/v1", + Models: []string{"grok-4.7", "own-within", "own-nowindow", "own-nooutput"}}); err != nil { + t.Fatal(err) + } + // the backend's own list: grok-4.7's window, as cli-chat-proxy.grok.com + // gives it, with no output of its own + if err := catalog.SaveLive("gx", up.URL+"/v1", []catalog.Model{ + {ID: "grok-4.7", Context: 256000}, + {ID: "own-within", Context: 128000, Output: 64000}, + {ID: "own-nowindow", Output: 500000}, + {ID: "own-nooutput", Context: 256000}, + }); err != nil { + t.Fatal(err) + } + // the fixture is the report's: models.dev's output over the backend's + // window in the catalog itself + e, ok := provider.ServedEntryOf("gx/grok-4.7") + if !ok || e.Context != 256000 || e.Output != 500000 { + t.Fatalf("fixture: window %d, output %d; want models.dev's 500000 over the backend's 256000", e.Context, e.Output) + } + + list := func(path, key, ua string) map[string]map[string]any { + t.Helper() + req := httptest.NewRequest("GET", path, nil) + if key != "" { + req.Header.Set("Authorization", "Bearer "+key) + } + if ua != "" { + req.Header.Set("User-Agent", ua) + } + rec := httptest.NewRecorder() + New().Handler().ServeHTTP(rec, req) + var out struct { + Data []map[string]any `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatalf("%s: %s", path, rec.Body) + } + got := map[string]map[string]any{} + for _, d := range out.Data { + got[d["id"].(string)] = d + } + return got + } + num := func(v any) any { + if n, ok := v.(float64); ok { + return int(n) + } + return v + } + + // /v1/models, as every client reads it + got := list("/v1/models", "", "") + for id, want := range map[string][2]any{ + "gx/grok-4.7": {256000, 256000}, + "gx/own-within": {128000, 64000}, + "gx/own-nowindow": {nil, 500000}, + "gx/own-nooutput": {256000, nil}, + } { + m := got[id] + if m == nil { + t.Fatalf("%s not listed: %v", id, got) + } + if w, o := num(m["context_window"]), num(m["max_output_tokens"]); w != want[0] || o != want[1] { + t.Errorf("/v1/models %s: context_window %v, max_output_tokens %v; want %v, %v", id, w, o, want[0], want[1]) + } + } + + // Cursor Private Inference's capabilities + c, _ := list("/v1/models", TokenFor("cursor-local"), "")["gx/grok-4.7"]["capabilities"].(map[string]any) + if num(c["context_length"]) != 256000 || num(c["max_output_tokens"]) != 256000 { + t.Errorf("Cursor's capabilities: %v", c) + } + + // Muse Code's limit + muse := list("/muse-code/models", "", "muse-build/1.4.2 (non-interactive; macos-aarch64; build 0)") + if meta, _ := muse["gx/grok-4.7"]["metadata"].(map[string]any)["muse-code"].(map[string]any); meta == nil { + t.Errorf("Muse's list: %v", muse) + } else if l, _ := meta["limit"].(map[string]any); num(l["context"]) != 256000 || num(l["output"]) != 256000 { + t.Errorf("Muse's limit: %v", meta["limit"]) + } + + // a group's member is asked for no more than the model's own limit, as + // before (withMaxOutput): what is published is the pair, not a new cap + // on what is asked + if err := provider.SaveGroup(provider.Group{ID: "gg", Name: "GG", Members: []string{"gx/grok-4.7"}}); err != nil { + t.Fatal(err) + } + if code, body := post(t, "/v1/chat/completions", `{"model":"`+provider.GroupPrefix+`gg","max_tokens":300000,"messages":[{"role":"user","content":"hi"}]}`); code != 200 { + t.Fatalf("%d %s", code, body) + } + var asked map[string]any + if err := json.Unmarshal(f.got, &asked); err != nil || num(asked["max_tokens"]) != 300000 { + t.Errorf("asked upstream: %s", f.got) + } +} diff --git a/internal/gateway/muse.go b/internal/gateway/muse.go index 69e47a7e..4f523ab1 100644 --- a/internal/gateway/muse.go +++ b/internal/gateway/muse.go @@ -39,7 +39,7 @@ func (s *Server) museModels(w http.ResponseWriter, r *http.Request) { // whose limit lacks either, and without a limit asks for replies // of 128K tokens, more than most models give; one magpie doesn't // know is told 128K and Muse's own 32K - ctx, out := cmp.Or(e.Context, museContext), e.Output + ctx, out := cmp.Or(e.Context, museContext), e.PublishedOutput() if out <= 0 { out = min(museOutput, ctx) } diff --git a/internal/gui/model_output_test.go b/internal/gui/model_output_test.go index dba622b7..a37d4aa5 100644 --- a/internal/gui/model_output_test.go +++ b/internal/gui/model_output_test.go @@ -56,3 +56,52 @@ func TestProviderModelOutputSaid(t *testing.T) { t.Fatalf("the user's: %v", got) } } + +// #1438 (yhong91): a model's reply limit is shown within the window shown +// beside it, as agents are told it and /v1/models gives it — models.dev's +// 500000 for grok-4.7 against the 256000 its backend lists. One within its +// window, or with no window known, is shown as it was, and so is a group's. +func TestProviderModelOutputWithinTheWindow(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) + t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache")) + os.MkdirAll(filepath.Dir(catalog.CachePath()), 0o755) + os.WriteFile(catalog.CachePath(), []byte(`{"xai":{"id":"xai","models":{"grok-4.7":{"id":"grok-4.7","limit":{"context":500000,"output":500000}}}}}`), 0o644) + catalog.Reset() + t.Cleanup(catalog.Reset) + if err := provider.Save(provider.Provider{ID: "gx", Name: "GX", Key: "k", Chat: "http://127.0.0.1:1/v1", Models: []string{"grok-4.7", "own-within", "own-nowindow"}}); err != nil { + t.Fatal(err) + } + if err := catalog.SaveLive("gx", "http://127.0.0.1:1/v1", []catalog.Model{ + {ID: "grok-4.7", Context: 256000}, {ID: "own-within", Context: 128000, Output: 64000}, {ID: "own-nowindow", Output: 500000}}); err != nil { + t.Fatal(err) + } + if err := provider.SaveGroup(provider.Group{ID: "g", Name: "G", Members: []string{"gx/grok-4.7"}}); err != nil { + t.Fatal(err) + } + p, err := provider.Find("gx") + if err != nil { + t.Fatal(err) + } + got := map[string][2]int{} + for _, m := range providerInfo(*p, nil).Models { + got[m.ID] = [2]int{m.Context, m.Output} + } + for id, want := range map[string][2]int{"grok-4.7": {256000, 256000}, "own-within": {128000, 64000}, "own-nowindow": {0, 500000}} { + if got[id] != want { + t.Errorf("%s: the page says window, output %v; want %v", id, got[id], want) + } + } + seen := false + for _, g := range providersState().Gateway.Groups { + seen = seen || g.ID == provider.GroupPrefix+"g" + if g.ID == provider.GroupPrefix+"g" && (g.Context != 256000 || g.Output != 256000) { + t.Errorf("the group: window %d, output %d", g.Context, g.Output) + } + } + if !seen { + t.Error("the group is not on the Gateway page") + } +} diff --git a/internal/gui/providers.go b/internal/gui/providers.go index d12e7af4..1db4577a 100644 --- a/internal/gui/providers.go +++ b/internal/gui/providers.go @@ -36,7 +36,7 @@ type modelJSON struct { Own bool `json:"ownImages,omitempty"` // its vendor's answer, which a staged Restore default shows On bool `json:"on"` // exposed to agents Context int `json:"context,omitempty"` // the window agents are told: the user's, else Listed - Output int `json:"output,omitempty"` // the reply limit agents are told (provider.ReplyLimit) + Output int `json:"output,omitempty"` // the reply limit agents are told: provider.ReplyLimit within Context (provider.OutputWithin) Listed int `json:"listed,omitempty"` // its window before the user's: its vendor's list's, else models.dev's Max int `json:"max,omitempty"` // the most its context may be set to, above Listed Free bool `json:"free,omitempty"` // costs the subscription nothing @@ -545,7 +545,8 @@ func providerInfo(p provider.Provider, agents []agentUse) providerJSON { if imageSet { images = said } - j := modelJSON{ID: m.ID, Name: m.Name, Efforts: provider.EffortsOf(m), On: on, Context: p.WindowOf(m), Output: p.ReplyLimitIn(m, set), Listed: provider.ListedWindow(m), Max: m.MaxContext, Free: m.Free, Rate: m.Rate, RateWas: m.RateWas, Images: images, ImageSet: imageSet, Own: own} + window := p.WindowOf(m) + j := modelJSON{ID: m.ID, Name: m.Name, Efforts: provider.EffortsOf(m), On: on, Context: window, Output: provider.OutputWithin(window, p.ReplyLimitIn(m, set)), Listed: provider.ListedWindow(m), Max: m.MaxContext, Free: m.Free, Rate: m.Rate, RateWas: m.RateWas, Images: images, ImageSet: imageSet, Own: own} if i := slices.IndexFunc(most, func(c catalog.Model) bool { return c.ID == m.ID }); j.Max == 0 && i >= 0 { j.Max = most[i].MaxContext } @@ -689,7 +690,7 @@ func providersState() providersJSON { if e.Group == "" { continue } - g := gwGroupJSON{ID: e.ID, Name: e.Name, Icons: e.Icons, Efforts: e.Efforts, Images: e.Images, Context: e.Context, Output: e.Output} + g := gwGroupJSON{ID: e.ID, Name: e.Name, Icons: e.Icons, Efforts: e.Efforts, Images: e.Images, Context: e.Context, Output: e.PublishedOutput()} if _, ms, ok := findGroup(e.ID); ok { for _, m := range ms { if id := m.Provider.ID + "/" + m.Model; !slices.Contains(g.Members, id) { diff --git a/internal/provider/limits_test.go b/internal/provider/limits_test.go index 9c5e7306..ead08135 100644 --- a/internal/provider/limits_test.go +++ b/internal/provider/limits_test.go @@ -705,3 +705,23 @@ func TestOutputChecksTheKeyItStores(t *testing.T) { t.Error("a key that names no provider and model was accepted") } } + +// #1438: a reply limit is handed out within the window beside it, and an +// unknown window or output, or one already within, is left as it is. +func TestOutputWithin(t *testing.T) { + for _, c := range []struct{ window, output, want int }{ + {256000, 500000, 256000}, // Grok's window, models.dev's output + {128000, 64000, 64000}, // within: unchanged + {128000, 128000, 128000}, // at the window: unchanged + {0, 500000, 500000}, // unknown window + {256000, 0, 0}, // unknown output + {0, 0, 0}, + } { + if got := OutputWithin(c.window, c.output); got != c.want { + t.Errorf("OutputWithin(%d, %d) = %d, want %d", c.window, c.output, got, c.want) + } + if got := (Entry{Context: c.window, Output: c.output}).PublishedOutput(); got != c.want { + t.Errorf("PublishedOutput of %d/%d = %d, want %d", c.window, c.output, got, c.want) + } + } +} diff --git a/internal/provider/models.go b/internal/provider/models.go index 4ec8e37a..b0e41c85 100644 --- a/internal/provider/models.go +++ b/internal/provider/models.go @@ -1505,6 +1505,26 @@ func (p Provider) replyLimit(m catalog.Model, s settings.Settings) int { return output } +// OutputWithin is a reply limit as it is published beside a window: kept +// within it. models.dev lists some models' output above the window their +// vendor's own list gives (Grok's grok-4.7: 500000 against the 256000 its +// backend says, #1438; deepseek-chat's 384000 against 128000, #338), and a +// client that reads both sizes its replies past what the window holds. An +// unknown window or output is left as it is. Every place that hands out the +// pair uses it — the agents' files, the gateway's model lists, the GUI and +// the CLI — while a request is still lowered only to the model's own limit +// (Entry.Output, withMaxOutput). +func OutputWithin(window, output int) int { + if window > 0 && output > window { + return window + } + return output +} + +// PublishedOutput is e's reply limit as it is handed out beside its window +// (OutputWithin). +func (e Entry) PublishedOutput() int { return OutputWithin(e.Context, e.Output) } + func outputOf(s settings.Settings, providerID, model string) int { if n := s.ModelOutputs[providerID+"/"+model]; n > 0 { return n diff --git a/model_cli.go b/model_cli.go index 72d6530a..1972025e 100644 --- a/model_cli.go +++ b/model_cli.go @@ -866,7 +866,13 @@ func modelOutput(args []string) error { fmt.Println(muted.Render(id), faint.Render("· is not a model this provider serves")) return nil } - fmt.Println(bold.Render(tokenCount(e.Output)), muted.Render("· "+id)) + // the limit agents are told and the gateway's model list gives: + // within the model's window (#1438) + out := e.PublishedOutput() + fmt.Println(bold.Render(tokenCount(out)), muted.Render("· "+id)) + if out < e.Output { + fmt.Println(faint.Render(" · " + tokenCount(e.Output) + ", kept within its " + tokenCount(e.Context) + " window")) + } switch { case s.ModelOutputs[id] > 0 && s.ModelOutputs[p.ID+"/*"] > 0: // the model's own limit is what it answers with, and --reset diff --git a/model_cli_test.go b/model_cli_test.go index 119fc029..1e473588 100644 --- a/model_cli_test.go +++ b/model_cli_test.go @@ -605,6 +605,34 @@ func TestModelLimitCmd(t *testing.T) { } } +// #1438: `magpie model output` says the reply limit agents are told and +// /v1/models gives, within the model's window, and what it was kept from; +// the user's own limit stays as they set it, for a wider window to take. +func TestModelOutputCmdWithinTheWindow(t *testing.T) { + groupsHome(t) + catalog.Changed = nil + if err := modelCmd([]string{"context", "a/m", "200k"}); err != nil { + t.Fatal(err) + } + if err := modelCmd([]string{"output", "a/m", "500k"}); err != nil { + t.Fatal(err) + } + out := saidArgs(t, "output", "a/m") + if !strings.HasPrefix(strings.TrimSpace(out), "200k") || !strings.Contains(out, "500k, kept within its 200k window") { + t.Errorf("output a/m: %q; want 200k, kept from 500k", out) + } + if n := settings.Load().ModelOutputs["a/m"]; n != 500_000 { + t.Errorf("the user's limit became %d", n) + } + // within the window: as it was, with no line about it + if err := modelCmd([]string{"output", "a/m", "64k"}); err != nil { + t.Fatal(err) + } + if out := saidArgs(t, "output", "a/m"); !strings.HasPrefix(strings.TrimSpace(out), "64k") || strings.Contains(out, "kept within") { + t.Errorf("output a/m: %q; want 64k as it is", out) + } +} + // A provider kept unlisted, or switched off, still serves its models and // still takes the window a user gives one, so the query has to answer about // the very ids the setter takes: "is not a model this provider serves" on