diff --git a/internal/gateway/fallback.go b/internal/gateway/fallback.go index c6b18402..5d111d0a 100644 --- a/internal/gateway/fallback.go +++ b/internal/gateway/fallback.go @@ -688,6 +688,24 @@ func passing(status int, header http.Header, body []byte, again int) (time.Durat return 0, false } +// withRoom is, of the candidates left, those another member than c's may +// answer a request of about tokens that c's model found too long: not an +// account of c's model at c's provider, which holds it no better, nor one +// whose window it is known to be past. +func withRoom(left []candidate, c candidate, tokens int) []candidate { + var out []candidate + for _, x := range left { + if x.p.ID == c.p.ID && x.model == c.model { + continue + } + if w := windowOf(x.p, x.model); w > 0 && tokens >= w { + continue + } + out = append(out, x) + } + return out +} + // matesFirst puts first, of the candidates left, the other keys or // accounts of the member c is of — its model, at its effort — that aren't // resting: what one account's safety filter refused, another may answer diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 224c9842..976f029e 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -1602,6 +1602,29 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro break } } + if !last && !hw.passing && !hw.refused && overflowed(hw.code(), hw.errBody()) { + // too long for this member's model, and as long on every + // account of it, but another member — another model, or this + // one at another vendor — may hold it (#700: the overflow went + // to Kimi Code, which took 0.85 of the request for the model's + // window, where the next member would have answered). Only + // those not known to be too small are left to ask; none, and + // the agent is told, to compact. Nobody rests. + tokens := 0 + if req, err := parse(from, attemptBody); err == nil { + tokens = estimate(req) + } + if left := withRoom(cands[i+1:], c, tokens); len(left) > 0 { + if other == nil { + other = &Try{Status: call.Status, Error: call.Error} + } + try.Fail = failOverflow + s.trace.update(tr, func(t *Route) { t.Tries[len(t.Tries)-1] = try }) + skipped = append(skipped, c.label()+": "+call.Error) + cands = append(cands[:i+1:i+1], left...) + continue + } + } if !last && hw.failed() && shapeRefused(hw.code(), hw.errBody()) { // a request this vendor's API can't read (xAI's 422 over an // input item it doesn't know, #350) another's may: the next is diff --git a/internal/gateway/overflow_member_test.go b/internal/gateway/overflow_member_test.go new file mode 100644 index 00000000..35187642 --- /dev/null +++ b/internal/gateway/overflow_member_test.go @@ -0,0 +1,65 @@ +package gateway + +import ( + "net/http/httptest" + "strings" + "testing" + + "github.com/yetone/magpie/internal/provider" +) + +// A conversation too long for one member's model is as long on every +// account of it, but not too long for another member, a model or vendor +// with a larger window (#700: Kimi Code on DeepSeek V4.1 Flash through +// magpie, told the context was 176k — 0.85 of a request one member called +// too long — where before the overflow went to the next member, which +// answered). The next member is asked; the agent is told it overflowed +// only when no member left may hold it. +func TestOverflowGoesToAMemberWithRoom(t *testing.T) { + fresh(t) + over := `{"error":{"message":"This model's maximum context length is 131072 tokens. However, you requested 207000 tokens","code":"context_length_exceeded"}}` + a := &scripted{replies: []reply{{400, "", over}}} + b := &scripted{replies: []reply{{200, "", chatOK}}} + scriptedOn(t, "a", provider.Chat, a) + scriptedOn(t, "b", provider.Chat, b) + if err := provider.SaveGroup(provider.Group{Name: "G", Members: []string{"a/m", "b/m"}, Routing: provider.Ordered}); err != nil { + t.Fatal(err) + } + s := New() + code, body := postAs(t, s, "", `{"model":"group/g","messages":[{"role":"user","content":"hi"}]}`) + if code != 200 || !strings.Contains(body, "hello") || a.n != 1 || b.n != 1 { + t.Fatalf("group: %d %s (a %d, b %d)", code, body, a.n, b.n) + } + r := s.trace.routes[len(s.trace.routes)-1] + if len(r.Tries) != 2 || r.Tries[0].Status != 400 || r.Tries[0].Fail != failOverflow || r.Tries[0].Rest != nil { + t.Fatalf("tries %+v", r.Tries) + } + + // a member whose window the request is known to be past isn't asked + big := strings.Repeat("word ", 2400) // about 3000 tokens as estimated + up := httptest.NewServer(b) + t.Cleanup(up.Close) + if err := provider.Save(provider.Provider{ID: "c", Name: "C", Key: "k", Chat: up.URL + "/v1", Models: []string{"m"}, Contexts: map[string]int{"m": 2000}}); err != nil { + t.Fatal(err) + } + if err := provider.SaveGroup(provider.Group{Name: "Small", Members: []string{"a/m", "c/m"}, Routing: provider.Ordered}); err != nil { + t.Fatal(err) + } + a.n, b.n = 0, 0 + code, body = postAs(t, s, "", `{"model":"group/small","messages":[{"role":"user","content":"`+big+`"}]}`) + if code != 400 || !strings.Contains(body, "context_length_exceeded") || a.n != 1 || b.n != 0 { + t.Fatalf("no room: %d %s (a %d, c %d)", code, body, a.n, b.n) + } + + // another account of the same model holds it no better: not asked + up2 := httptest.NewServer(a) + t.Cleanup(up2.Close) + if err := provider.Save(provider.Provider{ID: "keys", Name: "Keys", Key: "first", Keys: []provider.KeyAccount{{Key: "second"}}, Chat: up2.URL + "/v1", Models: []string{"m"}}); err != nil { + t.Fatal(err) + } + a.n = 0 + code, body = postAs(t, s, "", `{"model":"keys/m","messages":[{"role":"user","content":"hi"}]}`) + if code != 400 || !strings.Contains(body, "context_length_exceeded") || a.n != 1 { + t.Fatalf("accounts: %d %s (asked %d)", code, body, a.n) + } +} diff --git a/internal/gateway/routing.go b/internal/gateway/routing.go index c126f1ca..27e8b744 100644 --- a/internal/gateway/routing.go +++ b/internal/gateway/routing.go @@ -157,6 +157,9 @@ const ( // *_PROXY variables, the system's) didn't take the connection (#381) — // nothing reached the vendor, so the next one is asked, and nobody rests failProxy = "proxy" + // failOverflow: the conversation is too long for the member's model + // (#700) — another member, with room for it, is asked, and nobody rests + failOverflow = "overflow" ) // proxyDown is the error Go gives when the proxy itself can't be reached, diff --git a/internal/gui/assets/i18n.js b/internal/gui/assets/i18n.js index 3028f5df..8b051d94 100644 --- a/internal/gui/assets/i18n.js +++ b/internal/gui/assets/i18n.js @@ -1948,6 +1948,7 @@ const I18N = { "request not understood": "请求格式不被接受", "proxy not reachable": "代理连不上", "reasoning effort not in its plan": "套餐不支持该思考强度", + "too long for its model": "超出该模型上下文", "{who} answered {status}: its plan doesn't take the reasoning effort asked for, so another account goes on with it before any of the reply reaches {agent}. {who} serves other efforts, so it doesn't rest.": "{who} 返回 {status}:它的套餐不支持请求的思考强度,所以在任何回复到达 {agent} 之前就转给了另一个账号。{who} 仍可服务其他强度,所以不用休息。", "{who} answered {status}: its plan doesn't take the reasoning effort asked for, and no account left that does could answer, so {agent} gets an error saying so.": "{who} 返回 {status}:它的套餐不支持请求的思考强度,剩下也没有支持它且能应答的账号,所以 {agent} 收到一条说明原因的错误。", "{who}: the proxy magpie goes through didn't take the connection, so the request never reached the vendor and goes on to the next. Nothing is wrong with {who}, so it doesn't rest: once the proxy is up it is asked first again.": "{who}:magpie 所走的代理没有接受连接,请求根本没到厂商,所以转给了下一个。{who} 本身没出问题,所以不用休息:代理恢复后它仍会最先被问。", diff --git a/internal/gui/assets/routing.js b/internal/gui/assets/routing.js index f306d769..c2fa1bdb 100644 --- a/internal/gui/assets/routing.js +++ b/internal/gui/assets/routing.js @@ -168,7 +168,7 @@ // (WorkBuddy's credits, #659): "355 / 500 credits · 71% used" const quota = (w, used, left, vars) => (w.limit > 0 ? quotaCount(w) + " · " : "") + t(quotaLeft ? left : used, { n: pct(share(w)), ...vars }); const fill = (w) => Math.max(0, Math.min(100, share(w))) + "%"; - const FAIL = { rate: "rate limited", credit: "out of credit", quota: "quota used up", other: "failed", canceled: "canceled", foreign: "another account's reasoning", floor: "reply too short", verify: "needs verification", refused: "refused (safety filter)", shape: "request not understood", proxy: "proxy not reachable", effort: "reasoning effort not in its plan" }; + const FAIL = { rate: "rate limited", credit: "out of credit", quota: "quota used up", other: "failed", canceled: "canceled", foreign: "another account's reasoning", floor: "reply too short", verify: "needs verification", refused: "refused (safety filter)", shape: "request not understood", proxy: "proxy not reachable", effort: "reasoning effort not in its plan", overflow: "too long for its model" }; const failWord = (why) => t(FAIL[why] || "failed"); const API = { anthropic: "Anthropic", chat: "OpenAI", responses: "OpenAI Responses", gemini: "Gemini" }; const MODES = {