From b5cfca54ad569626d386a38485ec5b04dc09fa5c Mon Sep 17 00:00:00 2001 From: yetone Date: Sat, 3 Oct 2026 23:42:09 +0800 Subject: [PATCH] =?UTF-8?q?gateway:=20a=20conversation=20too=20long=20for?= =?UTF-8?q?=20one=20member=20of=20a=20group=20goes=20on=20to=20a=20member?= =?UTF-8?q?=20with=20room=20for=20it,=20as=20before=20(#700:=20ynky007=20?= =?UTF-8?q?=E2=80=94=20Kimi=20Code=20on=20DeepSeek=20V4.1=20Flash=20throug?= =?UTF-8?q?h=20magpie=20showed=20a=20176k=20context=20after=20updating=20t?= =?UTF-8?q?o=200.1.768)=20=E2=80=94=20magpie=20still=20writes=20max=5Fcont?= =?UTF-8?q?ext=5Fsize=20=3D=201048576=20for=20it=20into=20~/.kimi-code/con?= =?UTF-8?q?fig.toml=20(checked=20in=20a=20temp=20HOME=20with=20the=20model?= =?UTF-8?q?s.dev=20cache).=20176k=20is=20Kimi=20Code's=20own:=20on=20a=20c?= =?UTF-8?q?ontext=20overflow=20it=20takes=20floor(0.85=20=C3=97=20the=20re?= =?UTF-8?q?quest's=20tokens)=20as=20the=20model's=20window=20for=20the=20s?= =?UTF-8?q?ession=20(observeContextOverflow,=20OVERFLOW=5FCONTEXT=5FSAFETY?= =?UTF-8?q?=5FRATIO,=202.1.1),=20so=20it=20was=20told=20a=20request=20of?= =?UTF-8?q?=20about=20207k=20overflowed.=20Since=2011a09d41=20an=20overflo?= =?UTF-8?q?w=20(a=20400/413/422=20saying=20so,=20which=20tooLongRe=20now?= =?UTF-8?q?=20knows=20in=20many=20more=20words)=20is=20not=20retryable=20a?= =?UTF-8?q?nd=20goes=20to=20the=20agent=20at=20once,=20and=20b3ab3e42=20te?= =?UTF-8?q?lls=20some=20unexplained=20failures=20as=20one;=20before,=20a?= =?UTF-8?q?=20400=20with=20"exceeded"=20failed=20over=20to=20the=20next=20?= =?UTF-8?q?member,=20which=20answered.=20That=20is=20right=20for=20other?= =?UTF-8?q?=20accounts=20of=20the=20same=20model,=20not=20for=20a=20group?= =?UTF-8?q?=20member=20of=20another=20model=20or=20vendor,=20whose=20windo?= =?UTF-8?q?w=20may=20be=20larger.=20Now=20an=20overflow=20goes=20on=20to?= =?UTF-8?q?=20the=20members=20left=20that=20are=20not=20the=20same=20provi?= =?UTF-8?q?der=20and=20model=20and=20not=20known=20(windowOf)=20to=20be=20?= =?UTF-8?q?smaller=20than=20the=20request;=20nobody=20rests=20(Fail=20"ove?= =?UTF-8?q?rflow",=20shown=20as=20"too=20long=20for=20its=20model"),=20and?= =?UTF-8?q?=20if=20they=20fail=20on=20quota=20the=20agent=20is=20still=20t?= =?UTF-8?q?old=20the=20overflow,=20to=20compact.=20With=20none=20left=20th?= =?UTF-8?q?e=20agent=20is=20told=20at=20once,=20as=20now.=20TestOverflowGo?= =?UTF-8?q?esToAMemberWithRoom=20fails=20on=20the=20old=20code=20(400=20co?= =?UTF-8?q?ntext=5Flength=5Fexceeded,=20the=20second=20member=20never=20as?= =?UTF-8?q?ked)=20and=20passes;=20it=20also=20checks=20a=20member=20known?= =?UTF-8?q?=20too=20small=20and=20a=20second=20key=20of=20the=20same=20mod?= =?UTF-8?q?el=20are=20not=20asked.=20go=20vet,=20builds=20(darwin,=20GOOS?= =?UTF-8?q?=3Dlinux,=20GOOS=3Dwindows)=20and=20go=20test=20-tags=20nogui?= =?UTF-8?q?=20-count=3D1=20./...=20under=20a=20temp=20HOME=20pass.=20Not?= =?UTF-8?q?=20tried=20with=20the=20reporter's=20providers:=20which=20membe?= =?UTF-8?q?r=20refused=20at=20~207k=20is=20not=20known.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/gateway/fallback.go | 18 +++++++ internal/gateway/gateway.go | 23 +++++++++ internal/gateway/overflow_member_test.go | 65 ++++++++++++++++++++++++ internal/gateway/routing.go | 3 ++ internal/gui/assets/i18n.js | 1 + internal/gui/assets/routing.js | 2 +- 6 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 internal/gateway/overflow_member_test.go diff --git a/internal/gateway/fallback.go b/internal/gateway/fallback.go index c6b18402..5d111d0a 100644 --- a/internal/gateway/fallback.go +++ b/internal/gateway/fallback.go @@ -688,6 +688,24 @@ func passing(status int, header http.Header, body []byte, again int) (time.Durat return 0, false } +// withRoom is, of the candidates left, those another member than c's may +// answer a request of about tokens that c's model found too long: not an +// account of c's model at c's provider, which holds it no better, nor one +// whose window it is known to be past. +func withRoom(left []candidate, c candidate, tokens int) []candidate { + var out []candidate + for _, x := range left { + if x.p.ID == c.p.ID && x.model == c.model { + continue + } + if w := windowOf(x.p, x.model); w > 0 && tokens >= w { + continue + } + out = append(out, x) + } + return out +} + // matesFirst puts first, of the candidates left, the other keys or // accounts of the member c is of — its model, at its effort — that aren't // resting: what one account's safety filter refused, another may answer diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 224c9842..976f029e 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -1602,6 +1602,29 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro break } } + if !last && !hw.passing && !hw.refused && overflowed(hw.code(), hw.errBody()) { + // too long for this member's model, and as long on every + // account of it, but another member — another model, or this + // one at another vendor — may hold it (#700: the overflow went + // to Kimi Code, which took 0.85 of the request for the model's + // window, where the next member would have answered). Only + // those not known to be too small are left to ask; none, and + // the agent is told, to compact. Nobody rests. + tokens := 0 + if req, err := parse(from, attemptBody); err == nil { + tokens = estimate(req) + } + if left := withRoom(cands[i+1:], c, tokens); len(left) > 0 { + if other == nil { + other = &Try{Status: call.Status, Error: call.Error} + } + try.Fail = failOverflow + s.trace.update(tr, func(t *Route) { t.Tries[len(t.Tries)-1] = try }) + skipped = append(skipped, c.label()+": "+call.Error) + cands = append(cands[:i+1:i+1], left...) + continue + } + } if !last && hw.failed() && shapeRefused(hw.code(), hw.errBody()) { // a request this vendor's API can't read (xAI's 422 over an // input item it doesn't know, #350) another's may: the next is diff --git a/internal/gateway/overflow_member_test.go b/internal/gateway/overflow_member_test.go new file mode 100644 index 00000000..35187642 --- /dev/null +++ b/internal/gateway/overflow_member_test.go @@ -0,0 +1,65 @@ +package gateway + +import ( + "net/http/httptest" + "strings" + "testing" + + "github.com/yetone/magpie/internal/provider" +) + +// A conversation too long for one member's model is as long on every +// account of it, but not too long for another member, a model or vendor +// with a larger window (#700: Kimi Code on DeepSeek V4.1 Flash through +// magpie, told the context was 176k — 0.85 of a request one member called +// too long — where before the overflow went to the next member, which +// answered). The next member is asked; the agent is told it overflowed +// only when no member left may hold it. +func TestOverflowGoesToAMemberWithRoom(t *testing.T) { + fresh(t) + over := `{"error":{"message":"This model's maximum context length is 131072 tokens. However, you requested 207000 tokens","code":"context_length_exceeded"}}` + a := &scripted{replies: []reply{{400, "", over}}} + b := &scripted{replies: []reply{{200, "", chatOK}}} + scriptedOn(t, "a", provider.Chat, a) + scriptedOn(t, "b", provider.Chat, b) + if err := provider.SaveGroup(provider.Group{Name: "G", Members: []string{"a/m", "b/m"}, Routing: provider.Ordered}); err != nil { + t.Fatal(err) + } + s := New() + code, body := postAs(t, s, "", `{"model":"group/g","messages":[{"role":"user","content":"hi"}]}`) + if code != 200 || !strings.Contains(body, "hello") || a.n != 1 || b.n != 1 { + t.Fatalf("group: %d %s (a %d, b %d)", code, body, a.n, b.n) + } + r := s.trace.routes[len(s.trace.routes)-1] + if len(r.Tries) != 2 || r.Tries[0].Status != 400 || r.Tries[0].Fail != failOverflow || r.Tries[0].Rest != nil { + t.Fatalf("tries %+v", r.Tries) + } + + // a member whose window the request is known to be past isn't asked + big := strings.Repeat("word ", 2400) // about 3000 tokens as estimated + up := httptest.NewServer(b) + t.Cleanup(up.Close) + if err := provider.Save(provider.Provider{ID: "c", Name: "C", Key: "k", Chat: up.URL + "/v1", Models: []string{"m"}, Contexts: map[string]int{"m": 2000}}); err != nil { + t.Fatal(err) + } + if err := provider.SaveGroup(provider.Group{Name: "Small", Members: []string{"a/m", "c/m"}, Routing: provider.Ordered}); err != nil { + t.Fatal(err) + } + a.n, b.n = 0, 0 + code, body = postAs(t, s, "", `{"model":"group/small","messages":[{"role":"user","content":"`+big+`"}]}`) + if code != 400 || !strings.Contains(body, "context_length_exceeded") || a.n != 1 || b.n != 0 { + t.Fatalf("no room: %d %s (a %d, c %d)", code, body, a.n, b.n) + } + + // another account of the same model holds it no better: not asked + up2 := httptest.NewServer(a) + t.Cleanup(up2.Close) + if err := provider.Save(provider.Provider{ID: "keys", Name: "Keys", Key: "first", Keys: []provider.KeyAccount{{Key: "second"}}, Chat: up2.URL + "/v1", Models: []string{"m"}}); err != nil { + t.Fatal(err) + } + a.n = 0 + code, body = postAs(t, s, "", `{"model":"keys/m","messages":[{"role":"user","content":"hi"}]}`) + if code != 400 || !strings.Contains(body, "context_length_exceeded") || a.n != 1 { + t.Fatalf("accounts: %d %s (asked %d)", code, body, a.n) + } +} diff --git a/internal/gateway/routing.go b/internal/gateway/routing.go index c126f1ca..27e8b744 100644 --- a/internal/gateway/routing.go +++ b/internal/gateway/routing.go @@ -157,6 +157,9 @@ const ( // *_PROXY variables, the system's) didn't take the connection (#381) — // nothing reached the vendor, so the next one is asked, and nobody rests failProxy = "proxy" + // failOverflow: the conversation is too long for the member's model + // (#700) — another member, with room for it, is asked, and nobody rests + failOverflow = "overflow" ) // proxyDown is the error Go gives when the proxy itself can't be reached, diff --git a/internal/gui/assets/i18n.js b/internal/gui/assets/i18n.js index 3028f5df..8b051d94 100644 --- a/internal/gui/assets/i18n.js +++ b/internal/gui/assets/i18n.js @@ -1948,6 +1948,7 @@ const I18N = { "request not understood": "请求格式不被接受", "proxy not reachable": "代理连不上", "reasoning effort not in its plan": "套餐不支持该思考强度", + "too long for its model": "超出该模型上下文", "{who} answered {status}: its plan doesn't take the reasoning effort asked for, so another account goes on with it before any of the reply reaches {agent}. {who} serves other efforts, so it doesn't rest.": "{who} 返回 {status}：它的套餐不支持请求的思考强度，所以在任何回复到达 {agent} 之前就转给了另一个账号。{who} 仍可服务其他强度，所以不用休息。", "{who} answered {status}: its plan doesn't take the reasoning effort asked for, and no account left that does could answer, so {agent} gets an error saying so.": "{who} 返回 {status}：它的套餐不支持请求的思考强度，剩下也没有支持它且能应答的账号，所以 {agent} 收到一条说明原因的错误。", "{who}: the proxy magpie goes through didn't take the connection, so the request never reached the vendor and goes on to the next. Nothing is wrong with {who}, so it doesn't rest: once the proxy is up it is asked first again.": "{who}：magpie 所走的代理没有接受连接，请求根本没到厂商，所以转给了下一个。{who} 本身没出问题，所以不用休息：代理恢复后它仍会最先被问。", diff --git a/internal/gui/assets/routing.js b/internal/gui/assets/routing.js index f306d769..c2fa1bdb 100644 --- a/internal/gui/assets/routing.js +++ b/internal/gui/assets/routing.js @@ -168,7 +168,7 @@ // (WorkBuddy's credits, #659): "355 / 500 credits · 71% used" const quota = (w, used, left, vars) => (w.limit > 0 ? quotaCount(w) + " · " : "") + t(quotaLeft ? left : used, { n: pct(share(w)), ...vars }); const fill = (w) => Math.max(0, Math.min(100, share(w))) + "%"; - const FAIL = { rate: "rate limited", credit: "out of credit", quota: "quota used up", other: "failed", canceled: "canceled", foreign: "another account's reasoning", floor: "reply too short", verify: "needs verification", refused: "refused (safety filter)", shape: "request not understood", proxy: "proxy not reachable", effort: "reasoning effort not in its plan" }; + const FAIL = { rate: "rate limited", credit: "out of credit", quota: "quota used up", other: "failed", canceled: "canceled", foreign: "another account's reasoning", floor: "reply too short", verify: "needs verification", refused: "refused (safety filter)", shape: "request not understood", proxy: "proxy not reachable", effort: "reasoning effort not in its plan", overflow: "too long for its model" }; const failWord = (why) => t(FAIL[why] || "failed"); const API = { anthropic: "Anthropic", chat: "OpenAI", responses: "OpenAI Responses", gemini: "Gemini" }; const MODES = {