From b3ab3e42e9770a09a3ec29ac9c07511376d4e7fd Mon Sep 17 00:00:00 2001 From: yetone Date: Sat, 3 Oct 2026 16:14:38 +0800 Subject: [PATCH] =?UTF-8?q?gateway:=20a=20request=20well=20past=20the=20mo?= =?UTF-8?q?del's=20window=20that=20a=20vendor=20turns=20away=20without=20s?= =?UTF-8?q?aying=20why=20reaches=20the=20agent=20as=20too=20long,=20at=20o?= =?UTF-8?q?nce;=20Qoder's=20plugin=20to=200.2.4=20for=20its=20token=20usag?= =?UTF-8?q?e=20(found=20testing=20real=20accounts)=20=E2=80=94=20Devin=20a?= =?UTF-8?q?nswered=20a=20conversation=20of=20about=20300k=20tokens=20for?= =?UTF-8?q?=20swe-1.7-lightning=20(a=20202,752=20window)=20with=20a=20502?= =?UTF-8?q?=20"We=20are=20currently=20experiencing=20capacity=20issues?= =?UTF-8?q?=E2=80=A6",=20Qoder=20one=20of=201.8M=20for=20qmodel=20(1M)=20w?= =?UTF-8?q?ith=20a=20400=20"Error=20in=20upstream=20response",=20each=20an?= =?UTF-8?q?swered=20at=20a=20size=20the=20model=20holds;=20magpie=20asked?= =?UTF-8?q?=20Devin's=20again=20in=20place=20three=20times=20and=20handed?= =?UTF-8?q?=20the=20agent=20a=20502=20api=5Ferror,=20so=20Claude=20Code=20?= =?UTF-8?q?and=20Codex=20never=20compacted.=20tooLongUnsaid=20(fallback.go?= =?UTF-8?q?)=20takes=20a=20400/413/422/5xx=20that=20says=20nothing=20of=20?= =?UTF-8?q?quota,=20a=20rate=20limit,=20the=20account,=20the=20model=20or?= =?UTF-8?q?=20the=20conversation's=20length,=20for=20a=20request=20whose?= =?UTF-8?q?=20estimate=20is=201.2=20times=20the=20model's=20window=20or=20?= =?UTF-8?q?more=20(the=20user's=20context,=20else=20the=20catalog's;=20est?= =?UTF-8?q?imate=20counts=20four=20bytes=20a=20token,=20which=20most=20tex?= =?UTF-8?q?t=20runs=20under:=20Devin=20counted=20173,954=20tokens=20of=20o?= =?UTF-8?q?ne=20estimated=20at=20151,851),=20as=20one=20too=20long:=20"pro?= =?UTF-8?q?mpt=20is=20too=20long:=20about=20N=20tokens,=20more=20than=20's=20context=20window=20of=20M=20(=20answered=20:=20=E2=80=A6)",=20sent=20the=20agent's=20way=20(context?= =?UTF-8?q?=5Flength=5Fexceeded,=20Anthropic's=20"prompt=20is=20too=20long?= =?UTF-8?q?"),=20not=20asked=20again=20of=20it=20or=20another=20account,?= =?UTF-8?q?=20and=20nobody=20rests.=20TestOverflowTheVendorDidntSay=20fail?= =?UTF-8?q?s=20on=20the=20old=20code=20(a=20502=20asked=20three=20times,?= =?UTF-8?q?=20a=20400=20handed=20on)=20and=20passes;=20a=20request=20near?= =?UTF-8?q?=20the=20window,=20a=20small=20one,=20a=20429,=20a=20401=20and?= =?UTF-8?q?=20a=20used-up=20balance=20keep=20their=20errors.=20Qoder's=20p?= =?UTF-8?q?lugin=20read=20its=20stream=20only=20up=20to=20the=20finish,=20?= =?UTF-8?q?and=20Qoder=20sends=20the=20usage=20in=20a=20chunk=20of=20its?= =?UTF-8?q?=20own=20after=20it,=20so=20every=20Qoder=20request=20was=20cou?= =?UTF-8?q?nted=20as=200=20tokens:=20@magpie-community/opencode-qoder-auth?= =?UTF-8?q?=200.2.4=20reads=20it,=20and=20the=20mover's=20min=20is=20raise?= =?UTF-8?q?d=20to=20it.=20Tried=20with=20real=20accounts=20in=20a=20sandbo?= =?UTF-8?q?x=20magpie:=20Devin=20(chat=20and=20Anthropic)=20and=20Qoder=20?= =?UTF-8?q?now=20give=20400=20context=5Flength=5Fexceeded=20after=20one=20?= =?UTF-8?q?try,=20a=20WorkBuddy=20and=20Kiro=20overflow=20as=20before,=20s?= =?UTF-8?q?mall=20requests=20to=20all=20three=20answer,=20and=20Qoder=20re?= =?UTF-8?q?ports=20its=20tokens=20(streamed,=20not,=20and=20with=20a=20too?= =?UTF-8?q?l=20call).=20go=20vet,=20builds=20(darwin,=20GOOS=3Dlinux,=20GO?= =?UTF-8?q?OS=3Dwindows)=20and=20go=20test=20-tags=20nogui=20-count=3D1=20?= =?UTF-8?q?./...=20under=20a=20temp=20HOME=20pass=20but=20a=20provider=20t?= =?UTF-8?q?iming=20test=20that=20passes=20rerun.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/gateway/fallback.go | 55 +++++++++++++++++++++ internal/gateway/gateway.go | 15 +++++- internal/gateway/toolong_unsaid_test.go | 66 +++++++++++++++++++++++++ internal/provider/migrate_qoder.go | 2 +- 4 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 internal/gateway/toolong_unsaid_test.go diff --git a/internal/gateway/fallback.go b/internal/gateway/fallback.go index c600b7e2..c6b18402 100644 --- a/internal/gateway/fallback.go +++ b/internal/gateway/fallback.go @@ -596,6 +596,61 @@ func retryable(status int, body []byte) bool { return false } +// unsaidMargin is how far past a model's window a request's estimate +// must go to be taken as too long for it: estimate counts a token as four +// bytes, which most text runs under (Devin counted 173,954 tokens of one +// estimated at 151,851), so one past it by a fifth hardly fits. +const unsaidMargin = 1.2 + +// tooLongUnsaid says a failure that doesn't say why was the request being too +// long for the model, as some vendors put it: Devin's 502 "capacity +// issues", WorkBuddy's 400 "Invalid request parameters", Qoder's 400 +// "Error in upstream response" — each for a conversation well past the +// model's window, and answered at a size it holds. Retried, or asked of +// another account of the model, it fails the same, and the agent, told it +// as a vendor's error, never compacts. The message it is told instead +// says the prompt is too long. Not for a rate limit, an account's +// trouble, a model it doesn't serve, or one already saying it overflowed. +func tooLongUnsaid(p provider.Provider, model string, tokens, status int, body []byte) (string, bool) { + switch status { + case 400, 413, 422, 500, 502, 503, 504: + default: + return "", false + } + if overflowed(status, body) || failure(status, body) != failOther || quotaWords.Match(body) || + unservedWords.Match(body) || refusedWords.Match(body) || provider.EdgeBlocked(body) { + return "", false + } + window := windowOf(p, model) + if window <= 0 || float64(tokens) < float64(window)*unsaidMargin { + return "", false + } + msg := fmt.Sprintf("prompt is too long: about %d tokens, more than %s's context window of %d", tokens, model, window) + said := strings.TrimSpace(vendorMessage(body)) + if len(said) > 200 { + said = said[:200] + "…" + } + // the vendor's words, unless they'd read as a limit of another kind + if full := fmt.Sprintf("%s (%s answered %d: %s)", msg, p.Name, status, said); said != "" && tooLong(400, full) { + return full, true + } + return fmt.Sprintf("%s (%s answered %d)", msg, p.Name, status), true +} + +// windowOf is the tokens a provider's model takes, where known: the +// user's (Provider.ContextOf), else its catalog entry's. +func windowOf(p provider.Provider, model string) int { + if n := p.ContextOf(model); n > 0 { + return n + } + for _, e := range provider.Served() { + if e.Group == "" && e.Provider.ID == p.ID && (e.Model == model || e.ID == model) && e.Context > 0 { + return e.Context + } + } + return 0 +} + const ( // lastRetries is how many times the last one left is tried again after // a failure that passes — a busy vendor, a dropped connection. diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index cb880950..433b48b8 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -1443,6 +1443,16 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro stop() } hw.settle() + outgrew := false // a failure that didn't say so was the request's length + if !hw.passing && !hw.refused && hw.code() >= 400 { + if req, err := parse(from, attemptBody); err == nil { + if msg, ok := tooLongUnsaid(c.p, c.model, estimate(req), hw.code(), hw.errBody()); ok { + // told as an overflow: not asked again, of this one or + // another, and nobody rests + hw.failure, hw.failMsg, outgrew = http.StatusBadRequest, msg, true + } + } + } if hw.failure != 0 { // the stream failed before any of it was sent call.Status, call.Error = hw.failure, c.p.Name+": "+hw.failMsg } @@ -1496,7 +1506,7 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro continue } } - if updated && !hw.passing && hw.code() == 400 { + if updated && !outgrew && !hw.passing && hw.code() == 400 { // the upstream turned the update items away: asked again as // the agent sent it, and if that goes, never sent them again plainFor = c.who() + "|" + c.model @@ -1667,6 +1677,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro call.Status, call.Error = other.Status, other.Error writeError(w, from, call.Status, call.Error) break + } else if outgrew { + // in the agent's own words for it, for it to compact + writeError(w, from, call.Status, hw.failMsg) } else { hw.release() } diff --git a/internal/gateway/toolong_unsaid_test.go b/internal/gateway/toolong_unsaid_test.go new file mode 100644 index 00000000..e679c12e --- /dev/null +++ b/internal/gateway/toolong_unsaid_test.go @@ -0,0 +1,66 @@ +package gateway + +import ( + "strings" + "testing" + + "github.com/yetone/magpie/internal/provider" +) + +// Some vendors don't say a conversation is too long for the model: Devin +// answered one of about 225k tokens for swe-1.7-lightning (a 202,752 +// window) with a 502 "capacity issues", WorkBuddy one for glm-5.1 with a +// 400 "Invalid request parameters", Qoder one of 2M for qmodel with a 400 +// "Error in upstream response" — each answered at a size the model holds. +// Asked again in place three times, the agent was then handed a 502 and +// never compacted. A request well past the model's window that fails so +// is told as one too long, at once. +func TestOverflowTheVendorDidntSay(t *testing.T) { + big := strings.Repeat("word ", 2400) // about 3000 tokens as estimated + small := strings.Repeat("word ", 100) // about 125 + near := strings.Repeat("word ", 1760) // about 2200, which may yet fit + for _, tc := range []struct { + name, path, body string + code int + reply string + wantCode, calls int + want string + }{ + {"502 anthropic", "/v1/messages", big, 502, `{"error":{"message":"We are experiencing capacity issues"}}`, 400, 1, "prompt is too long"}, + {"400 chat", "/v1/chat/completions", big, 400, `{"error":{"message":"Invalid request parameters"}}`, 400, 1, "context_length_exceeded"}, + // not the conversation's length + {"small 502", "/v1/messages", small, 502, `{"error":{"message":"We are experiencing capacity issues"}}`, 502, 1 + lastRetries, "capacity issues"}, + {"near the window", "/v1/messages", near, 502, `{"error":{"message":"We are experiencing capacity issues"}}`, 502, 1 + lastRetries, "capacity issues"}, + {"429", "/v1/messages", big, 429, `{"error":{"message":"Rate limit reached"}}`, 429, 1 + rateRetries, "Rate limit"}, + {"401", "/v1/messages", big, 401, `{"error":{"message":"invalid token"}}`, 401, 1, "invalid token"}, + {"quota", "/v1/messages", big, 400, `{"error":{"message":"insufficient balance"}}`, 400, 1, "insufficient balance"}, + } { + t.Run(tc.name, func(t *testing.T) { + proto := provider.Anthropic + if strings.Contains(tc.path, "chat") { + proto = provider.Chat + } + f := &fake{code: tc.code, ctype: "application/json", reply: tc.reply} + up := setup(t, proto, f) + p := provider.Provider{ID: "fake", Name: "Fake", Key: "k", Models: []string{"m1"}, Contexts: map[string]int{"m1": 2000}} + if proto == provider.Chat { + p.Chat = up.URL + "/v1" + } else { + p.Anthropic = up.URL + } + if err := provider.Save(p); err != nil { + t.Fatal(err) + } + req := `{"model":"fake/m1","max_tokens":10,"messages":[{"role":"user","content":"` + tc.body + `"}]}` + code, body := post(t, tc.path, req) + if code != tc.wantCode || !strings.Contains(body, tc.want) || f.calls != tc.calls { + t.Fatalf("status %d after %d calls (want %d after %d): %s", code, f.calls, tc.wantCode, tc.calls, body) + } + if tc.wantCode != 400 || tc.name == "quota" { + if strings.Contains(body, "prompt is too long") || strings.Contains(body, "context_length_exceeded") { + t.Fatalf("told as an overflow: %s", body) + } + } + }) + } +} diff --git a/internal/provider/migrate_qoder.go b/internal/provider/migrate_qoder.go index 311301e9..439f824c 100644 --- a/internal/provider/migrate_qoder.go +++ b/internal/provider/migrate_qoder.go @@ -24,7 +24,7 @@ func init() { func qoderMover(id string) *mover { return &mover{ pkg: "@magpie-community/opencode-qoder-auth", - min: "0.2.3", // Qoder CN as "qoder-cn", its device-token chat as the built-in's; each model's credit rate; the sign-in renewed ahead of time through auth.refresh + min: "0.2.4", // Qoder CN as "qoder-cn", its device-token chat as the built-in's; each model's credit rate; the sign-in renewed ahead of time through auth.refresh; token usage read agents: []string{id}, out: func() ([]Moving, error) { // no refresh of the built-in's runs while the pairs are read