diff --git a/internal/gateway/fallback.go b/internal/gateway/fallback.go index c600b7e2..c6b18402 100644 --- a/internal/gateway/fallback.go +++ b/internal/gateway/fallback.go @@ -596,6 +596,61 @@ func retryable(status int, body []byte) bool { return false } +// unsaidMargin is how far past a model's window a request's estimate +// must go to be taken as too long for it: estimate counts a token as four +// bytes, which most text runs under (Devin counted 173,954 tokens of one +// estimated at 151,851), so one past it by a fifth hardly fits. +const unsaidMargin = 1.2 + +// tooLongUnsaid says a failure that doesn't say why was the request being too +// long for the model, as some vendors put it: Devin's 502 "capacity +// issues", WorkBuddy's 400 "Invalid request parameters", Qoder's 400 +// "Error in upstream response" — each for a conversation well past the +// model's window, and answered at a size it holds. Retried, or asked of +// another account of the model, it fails the same, and the agent, told it +// as a vendor's error, never compacts. The message it is told instead +// says the prompt is too long. Not for a rate limit, an account's +// trouble, a model it doesn't serve, or one already saying it overflowed. +func tooLongUnsaid(p provider.Provider, model string, tokens, status int, body []byte) (string, bool) { + switch status { + case 400, 413, 422, 500, 502, 503, 504: + default: + return "", false + } + if overflowed(status, body) || failure(status, body) != failOther || quotaWords.Match(body) || + unservedWords.Match(body) || refusedWords.Match(body) || provider.EdgeBlocked(body) { + return "", false + } + window := windowOf(p, model) + if window <= 0 || float64(tokens) < float64(window)*unsaidMargin { + return "", false + } + msg := fmt.Sprintf("prompt is too long: about %d tokens, more than %s's context window of %d", tokens, model, window) + said := strings.TrimSpace(vendorMessage(body)) + if len(said) > 200 { + said = said[:200] + "…" + } + // the vendor's words, unless they'd read as a limit of another kind + if full := fmt.Sprintf("%s (%s answered %d: %s)", msg, p.Name, status, said); said != "" && tooLong(400, full) { + return full, true + } + return fmt.Sprintf("%s (%s answered %d)", msg, p.Name, status), true +} + +// windowOf is the tokens a provider's model takes, where known: the +// user's (Provider.ContextOf), else its catalog entry's. +func windowOf(p provider.Provider, model string) int { + if n := p.ContextOf(model); n > 0 { + return n + } + for _, e := range provider.Served() { + if e.Group == "" && e.Provider.ID == p.ID && (e.Model == model || e.ID == model) && e.Context > 0 { + return e.Context + } + } + return 0 +} + const ( // lastRetries is how many times the last one left is tried again after // a failure that passes — a busy vendor, a dropped connection. diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index cb880950..433b48b8 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -1443,6 +1443,16 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro stop() } hw.settle() + outgrew := false // a failure that didn't say so was the request's length + if !hw.passing && !hw.refused && hw.code() >= 400 { + if req, err := parse(from, attemptBody); err == nil { + if msg, ok := tooLongUnsaid(c.p, c.model, estimate(req), hw.code(), hw.errBody()); ok { + // told as an overflow: not asked again, of this one or + // another, and nobody rests + hw.failure, hw.failMsg, outgrew = http.StatusBadRequest, msg, true + } + } + } if hw.failure != 0 { // the stream failed before any of it was sent call.Status, call.Error = hw.failure, c.p.Name+": "+hw.failMsg } @@ -1496,7 +1506,7 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro continue } } - if updated && !hw.passing && hw.code() == 400 { + if updated && !outgrew && !hw.passing && hw.code() == 400 { // the upstream turned the update items away: asked again as // the agent sent it, and if that goes, never sent them again plainFor = c.who() + "|" + c.model @@ -1667,6 +1677,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro call.Status, call.Error = other.Status, other.Error writeError(w, from, call.Status, call.Error) break + } else if outgrew { + // in the agent's own words for it, for it to compact + writeError(w, from, call.Status, hw.failMsg) } else { hw.release() } diff --git a/internal/gateway/toolong_unsaid_test.go b/internal/gateway/toolong_unsaid_test.go new file mode 100644 index 00000000..e679c12e --- /dev/null +++ b/internal/gateway/toolong_unsaid_test.go @@ -0,0 +1,66 @@ +package gateway + +import ( + "strings" + "testing" + + "github.com/yetone/magpie/internal/provider" +) + +// Some vendors don't say a conversation is too long for the model: Devin +// answered one of about 225k tokens for swe-1.7-lightning (a 202,752 +// window) with a 502 "capacity issues", WorkBuddy one for glm-5.1 with a +// 400 "Invalid request parameters", Qoder one of 2M for qmodel with a 400 +// "Error in upstream response" — each answered at a size the model holds. +// Asked again in place three times, the agent was then handed a 502 and +// never compacted. A request well past the model's window that fails so +// is told as one too long, at once. +func TestOverflowTheVendorDidntSay(t *testing.T) { + big := strings.Repeat("word ", 2400) // about 3000 tokens as estimated + small := strings.Repeat("word ", 100) // about 125 + near := strings.Repeat("word ", 1760) // about 2200, which may yet fit + for _, tc := range []struct { + name, path, body string + code int + reply string + wantCode, calls int + want string + }{ + {"502 anthropic", "/v1/messages", big, 502, `{"error":{"message":"We are experiencing capacity issues"}}`, 400, 1, "prompt is too long"}, + {"400 chat", "/v1/chat/completions", big, 400, `{"error":{"message":"Invalid request parameters"}}`, 400, 1, "context_length_exceeded"}, + // not the conversation's length + {"small 502", "/v1/messages", small, 502, `{"error":{"message":"We are experiencing capacity issues"}}`, 502, 1 + lastRetries, "capacity issues"}, + {"near the window", "/v1/messages", near, 502, `{"error":{"message":"We are experiencing capacity issues"}}`, 502, 1 + lastRetries, "capacity issues"}, + {"429", "/v1/messages", big, 429, `{"error":{"message":"Rate limit reached"}}`, 429, 1 + rateRetries, "Rate limit"}, + {"401", "/v1/messages", big, 401, `{"error":{"message":"invalid token"}}`, 401, 1, "invalid token"}, + {"quota", "/v1/messages", big, 400, `{"error":{"message":"insufficient balance"}}`, 400, 1, "insufficient balance"}, + } { + t.Run(tc.name, func(t *testing.T) { + proto := provider.Anthropic + if strings.Contains(tc.path, "chat") { + proto = provider.Chat + } + f := &fake{code: tc.code, ctype: "application/json", reply: tc.reply} + up := setup(t, proto, f) + p := provider.Provider{ID: "fake", Name: "Fake", Key: "k", Models: []string{"m1"}, Contexts: map[string]int{"m1": 2000}} + if proto == provider.Chat { + p.Chat = up.URL + "/v1" + } else { + p.Anthropic = up.URL + } + if err := provider.Save(p); err != nil { + t.Fatal(err) + } + req := `{"model":"fake/m1","max_tokens":10,"messages":[{"role":"user","content":"` + tc.body + `"}]}` + code, body := post(t, tc.path, req) + if code != tc.wantCode || !strings.Contains(body, tc.want) || f.calls != tc.calls { + t.Fatalf("status %d after %d calls (want %d after %d): %s", code, f.calls, tc.wantCode, tc.calls, body) + } + if tc.wantCode != 400 || tc.name == "quota" { + if strings.Contains(body, "prompt is too long") || strings.Contains(body, "context_length_exceeded") { + t.Fatalf("told as an overflow: %s", body) + } + } + }) + } +} diff --git a/internal/provider/migrate_qoder.go b/internal/provider/migrate_qoder.go index 311301e9..439f824c 100644 --- a/internal/provider/migrate_qoder.go +++ b/internal/provider/migrate_qoder.go @@ -24,7 +24,7 @@ func init() { func qoderMover(id string) *mover { return &mover{ pkg: "@magpie-community/opencode-qoder-auth", - min: "0.2.3", // Qoder CN as "qoder-cn", its device-token chat as the built-in's; each model's credit rate; the sign-in renewed ahead of time through auth.refresh + min: "0.2.4", // Qoder CN as "qoder-cn", its device-token chat as the built-in's; each model's credit rate; the sign-in renewed ahead of time through auth.refresh; token usage read agents: []string{id}, out: func() ([]Moving, error) { // no refresh of the built-in's runs while the pairs are read