diff --git a/docs/subsystems/gateway-routing.md b/docs/subsystems/gateway-routing.md index 52c416ce..a542022c 100644 --- a/docs/subsystems/gateway-routing.md +++ b/docs/subsystems/gateway-routing.md @@ -22,7 +22,7 @@ served by a plugin are in [Provider and plugin ownership](provider-plugins.md). | Order of keys and accounts | `Routing`: smart (""), `order`, `rotate`, `usage` (least used, with an `usageHalfLife` of 1h), `pace` or `weight` (`weightedFirst`). A failure rests a candidate (`failure`, `failureOf`). | [`routing.go`](../../internal/gateway/routing.go), [`weighted.go`](../../internal/gateway/weighted.go), [`internal/provider/routing.go`](../../internal/provider/routing.go) | | Sink | With `Sink` on, a key or account that answers 429 while it still has quota goes to the back, behind every one not rate limited since. Held in memory until restart. | [`sink.go`](../../internal/gateway/sink.go) | | Affinity | A conversation stays with the key or account that answered it, so the vendor's prompt cache is read again. Modes: auto (""), `session`, `turn`, `off`. Stored in `affinity.json` next to `providers.json`. | [`affinity.go`](../../internal/gateway/affinity.go) | -| One try | `attempt` sends one candidate the request. It goes through the provider's proxy (`p.Via`). A subscription with its own client is served specially, such as Claude through the genuine Claude Code. The request is relayed as is (`passthrough`) when the model is usable on the agent's API. Otherwise it is translated (`translate`). | [`gateway.go`](../../internal/gateway/gateway.go) | +| One try | `attempt` sends one candidate the request. It goes through the provider's proxy (`p.Via`). A subscription with its own client is served specially, such as Claude through the genuine Claude Code. The request is relayed as is (`passthrough`) when the model is usable on the agent's API. Otherwise it is translated (`translate`) to the first API `usable` gives: Chat, except Responses for OpenAI's models where their maker serves them (`ResponsesFirst`) and Anthropic's Messages for a Claude model where the provider has an Anthropic URL (`MessagesFirst`, which keeps its cache breakpoints, #997). `Provider.Native`, the API agents with a per-model API (Pi, omp, droid) are wired to ask on, follows the same order. | [`gateway.go`](../../internal/gateway/gateway.go) | | Trace | The Routing view's record of each decision, written where the decision is made: candidates, rests, which try answered. | [`trace.go`](../../internal/gateway/trace.go) | ## Runtime path @@ -52,8 +52,8 @@ served by a plugin are in [Provider and plugin ownership](provider-plugins.md). ## Verification ```sh -go test -tags nogui ./internal/gateway -run 'TestQuiet|TestSilentHeld|TestRotateSpreadsSessions|TestRouting|TestSmartRouting|TestFallback|TestNoFallbackForOtherErrors|TestLastFallbackErrorReachesTheAgent|TestRateLimitedSinksToTheBack|TestGroupRateLimitedSinks|TestAffinity|TestSeveralKeysOnTakeOverFromEachOther|TestSubscriptionAccountsTakeOver|TestGroup|TestTrace|TestImageTool|TestCodexNoCredits' -go test -tags nogui ./internal/provider -run 'TestGroup|TestCodexCreditsSwitch|TestRenewedAccountForgetsItsAllowance' +go test -tags nogui ./internal/gateway -run 'TestQuiet|TestSilentHeld|TestRotateSpreadsSessions|TestRouting|TestSmartRouting|TestFallback|TestNoFallbackForOtherErrors|TestLastFallbackErrorReachesTheAgent|TestRateLimitedSinksToTheBack|TestGroupRateLimitedSinks|TestAffinity|TestSeveralKeysOnTakeOverFromEachOther|TestSubscriptionAccountsTakeOver|TestGroup|TestTrace|TestImageTool|TestCodexNoCredits|TestClaudeGoesOnMessagesWhereTheRelayHasThem|TestAutoRelaysOnTheClientsOwnAPI' +go test -tags nogui ./internal/provider -run 'TestGroup|TestCodexCreditsSwitch|TestRenewedAccountForgetsItsAllowance|TestNative' ``` The gateway's `TestMain` gives the package a home of its own, so these tests diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 3832e139..831d5b84 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -3120,7 +3120,9 @@ func (s *Server) markUnfit(providerID, model string, proto provider.Protocol) { // the provider says and hasn't turned it away, preferred first: Chat // Completions, which every OpenAI-compatible vendor serves alike, except // for OpenAI's own models where their makers serve them, whose newest are -// Responses-first (and some Responses-only). +// Responses-first (and some Responses-only), and Claude where Anthropic's +// Messages API is served, which alone keeps its cache_control and +// thinking (#997). func (s *Server) usable(p provider.Provider, model string) []provider.Protocol { apis := p.APIs(model) var out []provider.Protocol @@ -3132,6 +3134,9 @@ func (s *Server) usable(p provider.Provider, model string) []provider.Protocol { if p.ResponsesFirst(model) { sort.SliceStable(out, func(i, j int) bool { return out[i] == provider.Responses && out[j] != provider.Responses }) } + if p.MessagesFirst(model) { + sort.SliceStable(out, func(i, j int) bool { return out[i] == provider.Anthropic && out[j] != provider.Anthropic }) + } return out } diff --git a/internal/gateway/messages_first_test.go b/internal/gateway/messages_first_test.go new file mode 100644 index 00000000..085434e0 --- /dev/null +++ b/internal/gateway/messages_first_test.go @@ -0,0 +1,146 @@ +package gateway + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/yetone/magpie/internal/provider" +) + +// relayOfEvery is a relay that answers on Chat, Responses and Anthropic's +// Messages alike, streamed when asked, and records the path and body of +// each request. +func relayOfEvery(t *testing.T) (*httptest.Server, func() (paths []string, bodies []string)) { + t.Helper() + var mu sync.Mutex + var paths, bodies []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + mu.Lock() + paths, bodies = append(paths, r.URL.Path), append(bodies, string(b)) + mu.Unlock() + stream := streamOf(b) + switch { + case r.URL.Path == "/v1/messages" && stream: + w.Header().Set("Content-Type", "text/event-stream") + io.WriteString(w, "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"m\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"x\",\"content\":[],\"usage\":{\"input_tokens\":1,\"output_tokens\":0}}}\n\n"+ + "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n"+ + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"ok\"}}\n\n"+ + "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\n"+ + "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":1}}\n\n"+ + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n") + case r.URL.Path == "/v1/messages": + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"id":"m","type":"message","role":"assistant","model":"x","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}`) + case r.URL.Path == "/v1/responses" && stream: + w.Header().Set("Content-Type", "text/event-stream") + io.WriteString(w, "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"item_id\":\"i\",\"output_index\":0,\"content_index\":0,\"delta\":\"ok\"}\n\n"+ + "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"r\",\"object\":\"response\",\"status\":\"completed\",\"model\":\"x\",\"output\":[{\"type\":\"message\",\"id\":\"i\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"ok\"}]}],\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}}}\n\n") + case r.URL.Path == "/v1/responses": + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"id":"r","object":"response","status":"completed","model":"x","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"ok"}]}],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}`) + case stream: + w.Header().Set("Content-Type", "text/event-stream") + io.WriteString(w, "data: {\"id\":\"c\",\"object\":\"chat.completion.chunk\",\"model\":\"x\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"ok\"}}]}\n\n"+ + "data: {\"id\":\"c\",\"object\":\"chat.completion.chunk\",\"model\":\"x\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\ndata: [DONE]\n\n") + default: + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"id":"c","object":"chat.completion","model":"x","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`) + } + })) + t.Cleanup(srv.Close) + return srv, func() ([]string, []string) { + mu.Lock() + defer mu.Unlock() + p, b := paths, bodies + paths, bodies = nil, nil + return p, b + } +} + +// A Claude model at a relay with an Anthropic URL beside its OpenAI one: a +// request on Messages goes on as it is, cache_control and all, and one on +// an API the relay doesn't serve is translated to Messages, not Chat, where +// cache_control is lost and every turn is billed uncached (#997; +// ReturnTrue on Discord: 0% cache hits until the OpenAI URL was left +// empty). One on an API the relay serves is relayed as the agent asked. +// The two-endpoint relay (an OpenAI URL and an Anthropic one) and the +// one-endpoint relay that speaks all three are both covered. +func TestClaudeGoesOnMessagesWhereTheRelayHasThem(t *testing.T) { + srv, seen := relayOfEvery(t) + for _, c := range []struct { + name string + p provider.Provider + responses string // where Codex's Responses request goes + }{ + {"OpenAI and Anthropic URLs", provider.Provider{Chat: srv.URL + "/v1", Anthropic: srv.URL}, "/v1/messages"}, + {"one URL, three APIs", provider.Provider{Chat: srv.URL + "/v1", Responses: srv.URL + "/v1", Anthropic: srv.URL}, "/v1/responses"}, + } { + t.Run(c.name, func(t *testing.T) { + fresh(t) + c.p.ID, c.p.Name, c.p.Key, c.p.Models = "relay", "Relay", "k", []string{"claude-sonnet-4-5"} + if err := provider.Save(c.p); err != nil { + t.Fatal(err) + } + for _, r := range []struct{ path, body string }{ + {"/v1/messages", `{"model":"relay/claude-sonnet-4-5","max_tokens":10,"stream":true,"system":[{"type":"text","text":"you are","cache_control":{"type":"ephemeral"}}],"messages":[{"role":"user","content":"hi"}]}`}, + {"/v1/responses", `{"model":"relay/claude-sonnet-4-5","stream":true,"input":"hi"}`}, + } { + seen() + if code, b := post(t, r.path, r.body); code != 200 { + t.Fatalf("%s: %d %s", r.path, code, b) + } + paths, bodies := seen() + want := "/v1/messages" + if r.path == "/v1/responses" { + want = c.responses + } + if len(paths) != 1 || paths[0] != want { + t.Fatalf("%s for a Claude model went to %v, want %s", r.path, paths, want) + } + if r.path == "/v1/messages" && !strings.Contains(bodies[0], `"cache_control"`) { + t.Errorf("cache_control lost on the way: %s", bodies[0]) + } + } + // Chat for Claude is relayed as Chat: the agent asked it so + seen() + if code, b := post(t, "/v1/chat/completions", `{"model":"relay/claude-sonnet-4-5","messages":[{"role":"user","content":"hi"}]}`); code != 200 { + t.Fatalf("chat: %d %s", code, b) + } + if paths, _ := seen(); len(paths) != 1 || paths[0] != "/v1/chat/completions" { + t.Errorf("chat for Claude went to %v, want it relayed as /v1/chat/completions", paths) + } + }) + } +} + +// Every API the client spoke is relayed on the same API when the provider +// serves it, for any model: Codex's Responses on Responses, Claude Code's +// Messages on Messages, Chat on Chat (#997: "自动好像只会走chat"). +func TestAutoRelaysOnTheClientsOwnAPI(t *testing.T) { + srv, seen := relayOfEvery(t) + fresh(t) + if err := provider.Save(provider.Provider{ID: "relay", Name: "Relay", Key: "k", Chat: srv.URL + "/v1", Responses: srv.URL + "/v1", Anthropic: srv.URL, + Models: []string{"glm-5", "gpt-5.1", "claude-sonnet-4-5"}}); err != nil { + t.Fatal(err) + } + for _, model := range []string{"glm-5", "gpt-5.1", "claude-sonnet-4-5"} { + for _, r := range []struct{ path, body string }{ + {"/v1/messages", `{"model":"relay/` + model + `","max_tokens":10,"messages":[{"role":"user","content":"hi"}]}`}, + {"/v1/responses", `{"model":"relay/` + model + `","input":"hi"}`}, + {"/v1/chat/completions", `{"model":"relay/` + model + `","messages":[{"role":"user","content":"hi"}]}`}, + } { + seen() + if code, b := post(t, r.path, r.body); code != 200 { + t.Fatalf("%s %s: %d %s", model, r.path, code, b) + } + if paths, _ := seen(); len(paths) != 1 || paths[0] != r.path { + t.Errorf("%s on %s went to %v", model, r.path, paths) + } + } + } +} diff --git a/internal/provider/messages_first_test.go b/internal/provider/messages_first_test.go new file mode 100644 index 00000000..134db32f --- /dev/null +++ b/internal/provider/messages_first_test.go @@ -0,0 +1,32 @@ +package provider + +import "testing" + +// The API a model is wired to agents on (Pi, omp, droid each speak the one +// magpie names for a model): a Claude model at a relay with an Anthropic URL +// beside its OpenAI one is Anthropic's Messages, not Chat, which drops its +// cache_control (#997). Other models, and a provider without an Anthropic +// URL, are as before. +func TestNativeClaudeIsMessagesWhereServed(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + both := Provider{ID: "relay", Chat: "https://relay.example/v1", Anthropic: "https://relay.example"} + every := Provider{ID: "relay", Chat: "https://relay.example/v1", Responses: "https://relay.example/v1", Anthropic: "https://relay.example"} + chatOnly := Provider{ID: "relay", Chat: "https://relay.example/v1"} + for _, c := range []struct { + p Provider + model string + want Protocol + }{ + {both, "claude-sonnet-4-5", Anthropic}, + {every, "claude-opus-4-1-20250805", Anthropic}, + {every, "anthropic/claude-sonnet-4.5", Anthropic}, + {both, "glm-5", Chat}, + {every, "gpt-5.1", Chat}, + {chatOnly, "claude-sonnet-4-5", Chat}, + } { + if got := c.p.Native(c.model); got != c.want { + t.Errorf("%+v Native(%s) = %q, want %q", c.p, c.model, got, c.want) + } + } +} diff --git a/internal/provider/native_test.go b/internal/provider/native_test.go index 27ba21de..2a79397c 100644 --- a/internal/provider/native_test.go +++ b/internal/provider/native_test.go @@ -39,7 +39,7 @@ func TestNative(t *testing.T) { {"Chat and Responses elsewhere", Provider{ID: "both", Chat: "https://x.test/v1", Responses: "https://x.test/v1"}, "gpt-5.5", Chat}, {"anthropic only", Provider{ID: "anthropic", Anthropic: "https://api.anthropic.com"}, "claude-sonnet-5", Anthropic}, {"Copilot GPT", copilot, "gpt-5.5", Responses}, - {"Copilot Claude", copilot, "claude-sonnet-5", Chat}, + {"Copilot Claude", copilot, "claude-sonnet-5", Anthropic}, {"Cursor", Provider{ID: "cursor", Chat: "https://cursor.test/v1", Account: &Account{Agent: "cursor"}}, "auto", ""}, {"Claude Code", Provider{ID: "claude", Anthropic: "https://api.anthropic.com", Account: &Account{Agent: "claude"}}, "claude-sonnet-5", ""}, {"nothing", Provider{ID: "none"}, "m", ""}, diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 2d12b00b..f81021a0 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -1028,6 +1028,23 @@ func (p Provider) ResponsesFirst(model string) bool { return openAIModel(model) } +// MessagesFirst: a Claude model where the provider has Anthropic's Messages +// API, which is best asked there though its Chat or Responses API serves +// it too — a relay that serves both drops cache_control on Chat, so every +// turn was billed uncached (#997; ReturnTrue on Discord: 0% cache hits +// until the relay's OpenAI URL was left empty), and thinking with it. +func (p Provider) MessagesFirst(model string) bool { + return p.Anthropic != "" && claudeModel(model) +} + +// claudeModel is whether model is one of Anthropic's Claude models by its +// name, after any vendor prefix (anthropic/claude-sonnet-4.5) or as +// Bedrock names it. +func claudeModel(model string) bool { + m := strings.ToLower(model[strings.LastIndex(model, "/")+1:]) + return strings.HasPrefix(m, "claude") || bedrockClaude(m) +} + // openAIModel is whether model is one of OpenAI's own by its name: a GPT, // a Codex or an o-series model, after any vendor prefix. func openAIModel(model string) bool { @@ -1039,7 +1056,8 @@ func openAIModel(model string) bool { // Native is the API model is best asked on at this provider: one it serves // the model on itself, so a request on it is relayed as it is rather than // translated — Responses for a ChatGPT sign-in, or an OpenAI model on -// OpenAI's API or Copilot's; Chat where that is served. "" when every +// OpenAI's API or Copilot's; Anthropic's Messages for a Claude model +// where that is served; Chat where that is served. "" when every // request is translated anyway: a sign-in served through its agent's own // API (Claude Code's binary, Cursor, Devin, Kiro, Code Assist). func (p Provider) Native(model string) Protocol { @@ -1062,6 +1080,9 @@ func (p Provider) Native(model string) Protocol { if p.ResponsesFirst(model) && slices.Contains(out, Responses) { return Responses } + if p.MessagesFirst(model) && slices.Contains(out, Anthropic) { + return Anthropic + } return out[0] }