diff --git a/README.md b/README.md index 06da8c23..c2d226e0 100644 --- a/README.md +++ b/README.md @@ -283,6 +283,72 @@ at start-up. A session already running therefore keeps the window it began with, while the gateway's own `/models` and every request from then on are right at once. +### The name a vendor knows a model by + +A relay often serves a model under an id of its own — a prefix it namespaces +with, a dated name, a `-latest` that is not what models.dev calls it. Say +which name to ask for: + +```sh +magpie model wire relay-b/model-2 # what the vendor is asked for +magpie model wire relay-b/model-2 vendor-c/model-2-preview # ask for it by this +magpie model wire 'relay-b/*' 'vendor-c/*' # every model, * being the model +magpie model wire relay-b/model-2 --reset # ask for it by its own name again +magpie model wires # every name your vendors are asked for models by +``` + +Quote the arguments with a `*` in them: zsh reads a bare `*` as a glob and +answers `no matches found`. In the name, `*` stands for the model itself, so +`'relay-b/*' 'vendor-c/*'` sends `vendor-c/model-3` for `model-3` and +`vendor-c/model-2` for `model-2` — one name for a relay that namespaces its +models, each still asked for by its own. A name with no `*` in it sends every +model of that key under that one name, which is the right answer only for a +relay that does serve them all alike. The model's own key wins over the +provider's, and `--reset` takes away only the one it is given, so resetting +`relay-b/model-2` while `'relay-b/*'` is set leaves the provider's name in +force — the CLI says which of the two is in force after every change, naming +the models a name for the whole provider leaves to their own, and +`--reset` over a name that was never given says so rather than ticking a +removal that took nothing away. + +Only the request that goes out carries that name, in the model field of a +chat, Responses or Anthropic Messages request and of an Anthropic token +count; image requests are left with the name magpie knows the model by. +Gemini CLI and Antigravity sign-ins go on Code Assist, and there the model +an effort picks is a variant of the model magpie knows — `gemini-3.7-flash` +at `high` is sent as `gemini-3.7-flash-high` — so a `*` in the name is that +variant: `'antigravity/*' 'vendor-c/*'` asks for `vendor-c/gemini-3.7-flash-high` +and `vendor-c/gemini-3.7-flash-low` each by its own, as it does everywhere +else. A name with no `*` in it, or one given for `antigravity/gemini-3.7-flash` +itself, is that one name at every level. + +Everything else keeps the name magpie knows the model by: the catalog agents +pick from, the routing groups' membership, `GET /models`, and what a call is +recorded and priced as. What the vendor's own reply said answered is kept +beside that, in the ledger's `served_model`, which is what it is for, and it +is compared with the name the vendor was asked for — so a model answered +under the relay's own id is not read as a swap. That is against the names +in force when the ledger is read, since a record keeps what the vendor +answered and not the name the request went out under: naming a model after +the call re-judges that call, which is then left reading as a swap. What a +provider *supports* — whether it takes a temperature, which reasoning levels +fit — is still asked about the model magpie knows, so a rename upstream does +not change how magpie behaves towards the model. + +An upstream name is **one provider's**, not the model's: another provider +serving the same id is asked for it under its own name, or this one. The +model it is given for has to be one that provider serves — `/*` +is the way to say one for every model — because a name for a model magpie +would never ask the provider for is not a name of its own: the model is +asked for by the name magpie knows it by, and nothing anywhere would say the +name given for it is not the one in force. A relay that serves a model under +an id of its own *and* lists it does get the name; what is refused is a +model the provider's list does not have at all. The model test uses the name +too, so a relay that only knows its own ids does not report a working model +as broken. An image model is the exception that follows the rule above: it is +tested by the name magpie knows it by, on the images API and on the chat it +falls back to, because that is how a drawing is asked for. + ### Routing groups A routing group is several models, from one provider or many, that an agent diff --git a/internal/gateway/codeassist.go b/internal/gateway/codeassist.go index e4594eb6..80dba831 100644 --- a/internal/gateway/codeassist.go +++ b/internal/gateway/codeassist.go @@ -23,59 +23,77 @@ const skipSignature = "skip_thought_signature_validator" var unsafeToolID = regexp.MustCompile(`[^a-zA-Z0-9_-]`) -// antigravityVariants are the ids of Antigravity's a model magpie offers -// stands for, by level, and antigravityBaseOf the model and level one of -// those ids is; vars so tests can stand in for them. -var ( - antigravityVariants = provider.AntigravityVariants - antigravityBaseOf = provider.AntigravityBase -) +// antigravityBaseOf is the model an id of Antigravity's own is one level +// of; a var so tests can stand in for it. +var antigravityBaseOf = provider.AntigravityBase -// antigravityModelID is Antigravity's id for a model magpie offers, at -// the effort asked for: the family's variant at it, or at the nearest -// level it has; with none asked, the family's default. One of -// Antigravity's own ids at a level (gemini-3.7-flash-low, one an agent -// was set to before) goes as it is when no effort is asked, and else is -// the family's variant at the effort asked for: the effort the client -// picks wins, as it does for the family. Any other id goes as it is. -func antigravityModelID(model, effort string) string { - if base, _, ok := antigravityBaseOf(model); ok { - if effort == "" { - return model - } - model = base +// accountAgent is the agent behind p's account, and "" where p is one of the +// user's own and has none: the question a rule keyed by the account has to be +// able to answer for, and one a provider's id cannot answer — a relay added +// under a built-in's id is that relay and not that account. +func accountAgent(p provider.Provider) string { + if p.Account == nil { + return "" } - vs, ok := antigravityVariants(model) - if !ok { - return model + return p.Account.Agent +} + +// codeAssistID is the id a request on the account's app goes out under: on +// Antigravity the variant the effort picks for a model that is a family of +// levels, and elsewhere the model magpie knows. It is keyed by the account +// (accountAgent) rather than by the provider's id, since that is the account +// the variants belong to, and it is the id a ledger judges what the vendor +// answered against (provider.AntigravitySentID), so the gateway and the +// ledger read one rule rather than one each. +// The envelope is built under it and named with it, so it is worked out +// once where both of those are and handed down. +func codeAssistID(model, effort, agent string) string { + if agent == "antigravity" { + return provider.AntigravitySentID(model, effort) } - if effort == "" { - return vs[""] + return model +} + +// codeAssistBody is the envelope for a request on p's account, with the +// vendor's own name for the model in its model field, over the names in +// force (wires). It is named with the id the envelope carries, which on +// Antigravity is the variant the effort picked (codeAssistID): a name given +// for the whole provider then asks for each level by its own id instead of +// one id for every level, so the effort keeps choosing and the relay is +// asked for the model it serves at that level. +// +// The id is worked out once here and handed to the builder, since the name +// in force is looked up against the very id the envelope is addressed to: +// asking twice reads the vendor's catalogue twice over for one answer. +func codeAssistBody(p provider.Provider, r *Request, model string, wires map[string]string) []byte { + sent := codeAssistID(model, r.Effort, p.Account.Agent) + body := buildCodeAssistSent(r, sent, p.Account.Agent) + if wire := provider.UpstreamNameAsIn(wires, p.ID, model, sent); wire != sent { + body = rewriteModel(body, wire) } - var levels []string - for _, l := range effortRank { - if vs[l] != "" { - levels = append(levels, l) - } - } - if id := vs[fitEffort(effort, levels)]; id != "" { - return id - } - return vs[""] + return body } // buildCodeAssist builds the Code Assist envelope for a request, for the // app the account belongs to: "gemini" or "antigravity". On Antigravity a // model that is a family of levels is asked for as the variant the effort -// picks (antigravityModelID). +// picks (codeAssistID). func buildCodeAssist(r *Request, model, agent string) []byte { + return buildCodeAssistSent(r, codeAssistID(model, r.Effort, agent), agent) +} + +// buildCodeAssistSent is buildCodeAssist with the id the envelope is +// addressed to already worked out, for the caller that names the envelope +// with that same id and would otherwise work it out a second time. There is +// no model but that id in here: on Antigravity the two differ, and the one +// the request goes out under is the one everything below is shaped from. +func buildCodeAssistSent(r *Request, sent, agent string) []byte { ag := agent == "antigravity" fixed := false // the id says the level it thinks at if ag { - model = antigravityModelID(model, r.Effort) - _, _, fixed = antigravityBaseOf(model) + _, _, fixed = antigravityBaseOf(sent) } - claude := strings.Contains(strings.ToLower(model), "claude") + claude := strings.Contains(strings.ToLower(sent), "claude") toolID := func(id string) string { if !ag || id == "" { return id @@ -201,7 +219,7 @@ func buildCodeAssist(r *Request, model, agent string) []byte { if len(r.Stop) > 0 { gen["stopSequences"] = r.Stop } - if tc := thinkingConfig(r, model, claude, fixed); tc != nil { + if tc := thinkingConfig(r, sent, claude, fixed); tc != nil { gen["thinkingConfig"] = tc // Claude's answer has to have room past its thinking if b, ok := tc["thinkingBudget"].(int); ok && claude && gen["maxOutputTokens"] == nil { @@ -211,7 +229,7 @@ func buildCodeAssist(r *Request, model, agent string) []byte { if len(gen) > 0 { req["generationConfig"] = gen } - b, _ := json.Marshal(map[string]any{"model": model, "request": req}) + b, _ := json.Marshal(map[string]any{"model": sent, "request": req}) return b } diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index b79c5faf..27b65be0 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -63,6 +63,33 @@ func agentOf(r *http.Request) string { // "" leaves the model as asked. Set by main. var StandIn func(agent, model string) string +// wiresKey carries the upstream names in force on a request's context: read +// once where the request comes in rather than once per place a name is +// looked up, since settings.Load reads and parses the whole file every call +// and one chat request looks names up in several places — the id a try went +// out under, the body the vendor is sent, the token count asked of it. The +// names of one request are then one set, the way the ledger reads them once +// for a whole report. +type wiresKey struct{} + +// withWires carries the names in force on r's context, read at most once and +// only where something asks for one: a request refused before any name is +// looked up pays for nothing. +func withWires(r *http.Request) *http.Request { + load := sync.OnceValue(func() map[string]string { return settings.Load().ModelWires }) + return r.WithContext(context.WithValue(r.Context(), wiresKey{}, load)) +} + +// wiresOf is the names in force for ctx: the ones withWires read, or the +// ones in force now for a request magpie made for itself and never carried +// them on. +func wiresOf(ctx context.Context) map[string]string { + if load, ok := ctx.Value(wiresKey{}).(func() map[string]string); ok { + return load() + } + return settings.Load().ModelWires +} + // standIn is StandIn's model for one magpie shows no entry for: not a // catalog id or model, nor a ready provider's "provider/model". func standIn(agent, asked string) string { @@ -563,8 +590,11 @@ func (s *Server) countTokens(w http.ResponseWriter, r *http.Request) { } } } + // the names in force, read at most once however many candidates are + // counted, and not at all where there are none + wires := sync.OnceValue(func() map[string]string { return settings.Load().ModelWires }) for i, c := range counts { - res, err := s.forward(r.Context(), c.p, provider.Anthropic, "/v1/messages/count_tokens", rewriteModel(body, model), r.Header) + res, err := s.forward(r.Context(), c.p, provider.Anthropic, "/v1/messages/count_tokens", rewriteModel(body, provider.UpstreamNameIn(wires(), c.p.ID, model)), r.Header) if err != nil { if r.Context().Err() == nil { s.restAfter(c, http.StatusBadGateway, nil, []byte(err.Error())) @@ -731,6 +761,9 @@ func estimate(req *Request) int { // serve routes one parsed-enough request to its provider. func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Protocol, body []byte) { start := time.Now() + // the upstream names in force for this request, read once here rather + // than once per place a name is looked up below + r = withWires(r) // secrets go as placeholders and come back as they were; the log has // what the vendor saw and said w, body, unmask := redacted(w, body) @@ -1077,7 +1110,7 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro call.Error = refusedError(c.p, c.model, hw.failMsg) } try := Try{ID: c.rest, Model: c.model, Effort: sent, Picked: picked, Fixed: c.effort, Start: began, Done: true, Status: call.Status, Millis: time.Since(began).Milliseconds(), Error: call.Error, - Served: call.Usage.Served, Swapped: swapped(c.model, call.Usage.Served)} + Served: call.Usage.Served, Swapped: swapped(provider.SentNameOnIn(wiresOf(r.Context()), c.p.ID, accountAgent(c.p), c.model, sent), call.Usage.Served)} try.TTFT, try.FirstText = hw.first.ms() // the request's, from when it came as its ms are: the time before // this try, the ones that failed first, is in it @@ -1566,7 +1599,7 @@ func codexClientHeader(k string) bool { // written, when the provider serves the model on another of its endpoints // but not this one. func (s *Server) passthrough(w http.ResponseWriter, r *http.Request, p provider.Provider, proto provider.Protocol, model string, body []byte, u *Usage) (status int, msg string, done bool) { - body = rewriteModel(body, model) + body = rewriteModel(body, provider.UpstreamNameIn(wiresOf(r.Context()), p.ID, model)) searchFn := false // Codex's tool search sent as a function switch proto { case provider.Responses: @@ -1864,6 +1897,9 @@ func (s *Server) forwardTranslated(ctx context.Context, p provider.Provider, to r.Effort, req = e, &r } } + // the names in force, read once however many endpoints the request is + // built for below + wires := wiresOf(ctx) web := req.WebSearch // the cache key was left out to see if it was what the upstream refused dropped := false @@ -1891,7 +1927,16 @@ func (s *Server) forwardTranslated(ctx context.Context, p provider.Provider, to body = asCompletionTokens(body) } if to == provider.CodeAssist && p.Account != nil { - body = buildCodeAssist(req, model, p.Account.Agent) + // the envelope is named in there, where the id it carries is + // known: on Antigravity that is the variant the effort picked, + // not the model magpie knows + body = codeAssistBody(p, req, model, wires) + } else if wire := provider.UpstreamNameIn(wires, p.ID, model); wire != model { + // the vendor's own name for the model goes in the model field + // and nowhere else: everything above shaped the request from + // the model magpie knows, which is what those decisions are + // about + body = rewriteModel(body, wire) } res, err := s.forward(ctx, p, to, pathOf(to), p.Prepare(body), in) if err != nil || res.StatusCode < 400 { diff --git a/internal/gateway/wire_test.go b/internal/gateway/wire_test.go new file mode 100644 index 00000000..a57f9738 --- /dev/null +++ b/internal/gateway/wire_test.go @@ -0,0 +1,1254 @@ +package gateway + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/yetone/magpie/internal/catalog" + "github.com/yetone/magpie/internal/provider" + "github.com/yetone/magpie/internal/settings" + "github.com/yetone/magpie/internal/usage" +) + +// asked is what a vendor was asked for, in order, from the several +// goroutines a model test runs at once. +type asked struct { + mu sync.Mutex + models []string +} + +func (a *asked) add(model string) { + a.mu.Lock() + a.models = append(a.models, model) + a.mu.Unlock() +} + +func (a *asked) all() []string { + a.mu.Lock() + defer a.mu.Unlock() + return slices.Clone(a.models) +} + +// wire takes what the vendor was asked for since the last call and says +// whether every one of those requests named a model the way a turn does: +// by the name on the wire, and nothing else. Taking it empties the record, +// so a probe that went out under the name magpie knows can't be vouched +// for by a later probe that went out under the right one. +func (a *asked) wire(t *testing.T, what string, want []string) []string { + t.Helper() + a.mu.Lock() + got := slices.Clone(a.models) + a.models = nil + a.mu.Unlock() + if len(got) == 0 { + t.Errorf("%s asked the vendor for no model at all", what) + } + for _, m := range got { + if !slices.Contains(want, m) { + t.Errorf("%s asked the vendor for %q; a turn only ever carries %v", what, m, want) + } + } + return got +} + +// asking is a vendor that answers with the model it was asked for, as a +// relay does when it serves models under ids of its own, and records what it +// was asked for. +func asking(t *testing.T, a *asked) *httptest.Server { + t.Helper() + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/models") { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"data":[{"id":"model-2"},{"id":"model-3"}]}`) + return + } + var b struct { + Model string `json:"model"` + } + json.NewDecoder(r.Body).Decode(&b) + a.add(b.Model) + w.Header().Set("Content-Type", "text/event-stream") + io.WriteString(w, `data: {"id":"x","model":`+strconv.Quote(b.Model)+ + `,"choices":[{"index":0,"delta":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],`+ + `"usage":{"prompt_tokens":5,"completion_tokens":3}}`+"\n\n") + io.WriteString(w, "data: [DONE]\n\n") + })) + t.Cleanup(s.Close) + return s +} + +// putProvider adds a provider to the file the configured and the signed-in +// ones are merged from, for a provider whose id Save reserves and a test +// still needs magpie to be able to find. +func putProvider(t *testing.T, p map[string]any) { + t.Helper() + b, err := json.Marshal(map[string]any{"providers": []map[string]any{p}}) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(provider.Path()), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(provider.Path(), b, 0o600); err != nil { + t.Fatal(err) + } +} + +// A relay that serves a model under an id of its own is asked for it by that +// id, and by nothing else: the catalog agents see, the routing groups and the +// usage records keep the name magpie knows the model by, so a rename upstream +// is not a rename here. +func TestUpstreamNameReachesTheVendorOnly(t *testing.T) { + fresh(t) + + var seen asked + relay, other := asking(t, &seen), asking(t, &seen) + + for _, p := range []provider.Provider{ + {ID: "relay-b", Name: "Relay B", Key: "k", Models: []string{"model-2"}, Chat: relay.URL}, + {ID: "other", Name: "Other", Key: "k", Models: []string{"model-2"}, Chat: other.URL}, + } { + if err := provider.Save(p); err != nil { + t.Fatal(err) + } + } + if err := provider.SetUpstreamName("relay-b/model-2", "vendor-c/model-2-preview"); err != nil { + t.Fatal(err) + } + + s := New() + ask := func(model string) { + rec := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/v1/messages", strings.NewReader( + `{"model":"`+model+`","max_tokens":1024,"stream":true,`+ + `"messages":[{"role":"user","content":"hi"}]}`)) + s.Handler().ServeHTTP(rec, req) + if rec.Code != 200 { + t.Fatalf("%s: status %d: %s", model, rec.Code, rec.Body.String()) + } + } + + // the relay is asked for the model by the name it serves it under + ask("relay-b/model-2") + if got := seen.all(); len(got) != 1 || got[0] != "vendor-c/model-2-preview" { + t.Fatalf("the relay was asked for %v; want the upstream name", got) + } + + // another provider serving the same model id is left alone: an upstream + // name is one provider's, not the model's + ask("other/model-2") + if got := seen.all(); len(got) != 2 || got[1] != "model-2" { + t.Fatalf("the other provider was asked for %v", got[1:]) + } + + // the catalog still knows the model by magpie's name for it + found := false + for _, e := range provider.Catalog() { + if e.ID == "relay-b/model-2" { + found = true + if e.Model != "model-2" { + t.Errorf("the catalog shows the model as %q; the canonical name is what agents see", e.Model) + } + } + } + if !found { + t.Fatal("relay-b/model-2 is not in the catalog") + } + if _, model, ok := provider.Resolve("relay-b/model-2"); !ok || model != "model-2" { + t.Errorf("Resolve gave %q; the wire name is not the model's identity", model) + } + + // a call is recorded under the canonical name; what the vendor's own + // answer said it was is kept beside it, which is what that field is for + rows := usage.Load(time.Time{}) + if len(rows) == 0 { + t.Fatal("no call was recorded") + } + for _, r := range rows { + if r.Model != "model-2" { + t.Errorf("a call was recorded as %q; the vendor's name is not the model's", r.Model) + } + } + if rows[0].Served != "vendor-c/model-2-preview" { + t.Errorf("the record says the vendor served %q", rows[0].Served) + } + + // taking the user's name away asks for the model's own again + if err := provider.SetUpstreamName("relay-b/model-2", ""); err != nil { + t.Fatal(err) + } + ask("relay-b/model-2") + if got := seen.all(); got[len(got)-1] != "model-2" { + t.Fatalf("after --reset the relay was asked for %q", got[len(got)-1]) + } +} + +// A name given for a provider outlives that provider, and it outlives it by +// the id: the provider can be deleted and another take the same id, and the +// name goes on asking that one's models for theirs. So a removal has to work +// off the key the name is stored at — which is all `magpie model wire +// relay-b/model-2 --reset` has left to work off once the provider is gone — +// and here it is: with the name off, the relay that took the id is asked for +// its own model by its own id. +func TestUpstreamNameDroppedWithItsProvider(t *testing.T) { + fresh(t) + + var gone, here asked + was, now := asking(t, &gone), asking(t, &here) + + for _, p := range []provider.Provider{ + {ID: "relay-b", Name: "Relay B", Key: "k", Models: []string{"model-2"}, Chat: was.URL}, + } { + if err := provider.Save(p); err != nil { + t.Fatal(err) + } + } + if err := provider.SetUpstreamName("relay-b/model-2", "vendor-c/model-2-preview"); err != nil { + t.Fatal(err) + } + if err := provider.Delete("relay-b"); err != nil { + t.Fatal(err) + } + // the name outlived the provider that had it, and the id is free: a + // relay taking it is asked for the model under the deleted one's + // vendor's name, which is the hazard the removal is for + if err := provider.Save(provider.Provider{ID: "relay-b", Name: "Relay B", Key: "k", + Models: []string{"model-2"}, Chat: now.URL}); err != nil { + t.Fatal(err) + } + + s := New() + ask := func(model string) { + rec := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/v1/messages", strings.NewReader( + `{"model":"`+model+`","max_tokens":1024,"stream":true,`+ + `"messages":[{"role":"user","content":"hi"}]}`)) + s.Handler().ServeHTTP(rec, req) + if rec.Code != 200 { + t.Fatalf("%s: status %d: %s", model, rec.Code, rec.Body.String()) + } + } + + ask("relay-b/model-2") + if got := here.all(); len(got) != 1 || got[0] != "vendor-c/model-2-preview" { + t.Fatalf("the relay that took the id was asked for %v; want the name the deleted one had", got) + } + + if dropped, err := provider.DropUpstreamName("relay-b/model-2"); err != nil || !dropped { + t.Fatalf("DropUpstreamName = %v, %v; want the name taken off by its key", dropped, err) + } + ask("relay-b/model-2") + if got := here.all(); len(got) != 2 || got[1] != "model-2" { + t.Fatalf("with the name off the relay that took the id was asked for %v; want model-2", got[1:]) + } + if got := gone.all(); len(got) != 0 { + t.Errorf("the deleted relay's was asked for %v; nothing sends to it now", got) + } +} + +// The same removal, with a provider shown by the id whose name is gone: the +// name a provider is shown by is a spelling any other provider answers to as +// well, so a ref spelled with the id b's provider had names that provider +// rather than the one the name under it is for. Taken through that name, the +// removal would take away the other provider's own name — a name that is +// really in force — and leave the one that has this relay's models going out +// renamed to a vendor that never heard of them, for whoever takes the id. +func TestUpstreamNameDroppedUnderAProviderShownAsTheGoneId(t *testing.T) { + fresh(t) + + var here, that asked + now, other := asking(t, &here), asking(t, &that) + + if err := provider.Save(provider.Provider{ID: "b", Name: "B", Key: "k", + Models: []string{"model-2"}, Chat: now.URL}); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("b/model-2", "vendor-c/model-2-preview"); err != nil { + t.Fatal(err) + } + if err := provider.Delete("b"); err != nil { + t.Fatal(err) + } + if err := provider.Save(provider.Provider{ID: "b", Name: "B Two", Key: "k", + Models: []string{"model-2"}, Chat: now.URL}); err != nil { + t.Fatal(err) + } + + s := New() + ask := func(model string) { + rec := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/v1/messages", strings.NewReader( + `{"model":"`+model+`","max_tokens":1024,"stream":true,`+ + `"messages":[{"role":"user","content":"hi"}]}`)) + s.Handler().ServeHTTP(rec, req) + if rec.Code != 200 { + t.Fatalf("%s: status %d: %s", model, rec.Code, rec.Body.String()) + } + } + + // the id b is free again, and the relay that took it is asked for its + // model under the deleted one's vendor's name + ask("b/model-2") + if got := here.all(); len(got) != 1 || got[0] != "vendor-c/model-2-preview" { + t.Fatalf("the relay that took the id was asked for %v; want the name the deleted one had", got) + } + + // that relay goes too, leaving the name in force for the id b still, and + // a relay of an id of its own is now shown by the id b — with a name of + // its own, which is the one a removal taken through that name would take + // away instead + if err := provider.Delete("b"); err != nil { + t.Fatal(err) + } + if err := provider.Save(provider.Provider{ID: "c", Name: "B", Key: "k", + Models: []string{"model-2"}, Chat: other.URL}); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("B/model-2", "vendor-c/model-2"); err != nil { + t.Fatal(err) + } + ask("c/model-2") + if got := that.all(); len(got) != 1 || got[0] != "vendor-c/model-2" { + t.Fatalf("the relay shown as B was asked for %v; want the name given for it", got) + } + + if dropped, err := provider.DropUpstreamName("b/model-2"); err != nil || !dropped { + t.Fatalf("DropUpstreamName = %v, %v; want the name taken off by its key", dropped, err) + } + if got := settings.Load().ModelWires; len(got) != 1 || got["c/model-2"] != "vendor-c/model-2" { + t.Fatalf("the settings hold %v; want the name the deleted relay had gone, and the other left in force", got) + } + + // which is all taking that name off does to what goes out: the relay + // shown as B is still asked for its model by the name given for it, and + // the one that took the freed id b is asked for its own + ask("c/model-2") + if got := that.all(); len(got) != 2 || got[1] != "vendor-c/model-2" { + t.Fatalf("with the other name off the relay shown as B was asked for %v; want the name given for it", got[1:]) + } + if got := here.all(); len(got) != 1 { + t.Errorf("the relay that took the id was asked for %v again; nothing sends to it now", got) + } + if err := provider.Save(provider.Provider{ID: "b", Name: "B Two", Key: "k", + Models: []string{"model-2"}, Chat: now.URL}); err != nil { + t.Fatal(err) + } + ask("b/model-2") + if got := here.all(); len(got) != 2 || got[1] != "model-2" { + t.Fatalf("with the name off the relay that took the id was asked for %v; want model-2", got[1:]) + } +} + +// A name given for a provider's every model has the model in it, so each +// model is asked for by its own: one name covers a relay that namespaces its +// models, and none of them collapses into another's — which is what the +// record, and the bill read from it, would otherwise say answered. +func TestUpstreamWildcardNamesEachModel(t *testing.T) { + fresh(t) + + var seen asked + relay := asking(t, &seen) + if err := provider.Save(provider.Provider{ID: "relay-b", Name: "Relay B", Key: "k", + Models: []string{"model-2", "model-3"}, Chat: relay.URL}); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("relay-b/*", "vendor-c/sol-*"); err != nil { + t.Fatal(err) + } + + s := New() + for _, model := range []string{"model-2", "model-3"} { + rec := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/v1/messages", strings.NewReader( + `{"model":"relay-b/`+model+`","max_tokens":1024,"stream":true,`+ + `"messages":[{"role":"user","content":"hi"}]}`)) + s.Handler().ServeHTTP(rec, req) + if rec.Code != 200 { + t.Fatalf("%s: status %d: %s", model, rec.Code, rec.Body.String()) + } + } + want := []string{"vendor-c/sol-model-2", "vendor-c/sol-model-3"} + if got := seen.all(); len(got) != 2 || got[0] != want[0] || got[1] != want[1] { + t.Fatalf("the relay was asked for %v; want %v — each model by its own name", got, want) + } + + // the reply naming the model it was asked for is that model answering: + // "vendor-c/sol-model-3" is not another model than the one asked for + r := lastRoute(s) + if r.Tries[0].Model != "model-3" { + t.Errorf("the try is for %q; a wire name is not the model's identity", r.Tries[0].Model) + } + if r.Served != "vendor-c/sol-model-3" { + t.Errorf("the route says %q served", r.Served) + } + if r.Swapped || r.Tries[0].Swapped { + t.Errorf("the model answered under the relay's own id is marked swapped: %+v", r.Tries[0]) + } + // and the ledger, which reads the records back, marks it the same way + rows, _, _ := usage.Ledger(usage.Month, usage.Filter{}) + if len(rows) == 0 { + t.Fatal("the ledger has no rows") + } + for _, row := range rows { + if row.Model != "model-2" && row.Model != "model-3" { + t.Errorf("a row is for %q; a wire name is not the model's", row.Model) + } + if row.Swapped { + t.Errorf("a row is marked swapped though the vendor served what it was asked for: %+v", row) + } + } +} + +// The first test drives a request that has to be translated, because the +// relay only speaks Chat Completions. These two cover the paths that test +// cannot reach: a request the provider understands as it came, which magpie +// relays without rebuilding, and the token count asked of the vendor. +func TestUpstreamNameOnNativeAndCountPaths(t *testing.T) { + for _, tc := range []struct { + name string + path string + }{ + {"a request relayed as it came", "/v1/messages"}, + {"a token count asked of the vendor", "/v1/messages/count_tokens"}, + } { + t.Run(tc.name, func(t *testing.T) { + fresh(t) + var seen asked + // the vendor's own reply, naming the model it was asked for + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var b struct { + Model string `json:"model"` + } + json.NewDecoder(r.Body).Decode(&b) + seen.add(b.Model) + w.Header().Set("Content-Type", "application/json") + if strings.HasSuffix(r.URL.Path, "count_tokens") { + io.WriteString(w, `{"input_tokens":7}`) + return + } + io.WriteString(w, `{"id":"x","model":`+strconv.Quote(b.Model)+ + `,"content":[{"type":"text","text":"hi"}],`+ + `"usage":{"input_tokens":5,"output_tokens":3}}`) + })) + t.Cleanup(up.Close) + // an Anthropic base, so /v1/messages is the provider's own + // protocol and the request is relayed rather than rebuilt + if err := provider.Save(provider.Provider{ + ID: "relay", Name: "Relay", Key: "k", + Models: []string{"sol"}, + Anthropic: up.URL, + }); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("relay/sol", "vendor-c/sol-2"); err != nil { + t.Fatal(err) + } + + s := New() + rec := httptest.NewRecorder() + req := httptest.NewRequest("POST", tc.path, strings.NewReader( + `{"model":"relay/sol","max_tokens":64,`+ + `"messages":[{"role":"user","content":"hi"}]}`)) + s.Handler().ServeHTTP(rec, req) + if rec.Code != 200 { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + if got := seen.all(); len(got) != 1 || got[0] != "vendor-c/sol-2" { + t.Fatalf("the vendor was asked for %v; want the upstream name", got) + } + // "vendor-c/sol-2" is the same model as "sol", and a name that + // is not a prefix or a dated version of it is exactly what a + // comparison with magpie's own name would get wrong + if tc.path == "/v1/messages" { + if r := lastRoute(s); r.Served != "vendor-c/sol-2" || r.Swapped { + t.Errorf("the model answered under the relay's own id is marked swapped: %+v", r) + } + } + }) + } +} + +// The lookup is the model's own name, then the provider's "*", then the +// canonical id — and --reset removes only the first, leaving a provider-wide +// name in force. The CLI says so; this is where it is true. +func TestUpstreamNamePrecedence(t *testing.T) { + fresh(t) + if err := provider.Save(provider.Provider{ID: "relay", Name: "Relay", Key: "k", + Chat: "https://relay.example/v1", Models: []string{"sol", "other"}}); err != nil { + t.Fatal(err) + } + p, err := provider.Find("relay") + if err != nil { + t.Fatal(err) + } + + // with nothing set, the vendor is asked for the model magpie knows + if got := provider.UpstreamName(*p, "sol"); got != "sol" { + t.Errorf("unset: %q; want the canonical id", got) + } + + // a name given for every model of the provider covers a model that has + // none of its own, with each model in it + if err := provider.SetUpstreamName("relay/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + for _, m := range []string{"sol", "other"} { + if got := provider.UpstreamName(*p, m); got != "vendor-c/"+m { + t.Errorf("%s under the provider's *: %q; want vendor-c/%s", m, got, m) + } + } + + // a name with no model in it is every model's own name under that one + if err := provider.SetUpstreamName("relay/*", "vendor-c/one"); err != nil { + t.Fatal(err) + } + for _, m := range []string{"sol", "other"} { + if got := provider.UpstreamName(*p, m); got != "vendor-c/one" { + t.Errorf("%s under a name with no *: %q; want vendor-c/one", m, got) + } + } + + // the model's own wins + if err := provider.SetUpstreamName("relay/sol", "vendor-c/sol-exact"); err != nil { + t.Fatal(err) + } + if got := provider.UpstreamName(*p, "sol"); got != "vendor-c/sol-exact" { + t.Errorf("exact: %q; want vendor-c/sol-exact", got) + } + if got := provider.UpstreamName(*p, "other"); got != "vendor-c/one" { + t.Errorf("other: %q; want the provider's vendor-c/one", got) + } + + // taking the model's away leaves the provider's in force + if err := provider.SetUpstreamName("relay/sol", ""); err != nil { + t.Fatal(err) + } + if got := provider.UpstreamName(*p, "sol"); got != "vendor-c/one" { + t.Errorf("after --reset: %q; want the provider's vendor-c/one still in force", got) + } + if err := provider.SetUpstreamName("relay/*", ""); err != nil { + t.Fatal(err) + } + if got := provider.UpstreamName(*p, "sol"); got != "sol" { + t.Errorf("after both are reset: %q; want the canonical id", got) + } + // what was given is kept as it was given, the "*" and all, for a list of + // them to show + if err := provider.SetUpstreamName("relay/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + if got := p.UpstreamNames()["*"]; got != "vendor-c/*" { + t.Errorf("the name given is %q; want it as it was given", got) + } +} + +// A wire name keyed by something no lookup will match is refused where it is +// given rather than saved as a silent no-op, and one that is only whitespace +// is the user taking their name away, not a stored empty one. +func TestUpstreamNameRejectsBadKeys(t *testing.T) { + fresh(t) + if err := provider.Save(provider.Provider{ID: "relay", Name: "Relay", Key: "k", + Chat: "https://relay.example/v1", Models: []string{"sol"}}); err != nil { + t.Fatal(err) + } + for _, key := range []string{"noslash", "/sol", "Not A Provider/sol", "relay/"} { + if err := provider.SetUpstreamName(key, "vendor-c/sol"); err == nil { + t.Errorf("the key %q was accepted", key) + } + } + // a key hand-edited into the file is that entry's own no-op, and must + // not keep every other setting from being saved + s := settings.Load() + if s.ModelWires == nil { + s.ModelWires = map[string]string{} + } + s.ModelWires["noslash"] = "vendor-c/sol" + if err := settings.Save(s); err != nil { + t.Fatalf("a wire name keyed badly by hand stopped the settings being saved: %v", err) + } + if err := provider.SetUpstreamName("relay/sol", " "); err != nil { + t.Errorf("a whitespace-only name should remove the override, not fail: %v", err) + } +} + +// A provider given another id takes the names the user gave its models with +// it: a model asked for by the relay's own id would be asked for by magpie's +// again, and the relay would be asked for a model it does not have. +func TestUpstreamNamesFollowTheProviderRename(t *testing.T) { + fresh(t) + if err := provider.Save(provider.Provider{ID: "relay-b", Name: "Relay B", Key: "k", + Chat: "https://relay.example/v1", Models: []string{"model-2", "model-3"}}); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("relay-b/model-2", "vendor-c/model-2-preview"); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("relay-b/*", "vendor-c/sol-*"); err != nil { + t.Fatal(err) + } + if err := provider.Rename("relay-b", "relay-c"); err != nil { + t.Fatal(err) + } + p, err := provider.Find("relay-c") + if err != nil { + t.Fatal(err) + } + if got := provider.UpstreamName(*p, "model-2"); got != "vendor-c/model-2-preview" { + t.Errorf("the model's own name is %q after the rename", got) + } + if got := provider.UpstreamName(*p, "model-3"); got != "vendor-c/sol-model-3" { + t.Errorf("the provider's name is %q after the rename; it follows the id", got) + } +} + +// A relay that only knows its own ids would fail a test naming the model +// magpie knows it by, though every real turn reaches it: the provider's own +// tests ask for the model the way a request does. Each of the two probes is +// held to that on its own, on what it asked the relay for while it was the +// only one running — a provider test asking for magpie's name and a model +// test asking for the relay's, read together, both look right. +func TestProviderTestsAskTheWireName(t *testing.T) { + fresh(t) + var seen asked + relay := asking(t, &seen) + if err := provider.Save(provider.Provider{ID: "relay-b", Name: "Relay B", Key: "k", + Models: []string{"model-2", "model-3"}, Chat: relay.URL}); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("relay-b/*", "vendor-c/sol-*"); err != nil { + t.Fatal(err) + } + p, err := provider.Find("relay-b") + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + // the two names the relay serves these models by + wire := []string{"vendor-c/sol-model-2", "vendor-c/sol-model-3"} + + results := p.Test(ctx) + if len(results) == 0 { + t.Fatal("the provider was not tested at all") + } + for _, r := range results { + if !r.OK { + t.Errorf("the provider's own test failed: %+v", r) + } + } + seen.wire(t, "the provider's own test", wire) + + for _, r := range p.TestModels(ctx, []string{"model-2", "model-3"}) { + if !r.OK { + t.Errorf("testing %q failed: %+v", r.Model, r) + } + } + // each model by its own wire name, and both of them asked for + if got := seen.wire(t, "testing the models", wire); len(got) != len(wire) { + t.Errorf("testing the models asked the relay for %v; want one request per model", got) + } + // the model reported is the one magpie knows, not the relay's own id + for _, r := range p.TestModels(ctx, []string{"model-2"}) { + if r.Model != "model-2" { + t.Errorf("a test reports the model as %q; the canonical id is what the user picked", r.Model) + } + } +} + +// A drawing is the one request that keeps the name magpie knows the model by: +// the gateway builds those bodies itself, on the images API or on chat with +// modalities, and no upstream name is written into one. A provider test that +// asked for the vendor's own name would be testing a request no turn ever +// makes, and would report a working drawing model as broken. +func TestProviderDrawingTestsAskTheCanonicalName(t *testing.T) { + fresh(t) + var seen asked + relay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var b struct { + Model string `json:"model"` + } + json.NewDecoder(r.Body).Decode(&b) + seen.add(b.Model) + if strings.HasSuffix(r.URL.Path, "/images/generations") { + // not served here, so the chat fallback is taken as the + // gateway takes it + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("Content-Type", "text/event-stream") + io.WriteString(w, `data: {"id":"x","model":`+strconv.Quote(b.Model)+ + `,"choices":[{"index":0,"delta":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],`+ + `"usage":{"prompt_tokens":5,"completion_tokens":3}}`+"\n\n") + io.WriteString(w, "data: [DONE]\n\n") + })) + t.Cleanup(relay.Close) + if err := provider.Save(provider.Provider{ID: "relay-b", Name: "Relay B", Key: "k", + Models: []string{"imagine-2"}, Chat: relay.URL}); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("relay-b/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + p, err := provider.Find("relay-b") + if err != nil { + t.Fatal(err) + } + for _, r := range p.TestModels(context.Background(), []string{"imagine-2"}) { + if !r.OK { + t.Fatalf("testing the drawing model failed: %+v", r) + } + } + // the images request and the chat one it fell back to, neither carrying + // the upstream name + if got := seen.all(); len(got) != 2 || got[0] != "imagine-2" || got[1] != "imagine-2" { + t.Fatalf("the relay was asked for %v; a drawing goes out under the name magpie knows the model by", got) + } +} + +// A name that is only whitespace is the user taking their name away, not one +// to be kept: it is a legal removal, so it must not fail, and it must leave +// nothing behind — no key in the settings, and the model asked for by the +// name magpie knows it by. A wire name that is a space in the file would be +// sent to the vendor as the model, and a list of them would show it as one +// the user gave. +func TestUpstreamNameBlankIsAReset(t *testing.T) { + fresh(t) + if err := provider.Save(provider.Provider{ID: "relay", Name: "Relay", Key: "k", + Chat: "https://relay.example/v1", Models: []string{"sol", "other"}}); err != nil { + t.Fatal(err) + } + p, err := provider.Find("relay") + if err != nil { + t.Fatal(err) + } + + if err := provider.SetUpstreamName("relay/sol", "vendor-c/sol"); err != nil { + t.Fatal(err) + } + if got := provider.UpstreamName(*p, "sol"); got != "vendor-c/sol" { + t.Fatalf("before the reset: %q; want vendor-c/sol", got) + } + + // blank takes a name that was given away, and is not an error + if err := provider.SetUpstreamName("relay/sol", " "); err != nil { + t.Fatalf("a whitespace-only name is the user taking theirs away, not a bad input: %v", err) + } + if n, ok := settings.Load().ModelWires["relay/sol"]; ok { + t.Errorf("the settings kept a wire name of %q for relay/sol", n) + } + if got := provider.UpstreamName(*p, "sol"); got != "sol" { + t.Errorf("after a blank name: %q; want the canonical id", got) + } + if _, ok := p.UpstreamNames()["sol"]; ok { + t.Error("a name taken away is still listed as one the user gave") + } + + // and blank for a key there is nothing under writes no key at all + if err := provider.SetUpstreamName("relay/other", "\t\n "); err != nil { + t.Fatalf("a whitespace-only name is the user taking theirs away, not a bad input: %v", err) + } + if n, ok := settings.Load().ModelWires["relay/other"]; ok { + t.Errorf("a blank name wrote %q for relay/other", n) + } + if got := provider.UpstreamName(*p, "other"); got != "other" { + t.Errorf("after a blank name: %q; want the canonical id", got) + } + if got := p.UpstreamNames(); len(got) != 0 { + t.Errorf("the names given are %v; a blank one is none", got) + } + + // a provider-wide name in force is what a model's own blank name falls + // back to, not a blank name of its own + if err := provider.SetUpstreamName("relay/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("relay/sol", " "); err != nil { + t.Fatalf("a whitespace-only name is the user taking theirs away, not a bad input: %v", err) + } + if got := provider.UpstreamName(*p, "sol"); got != "vendor-c/sol" { + t.Errorf("with the provider's name in force: %q; want vendor-c/sol", got) + } +} + +// A name left blank in the file — by an older magpie, or by hand — is no +// name: the model is asked for by the name magpie knows it by, and a list of +// them does not show a blank as one the user gave. It is the same thing a +// blank name is where it is set, and the file is no more exempt. +func TestUpstreamNameBlankInTheFileIsNoName(t *testing.T) { + fresh(t) + if err := provider.Save(provider.Provider{ID: "relay", Name: "Relay", Key: "k", + Chat: "https://relay.example/v1", Models: []string{"sol", "other"}}); err != nil { + t.Fatal(err) + } + p, err := provider.Find("relay") + if err != nil { + t.Fatal(err) + } + + s := settings.Load() + s.ModelWires = map[string]string{ + "relay/sol": " ", + "relay/other": "vendor-c/other", + } + if err := settings.Save(s); err != nil { + t.Fatal(err) + } + + if got := provider.UpstreamName(*p, "sol"); got != "sol" { + t.Errorf("a blank name in the file: %q; want the canonical id", got) + } + if got := provider.UpstreamName(*p, "other"); got != "vendor-c/other" { + t.Errorf("a name beside a blank one: %q; want vendor-c/other", got) + } + names := p.UpstreamNames() + if _, ok := names["sol"]; ok { + t.Errorf("a blank name in the file is listed as one given: %v", names) + } + if names["other"] != "vendor-c/other" { + t.Errorf("the names given are %v; a blank one must not hide the rest", names) + } + + // the same for the provider-wide key, which covers the models a blank + // model's own name does not reach + s = settings.Load() + s.ModelWires["relay/*"] = " " + if err := settings.Save(s); err != nil { + t.Fatal(err) + } + if got := provider.UpstreamName(*p, "sol"); got != "sol" { + t.Errorf("a blank name in the file: %q; want the canonical id", got) + } + if got := p.UpstreamNames(); len(got) != 1 || got["other"] != "vendor-c/other" { + t.Errorf("the names given are %v; a blank one is none", got) + } +} + +// Every "*" in a wire name is the model, wherever it stands and however many +// there are — the rule the settings' own comment, the CLI help and the README +// all give, and what the lookup does. A name with none is that one name for +// every model of the key, which is what a query lists back as it was given, +// "*" and all, rather than a name with one filled in. +func TestUpstreamNameWildcardStandsWhereverItStands(t *testing.T) { + fresh(t) + if err := provider.Save(provider.Provider{ID: "relay", Name: "Relay", Key: "k", + Chat: "https://relay.example/v1", Models: []string{"sol", "other"}}); err != nil { + t.Fatal(err) + } + p, err := provider.Find("relay") + if err != nil { + t.Fatal(err) + } + + for _, tc := range []struct { + wires map[string]string + model string + want string + why string + }{ + {map[string]string{"relay/sol": "vendor-c/*"}, "sol", "vendor-c/sol", "a trailing *"}, + {map[string]string{"relay/sol": "vendor/*/pro"}, "sol", "vendor/sol/pro", "a * in the middle"}, + {map[string]string{"relay/sol": "*-pro"}, "sol", "sol-pro", "a leading *"}, + {map[string]string{"relay/sol": "a/*-b/*"}, "sol", "a/sol-b/sol", "two *"}, + {map[string]string{"relay/sol": "*"}, "sol", "sol", "nothing but a *"}, + {map[string]string{"relay/*": "vendor/*/pro"}, "sol", "vendor/sol/pro", "the provider's, mid-name"}, + {map[string]string{"relay/*": "vendor-c/one"}, "sol", "vendor-c/one", "no * at all"}, + {map[string]string{"relay/*": "vendor-c/one"}, "other", "vendor-c/one", "no *, other model"}, + {map[string]string{"relay/sol": "vendor-c/sol"}, "other", "other", "another provider's model"}, + } { + if got := provider.UpstreamNameIn(tc.wires, "relay", tc.model); got != tc.want { + t.Errorf("%s: %q; want %q", tc.why, got, tc.want) + } + } + + // what a query shows is what was given, "*" and all, so a name given for + // every model can be read as what it sends rather than as one model's + if err := provider.SetUpstreamName("relay/*", "vendor/*/pro"); err != nil { + t.Fatal(err) + } + if got := p.UpstreamNames()["*"]; got != "vendor/*/pro" { + t.Errorf("the name given for every model is listed as %q; want it as it was given", got) + } + for _, m := range []string{"sol", "other"} { + if got := provider.UpstreamName(*p, m); got != "vendor/"+m+"/pro" { + t.Errorf("%s under the provider's name: %q; want vendor/%s/pro", m, got, m) + } + } + + // a name with no "*" in it is that one name for every model of the key, + // and is listed as it was given too + if err := provider.SetUpstreamName("relay/*", "vendor-c/one"); err != nil { + t.Fatal(err) + } + if got := p.UpstreamNames()["*"]; got != "vendor-c/one" { + t.Errorf("the name given for every model is listed as %q; want vendor-c/one", got) + } + for _, m := range []string{"sol", "other"} { + if got := provider.UpstreamName(*p, m); got != "vendor-c/one" { + t.Errorf("%s under a name with no *: %q; want vendor-c/one", m, got) + } + } + + // and a single model's own name, wildcard and all, beside the + // provider's + if err := provider.SetUpstreamName("relay/sol", "vendor-c/sol-*"); err != nil { + t.Fatal(err) + } + names := p.UpstreamNames() + if names["sol"] != "vendor-c/sol-*" || names["*"] != "vendor-c/one" { + t.Errorf("the names given are %v; each is listed as it was given", names) + } + if got := provider.UpstreamName(*p, "sol"); got != "vendor-c/sol-sol" { + t.Errorf("the model's own: %q; want vendor-c/sol-sol", got) + } + if got := provider.UpstreamName(*p, "other"); got != "vendor-c/one" { + t.Errorf("the other model: %q; want the provider's vendor-c/one", got) + } +} + +// A name given for a provider's every model is not one id for all of them: an +// Antigravity account is asked for the variant its effort picks, and the name +// goes with that variant. Filling the "*" in with the model magpie knows +// would send one id whatever level is asked for, so the effort stops choosing +// and a relay that serves gemini-3.7-flash-high is asked for the base id it +// has no answer for. +func TestUpstreamNameFollowsTheAntigravityVariant(t *testing.T) { + fresh(t) + var raw []catalog.Model + for _, l := range []string{"gemini-3.7-flash-high - Gemini 3.7 Flash (High)", "gemini-3.7-flash-low - Gemini 3.7 Flash (Low)", + "gemini-3.7-flash-medium - Gemini 3.7 Flash (Medium)"} { + id, name, _ := strings.Cut(l, " - ") + raw = append(raw, catalog.Model{ID: id, Name: name}) + } + if err := catalog.SaveLive("antigravity", "", raw); err != nil { + t.Fatal(err) + } + // a wire name is given for a model the provider really serves, so + // magpie must be able to find the provider that serves it. The account + // is built at runtime and its id is reserved (Save refuses it), so the + // test registers the provider under that id in the file directly: the + // model a name is keyed by is the family the live list stands for. + putProvider(t, map[string]any{ + "id": "antigravity", "name": "Antigravity", "key": "k", + "chat": "https://antigravity.test/v1", + }) + if err := provider.SetUpstreamName("antigravity/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + ask := func(agent, model, effort string) string { + t.Helper() + p := provider.Provider{ID: "antigravity", Name: "Antigravity", Account: &provider.Account{Agent: agent}} + r := codeAssistRequest(t) + r.Thinking, r.Effort = false, effort + return modelOf(codeAssistBody(p, r, model, settings.Load().ModelWires)) + } + + // the level the effort picked is in the name it is asked for by + for _, c := range []struct{ effort, want string }{ + {"high", "vendor-c/gemini-3.7-flash-high"}, + {"medium", "vendor-c/gemini-3.7-flash-medium"}, + {"low", "vendor-c/gemini-3.7-flash-low"}, + {"", "vendor-c/gemini-3.7-flash-high"}, + } { + if got := ask("antigravity", "gemini-3.7-flash", c.effort); got != c.want { + t.Errorf("at %q the vendor is asked for %q; want %q — each level by its own name", c.effort, got, c.want) + } + } + + // a name given for the model itself is that model's, whichever variant + // answers for it: the exact key still wins over the provider's + if err := provider.SetUpstreamName("antigravity/gemini-3.7-flash", "vendor-c/flash"); err != nil { + t.Fatal(err) + } + for _, effort := range []string{"high", "low"} { + if got := ask("antigravity", "gemini-3.7-flash", effort); got != "vendor-c/flash" { + t.Errorf("the model's own name at %q: %q; want vendor-c/flash", effort, got) + } + } + + // a Gemini CLI account has no variants, so the model magpie knows is + // what is asked for, under the provider's name + if err := provider.SetUpstreamName("antigravity/gemini-3.7-flash", ""); err != nil { + t.Fatal(err) + } + if got := ask("gemini", "gemini-3.7-flash", "high"); got != "vendor-c/gemini-3.7-flash" { + t.Errorf("a Gemini CLI account: %q; want vendor-c/gemini-3.7-flash", got) + } + + // and with no name given, the envelope goes out as it was built + if err := provider.SetUpstreamName("antigravity/*", ""); err != nil { + t.Fatal(err) + } + if got := ask("antigravity", "gemini-3.7-flash", "low"); got != "gemini-3.7-flash-low" { + t.Errorf("with nothing given: %q; want the variant as it is", got) + } +} + +// A turn's route and its ledger row are two views of the one call, so they +// have to judge it against one name: on an Antigravity account the call +// goes out under the variant of the model the effort picks, so a relay that +// answers with that very id has not swapped anything — and one that answers +// with another level's id has, whichever of the two the reader is looking +// at. The route read the model magpie knows where the ledger read the id +// that went out, and told the user two different things about the same +// reply. +// +// The provider here is one of the user's own under an id of its own, which is +// the shape the two views agree on by rule and not by coincidence: a relay's +// id names no account, so neither view reaches for a variant, and both read +// the reply back by the one name that went out. An Antigravity account's +// provider is not a shape a test can build here — it speaks Code Assist, +// which magpie cannot point at a relay — and the rule keyed by the account is +// read in internal/provider, where the account is what a caller hands in. +func TestUpstreamNameReadTheSameByTheRouteAndTheLedger(t *testing.T) { + fresh(t) + var raw []catalog.Model + for _, l := range []string{"gemini-3.7-flash-high - High", "gemini-3.7-flash-low - Low", "gemini-3.7-flash-medium - Medium"} { + id, name, _ := strings.Cut(l, " - ") + raw = append(raw, catalog.Model{ID: id, Name: name}) + } + if err := catalog.SaveLive("antigravity", "", raw); err != nil { + t.Fatal(err) + } + // a relay that answers with an id of its own choosing, the way a vendor + // serves a model under a name of its own and says which answered + answers := "" + relay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var b struct { + Model string `json:"model"` + } + json.NewDecoder(r.Body).Decode(&b) + w.Header().Set("Content-Type", "text/event-stream") + io.WriteString(w, `data: {"id":"x","model":`+strconv.Quote(answers)+ + `,"choices":[{"index":0,"delta":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],`+ + `"usage":{"prompt_tokens":5,"completion_tokens":3}}`+"\n\n") + io.WriteString(w, "data: [DONE]\n\n") + })) + t.Cleanup(relay.Close) + putProvider(t, map[string]any{ + "id": "ag-relay", "name": "AG Relay", "key": "k", "chat": relay.URL, + "models": []string{"gemini-3.7-flash"}, + }) + if err := provider.SetUpstreamName("ag-relay/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + s := New() + turn := func() (Route, usage.Row) { + t.Helper() + rec := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/v1/messages", strings.NewReader( + `{"model":"ag-relay/gemini-3.7-flash","max_tokens":1024,"stream":true,`+ + `"thinking":{"type":"enabled","budget_tokens":24000},`+ + `"messages":[{"role":"user","content":"hi"}]}`)) + s.Handler().ServeHTTP(rec, req) + if rec.Code != 200 { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + rows, _, _ := usage.Ledger(usage.All, usage.Filter{Query: "gemini-3.7-flash"}) + if len(rows) == 0 { + t.Fatal("the ledger holds no row for the turn") + } + return lastRoute(s), rows[0] // newest first: the turn just made + } + + // the id the call went out under is the one the wire gave it, so a relay + // answering with that very id has swapped nothing — in the route and in + // the ledger alike + answers = "vendor-c/gemini-3.7-flash" + route, row := turn() + if route.Swapped || row.Swapped { + t.Errorf("the relay answered with the id the call went out under, which is no swap: the route says %v, the ledger %v", route.Swapped, row.Swapped) + } + + // another model's id is another model, and both views say so + answers = "vendor-c/gemini-3.7-flash-high" + route, row = turn() + if !route.Swapped || !row.Swapped { + t.Errorf("the relay answered with another level's id: route swapped %v, ledger %v", route.Swapped, row.Swapped) + } +} + +// A route reads a turn's reply against the name the call really went out +// under, and on an Antigravity account that name is the variant of the model +// the effort picked — so the account is the question, not the provider's id. +// The two are one id today because an account provider is built with its +// account's own id, and a test that builds a provider of an account's own id +// with no account behind it says nothing about which of the two the rule is +// keyed by: it is the shape where they come apart, and the relay here is +// asked for the model magpie knows, so no level of the family is what the +// route has to read the reply back by. +// +// Only the route is asserted: a record keeps the provider id and no account, +// and reads itself back by that id (SentNameIn), which is the whole of what +// the ledger has to work from. +func TestRouteJudgesTheUpstreamNameByTheAccountAndNotByTheProvidersId(t *testing.T) { + fresh(t) + var raw []catalog.Model + for _, l := range []string{"gemini-3.7-flash-high - High", "gemini-3.7-flash-low - Low", "gemini-3.7-flash-medium - Medium"} { + id, name, _ := strings.Cut(l, " - ") + raw = append(raw, catalog.Model{ID: id, Name: name}) + } + if err := catalog.SaveLive("antigravity", "", raw); err != nil { + t.Fatal(err) + } + answers := "" + relay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + io.WriteString(w, `data: {"id":"x","model":`+strconv.Quote(answers)+ + `,"choices":[{"index":0,"delta":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],`+ + `"usage":{"prompt_tokens":5,"completion_tokens":3}}`+"\n\n") + io.WriteString(w, "data: [DONE]\n\n") + })) + t.Cleanup(relay.Close) + putProvider(t, map[string]any{ + "id": "antigravity", "name": "Antigravity", "key": "k", "chat": relay.URL, + "models": []string{"gemini-3.7-flash"}, + }) + if err := provider.SetUpstreamName("antigravity/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + s := New() + turn := func() Route { + t.Helper() + rec := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/v1/messages", strings.NewReader( + `{"model":"antigravity/gemini-3.7-flash","max_tokens":1024,"stream":true,`+ + `"thinking":{"type":"enabled","budget_tokens":24000},`+ + `"messages":[{"role":"user","content":"hi"}]}`)) + s.Handler().ServeHTTP(rec, req) + if rec.Code != 200 { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + return lastRoute(s) + } + + // the relay was asked for the model under its provider's own name, and + // answered with that id: nothing swapped + answers = "vendor-c/gemini-3.7-flash" + if r := turn(); r.Swapped { + t.Errorf("the relay answered with the id it was asked for, which is no swap: the route says %v", r.Swapped) + } + + // a level of the family is another model, and the route reads it as one + answers = "vendor-c/gemini-3.7-flash-high" + if r := turn(); !r.Swapped { + t.Errorf("the relay answered with a level's id: the route says swapped %v", r.Swapped) + } +} + +// The id an Antigravity envelope is named with is the id a ledger reads the +// vendor's reply back against, so it is one rule and not two: a model of the +// family and one of the family's own variant ids — an id a pick or an +// agent's model list kept from before the families were one model — each go +// out at, and are judged against, the same id at every level. +func TestAntigravityEnvelopeIsNamedAsTheLedgerJudgesIt(t *testing.T) { + fresh(t) + var raw []catalog.Model + for _, l := range []string{"gemini-3.7-flash-high - High", "gemini-3.7-flash-low - Low", "gemini-3.7-flash-medium - Medium"} { + id, name, _ := strings.Cut(l, " - ") + raw = append(raw, catalog.Model{ID: id, Name: name}) + } + if err := catalog.SaveLive("antigravity", "", raw); err != nil { + t.Fatal(err) + } + for _, model := range []string{"gemini-3.7-flash", "gemini-3.7-flash-high", "gemini-3.7-flash-low"} { + for _, effort := range []string{"high", "medium", "low", "minimal", "xhigh", ""} { + sent := codeAssistID(model, effort, "antigravity") + judged := provider.SentNameIn(nil, "antigravity", model, effort) + if sent != judged { + t.Errorf("%s at %q goes out as %q and is judged against %q", model, effort, sent, judged) + } + if usage.Swapped(sent, sent) { + t.Errorf("%s at %q: the id the call goes out under is read as another model answering", model, effort) + } + } + } + // and a Gemini CLI account is asked for the model magpie knows, which is + // what its ledger row reads back too + if got := codeAssistID("gemini-3.7-flash", "high", "gemini"); got != "gemini-3.7-flash" { + t.Errorf("a Gemini CLI account is asked for %q; want the model magpie knows", got) + } +} + +// A turn reads the upstream names once, wherever in it they are looked up: +// the id a try goes out under, the body the vendor is sent, the token count +// asked of it. settings.Load reads and parses the whole file every call, so +// a name looked up per place made every turn pay for the settings once per +// place — a turn that named no model at all paying just the same. Reading +// them once also keeps a turn to the names that stood when it came in, so a +// name given while it is on its way is for the turns after it and not for +// the one in the air. +func TestUpstreamNamesAreReadOnceInATurn(t *testing.T) { + fresh(t) + var seen []string + named := false + // a relay that is given a name for the model it serves while a turn is + // on its way, and answers with the model it was asked for, as a relay + // serving a model under an id of its own does + relay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var b struct { + Model string `json:"model"` + } + json.NewDecoder(r.Body).Decode(&b) + seen = append(seen, b.Model) + if !named { + named = true + st := settings.Load() + st.ModelWires = map[string]string{"relay/model-2": "vendor-c/model-2-renamed"} + if err := settings.Save(st); err != nil { + t.Error(err) + } + } + w.Header().Set("Content-Type", "text/event-stream") + io.WriteString(w, `data: {"id":"x","model":`+strconv.Quote(b.Model)+ + `,"choices":[{"index":0,"delta":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],`+ + `"usage":{"prompt_tokens":5,"completion_tokens":3}}`+"\n\n") + io.WriteString(w, "data: [DONE]\n\n") + })) + t.Cleanup(relay.Close) + if err := provider.Save(provider.Provider{ID: "relay", Name: "Relay", Key: "k", + Models: []string{"model-2"}, Anthropic: relay.URL}); err != nil { + t.Fatal(err) + } + s := New() + turn := func() Route { + t.Helper() + rec := httptest.NewRecorder() + s.Handler().ServeHTTP(rec, httptest.NewRequest("POST", "/v1/messages", strings.NewReader( + `{"model":"relay/model-2","max_tokens":1024,"stream":true,`+ + `"messages":[{"role":"user","content":"hi"}]}`))) + if rec.Code != 200 { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + return lastRoute(s) + } + + // the first turn went out under the names that stood when it came in, and + // is judged against those and not the ones given since: read afresh at + // every place it looks, its own record of the reply would name the model + // the relay never answered for + route := turn() + if got := seen; len(got) != 1 || got[0] != "model-2" { + t.Fatalf("the relay was asked for %v; the first turn goes out under the names it came in with", got) + } + if route.Served != "model-2" || route.Swapped { + t.Errorf("the relay answered with the model the turn went out under, which is no swap: %+v", route) + } + // and the next turn is the one the name given mid-flight is for + route = turn() + if got := seen; len(got) != 2 || got[1] != "vendor-c/model-2-renamed" { + t.Fatalf("the relay was asked for %v; the second turn goes out under the name given", got) + } + if route.Served != "vendor-c/model-2-renamed" || route.Swapped { + t.Errorf("the relay answered with the name the turn went out under, which is no swap: %+v", route) + } +} diff --git a/internal/gui/settings_keep_test.go b/internal/gui/settings_keep_test.go index dc002b7a..a61c5ce8 100644 --- a/internal/gui/settings_keep_test.go +++ b/internal/gui/settings_keep_test.go @@ -74,6 +74,7 @@ func TestSettingsSaveKeepsWhatItDoesNotSend(t *testing.T) { ModelImages: map[string]bool{"p/m": true}, ModelOutputs: map[string]int{"p/m": 131072}, ModelPrices: map[string]settings.ModelPrice{"p/m": {Input: &one, Output: &two}}, + ModelWires: map[string]string{"p/m": "vendor-c/m"}, RedactRules: []redact.Rule{{Kind: "prefix", Prefix: "oc_sk_"}}, LAN: true, LANKey: "sk-lan", diff --git a/internal/provider/modelprefs.go b/internal/provider/modelprefs.go index 7b543e46..bcb357ec 100644 --- a/internal/provider/modelprefs.go +++ b/internal/provider/modelprefs.go @@ -452,3 +452,395 @@ func (p Provider) serves(model string) bool { has := func(m catalog.Model) bool { return m.ID == model } return slices.ContainsFunc(p.Available(), has) || slices.ContainsFunc(p.Exposed(), has) } + +// ServesModel is whether the provider really serves model, listed or +// exposed: the question SetUpstreamName asks before it keeps a name for +// one, and the same one, since a name given for a model the provider does +// not serve is a request magpie would never send it. It is the answer a +// caller told to take a name off needs as well: a model the provider has +// nothing like is not asked for by any name, so reporting which one is in +// force for it would say magpie asks for a model it does not know. +func ServesModel(p Provider, model string) bool { return p.serves(model) } + +// UpstreamName is the name to send a provider for one of its models: the one +// the user gave, for the model or for every model of that provider, else the +// name magpie knows the model by. Every "*" in the name the user gave is that +// model, wherever it stands: "vendor-c/*" for a provider's every model sends +// vendor-c/model-3 for model-3, so one name covers a relay that namespaces +// its models and still asks for each of them by its own, and "vendor/*/pro" +// sends vendor/model-3/pro. A name with no "*" in it is that one name for +// every model of the key, which is the right answer only for a relay that +// does serve them all alike. +// +// The two are deliberately different things. Everything a user or an agent +// sees — the catalog, the routing groups' membership, what a call is recorded +// and priced as — is in magpie's name for the model, so a rename upstream +// never renames a model here. Only the request that goes out carries the +// vendor's name, and what a reply says answered is compared with it, so a +// model asked for by the name the vendor serves it under is not read as +// another one. +func UpstreamName(p Provider, model string) string { + return UpstreamNameIn(settings.Load().ModelWires, p.ID, model) +} + +// UpstreamNameIn is UpstreamName over a settings' ModelWires already read, +// for a caller that walks a whole ledger of models in a row. A name that is +// only whitespace — one an older magpie stored, or one hand-edited into the +// file — is no name, as it is where one is given. +func UpstreamNameIn(wires map[string]string, pid, model string) string { + n := strings.TrimSpace(wires[pid+"/"+model]) + if n == "" { + n = strings.TrimSpace(wires[pid+"/*"]) + } + if n == "" { + return model + } + return strings.ReplaceAll(n, "*", model) +} + +// UpstreamNameAs is UpstreamName for a request that goes out asking for sent, +// which is not always the model magpie knows: an Antigravity account is asked +// for the variant of it the effort picks, and that is the id the body carries. +// The two levels of the lookup are kept apart, since they are about different +// things. A name given for the model itself is the model's, whichever variant +// answers for it, so a "*" in it is that model and not the variant: the vendor +// is asked for the model by the name its owner gave, as it is everywhere else +// in magpie, and not for a "*" it never heard of. A "*" in the provider's own +// name is whatever the request is going out under, so one name for a provider +// asks for gemini-3.7-flash-high and gemini-3.7-flash-low each by its own, +// rather than for the one id whatever level the effort picked. +func UpstreamNameAs(p Provider, model, sent string) string { + return UpstreamNameAsIn(settings.Load().ModelWires, p.ID, model, sent) +} + +// UpstreamNameAsIn is UpstreamNameAs over a settings' ModelWires already +// read. A name that is only whitespace is no name, as it is in +// UpstreamNameIn: the lookup falls through to the level below it, and a +// request with no name at all goes out under the id it was built with. +func UpstreamNameAsIn(wires map[string]string, pid, model, sent string) string { + if sent == model { + return UpstreamNameIn(wires, pid, model) + } + if n := strings.TrimSpace(wires[pid+"/"+model]); n != "" { + return strings.ReplaceAll(n, "*", model) + } + if n := strings.TrimSpace(wires[pid+"/*"]); n != "" { + return strings.ReplaceAll(n, "*", sent) + } + return sent +} + +// SentNameIn is the name a call on the provider's model goes out under at +// that effort: the name the vendor would be asked for now, which is the one +// magpie knows the model by unless the user gave one. It is UpstreamNameAs for +// the id that call asks for, which is not the model magpie knows everywhere: +// on an Antigravity account it is the variant of it that effort picks, so a +// name given for the provider asks for each level by its own id, and a ledger +// reading a record back reads the vendor's reply against the name that +// really went out — the same one the gateway sent — rather than against the +// family it is one level of. +// +// The effort is the one the gateway sent, already fitted to the model's own +// levels, which is what a record keeps of it, so a level of the model's own +// is one of its variants and goes out as that one. An effort that is no +// level of the family names the same id the gateway asked for all the same: +// the level of the family's own nearest (AntigravitySentID), which is where +// the envelope went out. +// +// It is SentNameOnIn for the caller that has nothing but the id — a record in +// the ledger, which keeps the id the call went out on and nothing else. A +// provider that is an account is built with that account's own id as its own +// (googleProvider and the rest of the account providers), so the id says +// which account it was, and a provider of the user's own is no account's +// whatever its id is. A record's provider need not even be signed in now, +// which is why nothing here goes looking for the account behind the id: the +// id a call went out on is all a record has and all it can be read by. +func SentNameIn(wires map[string]string, pid, model, effort string) string { + return SentNameOnIn(wires, pid, pid, model, effort) +} + +// SentNameOnIn is SentNameIn for a caller holding the provider itself, and so +// the account behind it: the call goes out on that account, and it is the +// account the id the effort picks is a question about. A provider's id is +// that account's own id today (googleProvider and the rest of the account +// providers build each one with it), so passing the id for the account reads +// the same — but the two are one id by construction rather than by rule, and +// a provider of the user's own has no account at all whatever its id is. So +// the route, which is holding the provider, asks about the account and lets +// the gateway's dispatch (codeAssistID, on the same agent) be the one rule. +func SentNameOnIn(wires map[string]string, pid, account, model, effort string) string { + return UpstreamNameAsIn(wires, pid, model, antigravitySentID(account, model, effort)) +} + +// antigravitySentID is the id a call on the model goes out under at that +// effort on an Antigravity account. Every other account is asked for the +// model magpie knows, and so is one of a model this account does not serve +// as a family of levels. +func antigravitySentID(account, model, effort string) string { + if account != "antigravity" { + return model + } + return AntigravitySentID(model, effort) +} + +// AntigravitySentID is the id a request for model goes out under at that +// effort on an Antigravity account: the variant of the family the effort +// picks, at the level of that family nearest the effort asked for; with no +// effort asked for, the family's own default. One of the family's own +// variant ids — an id a pick or an agent's model list kept from before the +// families were one model — goes as it is when no effort is asked for, and +// at the level the effort picks otherwise, since the effort the client +// asks for wins, as it does for the family. Any other id goes as it is. +// +// The gateway sends the envelope under this and the ledger judges what the +// vendor answered against it, so there is one of it rather than one each: +// a reply naming the id a call went out under is that model answering, and +// read against the family instead every level of it reads as another +// model having swapped in, in the routing view and in the ledger alike. +func AntigravitySentID(model, effort string) string { + if base, _, ok := AntigravityBase(model); ok { + if effort == "" { + return model + } + model = base + } + vs, ok := AntigravityVariants(model) + if !ok { + return model + } + if effort == "" { + return vs[""] + } + var levels []string + for _, l := range Levels { + if vs[l] != "" { + levels = append(levels, l) + } + } + if id := vs[nearestEffort(effort, levels)]; id != "" { + return id + } + return vs[""] +} + +// nearestEffort is the level of levels nearest want in Levels' order, a tie +// going up, or want itself where the levels are none or hold it: an effort a +// family does not take is asked at the level of its own nearest, the rule +// the gateway fits an effort to a model's levels by on the way out +// (fitEffort), kept here where a ledger reading a record back can reach it +// too. +func nearestEffort(want string, levels []string) string { + at := slices.Index(Levels, want) + if len(levels) == 0 || at < 0 || slices.Contains(levels, want) { + return want + } + best, dist := want, len(Levels) + for _, l := range levels { + i := slices.Index(Levels, l) + if i < 0 || l == "none" { + continue + } + d := i - at + if d < 0 { + d = -d + } + if d < dist || d == dist && i > at { + best, dist = l, d + } + } + return best +} + +// SetUpstreamName is the name to send a provider for one of its models; an +// empty name takes the user's away and sends the one magpie knows. A name is +// only given for a model the provider really serves: a relay lists the model +// under the id magpie knows it by and asks for it under the vendor's own, and +// one it does not list is a request magpie would never send, so naming it is +// refused here, as a price and a limit are for the same id. A "*" stands +// for every model of the provider rather than for one, and a removal is +// exempt, so an entry left for a model that has since gone can still be +// taken away. +// +// A provider magpie asks through an agent's own backend has no model field +// to put a name in — the request goes out carrying the id the agent knows — +// so a name given for one of its models would never be sent (asksOwnBackend). +// +// The ref is taken as the provider it names, by its id now, and the name is +// kept under that id: a ref is as likely to be spelled with an id the +// provider had before a rename (Was) or with the name it is shown by, and +// every lookup of a wire name goes by the id the provider has (UpstreamName). +// A name under any other spelling of the provider is one no lookup will ever +// match — a name the provider is never asked for, and a removal that takes +// away nothing. +// +// The key is checked here, where it is given, and not as the whole settings +// are written: a hand-edited key no lookup will ever match is a no-op of the +// user's own making, and must not keep every other setting from being saved. +// +// Kept in settings, and saved without telling the agents, like a price: which +// model an agent picks, and what it is called, have not changed — only the +// name the request goes out under. +// +// A removal goes to DropUpstreamName, which works off the key rather than +// off this provider, so that taking a name away is the same operation +// whether or not the provider it was given for is still there. +// +// A blank name is a removal, and whether there was a name there to take away +// is DropUpstreamName's to say, not this one's: a caller with something of +// its own to say about a name that was not there — `magpie model wire -- +// reset`, which tells a mistyped id from a model already asked for by the +// name magpie knows it by — asks DropUpstreamName for the removal itself, +// the way `magpie model price --reset` asks DropModelPrice. A caller that +// only wants the model left as it is has nothing to tell the user either +// way, which is why a blank name is an error and not a second answer. +func SetUpstreamName(ref, name string) error { + p, model, err := splitRef(ref) + if err != nil { + return err + } + key := p.ID + "/" + model + if err := settings.CheckModelKey("wire name", key); err != nil { + return err + } + if name = strings.TrimSpace(name); name == "" { + _, err := DropUpstreamName(key) + return err + } + if model != "*" && !p.serves(model) { + return fmt.Errorf("%s has no model %s (magpie provider %s lists them)", p.ID, model, p.ID) + } + if p.asksOwnBackend() { + return fmt.Errorf("%s is %s's own account: magpie asks it for the model by the id it knows, so there is no upstream name to give it", p.Name, p.Account.Agent) + } + s := settings.Load() + if s.ModelWires == nil { + s.ModelWires = map[string]string{} + } + s.ModelWires[key] = name + return settings.Save(s) +} + +// DropUpstreamName takes a name off under the key it is stored at, and says +// whether there was one to take. A name outlives the provider it was given +// for: that provider can be deleted, and the name is then in force for +// whichever provider takes that id next — its requests would go out renamed +// to a vendor that never heard of it, and nothing would say so — so a +// removal that resolved the provider first would leave the user no way to +// take away the one name doing that. A key holding no name is not an error: +// there is nothing there to take away, which is the state it is left in +// either way, and false says so to a caller that has something else to say. +// +// Which key a ref is taken at is wireKey's: the name comes off from where it +// is really kept, rather than from under whichever provider the ref names +// now. +func DropUpstreamName(key string) (bool, error) { + key = strings.TrimPrefix(strings.TrimSpace(key), "magpie/") + if strings.HasPrefix(key, GroupPrefix) { + return false, errors.New("that is a routing group, not a provider's model") + } + pid, model, ok := strings.Cut(key, "/") + if !ok || pid == "" || model == "" { + return false, fmt.Errorf("name a model as provider/model, not %q", key) + } + s := settings.Load() + key = wireKey(s.ModelWires, pid, model) + if err := settings.CheckModelKey("wire name", key); err != nil { + return false, err + } + if _, ok := s.ModelWires[key]; !ok { + return false, nil + } + delete(s.ModelWires, key) + return true, settings.Save(s) +} + +// HasUpstreamName reports whether a name is kept under exactly the key it is +// given, the provider's part spelled as it is. No other spelling of the +// provider is tried: a caller deciding whether to remove a name by this key +// has to know that this key is the one a name is kept under, which is the +// one question wireKey's order is about. +// +// A key spelled as an agent spells one — with magpie/ in front of it, and +// the spaces a ref is typed with — is that same key, as it is for a removal +// (DropUpstreamName): a name kept under it is there whichever way it is +// spelled, and a check that answered for one spelling only would say a name +// was not kept where it is. The one caller has had that prefix taken off +// already (splitModelRef), so nothing depends on it yet. +func HasUpstreamName(key string) bool { + key = strings.TrimPrefix(strings.TrimSpace(key), "magpie/") + _, ok := settings.Load().ModelWires[key] + return ok +} + +// wireKey is the key a name for "pid/model", as the user spelled it, is kept +// under and taken off by, and the three spellings are tried in one order: +// the ref as it is given, the id the provider has or had, the name it is +// shown by. The first is where a name really is once the provider it was +// given for is gone — nothing rewrites the key, so it is still there under +// an id no provider has now, and it is the one a removal has to reach. +// Resolving that id through the name another provider is shown by instead +// would take away that provider's own name instead, and leave the one that +// is in force for whoever takes the id next in the file. +// +// The id a provider has, or had before a rename (Was), is where a name is +// written and read (SetUpstreamName, UpstreamName), and the name a provider +// is shown by is the last of the three, for a ref spelled that way and with +// nothing kept under either of the other two. +func wireKey(wires map[string]string, pid, model string) string { + key := pid + "/" + model + if _, ok := wires[key]; ok { + return key + } + if p, ok := byIDOrWas(pid); ok { + return p.ID + "/" + model + } + if p, err := Find(pid); err == nil { + return p.ID + "/" + model + } + return key +} + +// UpstreamNames are the names the user gave the provider's models, by model +// id, with the one given for all of them under "*" — each as it will be sent, +// trimmed, "*" and all, which is what a list of them shows. A name that is +// only whitespace is none, and is not listed. +func (p Provider) UpstreamNames() map[string]string { + out := map[string]string{} + for k, n := range settings.Load().ModelWires { + if m, ok := strings.CutPrefix(k, p.ID+"/"); ok { + if n = strings.TrimSpace(n); n != "" { + out[m] = n + } + } + } + return out +} + +// asksOwnBackend reports whether p is a signed-in account magpie reaches +// through that agent's own backend rather than through a vendor API: the +// request goes out carrying the model id the agent already knows, with no +// model field for magpie to name differently, so a wire name given for one +// of its models would never be sent. It is the account dispatch in +// internal/gateway that decides which agents those are, so this and that +// have to keep saying the same thing — a name accepted for an account the +// gateway asks its own way is a name never sent — and +// TestAsksOwnBackendAgreesWithTheGatewayDispatch reads the dispatch itself +// rather than a list written out a second time to keep them so. +// +// Codex is deliberately not among them: it is served by the ChatGPT backend +// (codex_backend.go), which has no provider id to key a name by. +func (p Provider) asksOwnBackend() bool { + if p.Account == nil { + return false + } + switch p.Account.Agent { + case "claude", "cursor", "devin", "kiro", "zed", CommandCodePlanID: + return true + } + // Qoder is one backend however many sites it is signed in to, and the + // gateway's dispatch covers the whole family rather than the site the + // account was made on, so the family is asked of rather than one id of + // it: a name refused for the one subscription has to be refused for + // the other too, which is as much magpie's own there. + return slices.Contains(qoderAgents, p.Account.Agent) +} diff --git a/internal/provider/modelprefs_test.go b/internal/provider/modelprefs_test.go index 749069ae..04bb4552 100644 --- a/internal/provider/modelprefs_test.go +++ b/internal/provider/modelprefs_test.go @@ -1,11 +1,16 @@ package provider import ( + "go/ast" + "go/parser" + "go/token" + "go/types" "os" "path/filepath" "reflect" "runtime" "slices" + "strconv" "strings" "testing" @@ -311,3 +316,881 @@ func TestRenameMovesEveryPerModelMap(t *testing.T) { t.Errorf("Visible, which is by agent id, was moved to %v", got) } } + +// A wire name is given only for a model the provider really serves: a relay +// lists the model under the id magpie knows and asks for it under the +// vendor's own, but one the provider does not serve is a request magpie would +// never send, so naming it is refused where it is given, as a price and a +// limit are for the same id. "*" stands for every model of the provider +// rather than for one and is allowed; a removal is exempt, so an entry left +// for a model that has since gone can still be taken away. +func TestSetUpstreamNameRefusesAModelNotServed(t *testing.T) { + prefsHome(t) + // "a" serves "sol" (prefsHome) + if err := SetUpstreamName("a/nope", "vendor-c/sol"); err == nil { + t.Error("a name for a model the provider does not serve was accepted") + } + if _, ok := settings.Load().ModelWires["a/nope"]; ok { + t.Error("a refused name was stored anyway") + } + // the whole-provider name stands on no one model, so it is allowed + if err := SetUpstreamName("a/*", "vendor-c/sol"); err != nil { + t.Errorf("a name for every model of the provider was refused: %v", err) + } + // a model that has since gone off the provider's list can still have the + // name it was given taken away: the removal is what the user is after + if err := SetUpstreamName("a/sol", "vendor-c/sol-2"); err != nil { + t.Fatal(err) + } + if err := SetUpstreamName("a/sol", ""); err != nil { + t.Errorf("taking away the name of a model now gone was refused: %v", err) + } + if _, ok := settings.Load().ModelWires["a/sol"]; ok { + t.Error("the name of a model now gone was not taken away") + } +} + +// A provider magpie reaches through an agent's own backend is asked for the +// model by the id that agent already knows, with no model field for a wire +// name to change; a name given for one of its models would never be sent, and +// comparing the served model against it would read every call as swapped. It +// is refused where it is given instead, so neither the gateway nor the usage +// ledger can read one off it. A key the user gave a provider is not one of +// these and still takes a name. +func TestSetUpstreamNameRefusesAnAccountBackend(t *testing.T) { + prefsHome(t) + // a Kiro account with a saved key is one magpie reaches through Kiro's + // own backend; the live list is the model it serves + if err := Save(Provider{ID: "kiro", Name: "Kiro", Key: "k"}); err != nil { + t.Fatal(err) + } + if err := catalog.SaveLive("kiro", "", []catalog.Model{{ID: "auto", Name: "Auto"}}); err != nil { + t.Fatal(err) + } + p, err := Find("kiro") + if err != nil { + t.Fatal(err) + } + if p.Account == nil || p.Account.Agent != "kiro" { + t.Fatalf("the provider is not the account the test set up: %+v", p.Account) + } + if !p.asksOwnBackend() { + t.Fatal("a signed-in account backend is not recognised as one") + } + if err := SetUpstreamName("kiro/auto", "vendor-c/auto"); err == nil { + t.Error("a name for an account backend's model was accepted") + } + if _, ok := settings.Load().ModelWires["kiro/auto"]; ok { + t.Error("a refused account-backend name was stored anyway") + } + // Codex is not an account backend for this purpose: it is served by the + // ChatGPT backend, which has no provider id to key a name by + if (Provider{Account: &Account{Agent: "codex"}}).asksOwnBackend() { + t.Error("the Codex subscription was taken for an account backend") + } + // a key the user gave a provider is not an account backend and takes one + if err := SetUpstreamName("a/sol", "vendor-c/sol"); err != nil { + t.Errorf("a name for a keyed provider's model was refused: %v", err) + } +} + +// A wire name is the name of a model of a provider, not of a spelling of +// that provider's id: it is kept, looked up and taken away under the id the +// provider has now, so a ref spelled with the id it had before a rename — +// which an agent still running on an older config sends — names that same +// model, and leaves no second key behind that no lookup will ever match. +func TestSetUpstreamNameFollowsARenamedProvider(t *testing.T) { + prefsHome(t) + if err := SetUpstreamName("a/sol", "vendor-c/sol"); err != nil { + t.Fatal(err) + } + if err := Rename("a", "c"); err != nil { + t.Fatal(err) + } + p, err := Find("c") + if err != nil { + t.Fatal(err) + } + // the rename took the name with it + if got := settings.Load().ModelWires["c/sol"]; got != "vendor-c/sol" { + t.Fatalf("the settings hold %q", got) + } + // giving one by the id the provider had is giving one for that model, + // not a name the provider is never asked for + if err := SetUpstreamName("a/sol", "vendor-c/sol-2"); err != nil { + t.Fatal(err) + } + if got := settings.Load().ModelWires; len(got) != 1 || got["c/sol"] != "vendor-c/sol-2" { + t.Fatalf("the settings hold %v", got) + } + if got := UpstreamName(*p, "sol"); got != "vendor-c/sol-2" { + t.Fatalf("the provider is asked for %q", got) + } + // and taking it away by that id takes away the one in force + if err := SetUpstreamName("a/sol", ""); err != nil { + t.Fatal(err) + } + if got := settings.Load().ModelWires; len(got) != 0 { + t.Fatalf("the settings still hold %v", got) + } + if got := UpstreamName(*p, "sol"); got != "sol" { + t.Fatalf("the provider is asked for %q", got) + } +} + +// The same for the one name given for every model of a provider: it is kept +// under the id the provider has now, and a removal by the id it had takes +// it away rather than leaving it in force under a key nothing matches. +func TestSetUpstreamNameWildcardFollowsARenamedProvider(t *testing.T) { + prefsHome(t) + if err := SetUpstreamName("a/*", "vendor-c/sol"); err != nil { + t.Fatal(err) + } + if err := Rename("a", "c"); err != nil { + t.Fatal(err) + } + p, err := Find("c") + if err != nil { + t.Fatal(err) + } + if err := SetUpstreamName("a/*", "vendor-c/sol-2"); err != nil { + t.Fatal(err) + } + if got := settings.Load().ModelWires; len(got) != 1 || got["c/*"] != "vendor-c/sol-2" { + t.Fatalf("the settings hold %v", got) + } + if got := UpstreamName(*p, "sol"); got != "vendor-c/sol-2" { + t.Fatalf("the provider is asked for %q", got) + } + if err := SetUpstreamName("a/*", ""); err != nil { + t.Fatal(err) + } + if got := settings.Load().ModelWires; len(got) != 0 { + t.Fatalf("the settings still hold %v", got) + } + if got := UpstreamName(*p, "sol"); got != "sol" { + t.Fatalf("the provider is asked for %q", got) + } +} + +// The provider of a ref is the provider it names, however it is spelled, as +// it already is for the setters beside this one: a name a user reaches for +// a provider by is that provider, and the wire name is kept under the id +// it has. What the key check is for is a provider id that is not spelled +// the way a provider's is — a file magpie did not write, whose keys no +// lookup goes by — and not the name the provider is shown by. +func TestSetUpstreamNameTakesTheNameAProviderIsCalledBy(t *testing.T) { + prefsHome(t) + // the same list "a" serves sol from in prefsHome + if err := Save(Provider{ID: "relay", Name: "My Relay", Catalog: "a", Key: "k", + Chat: "http://127.0.0.1:1/v1", Models: []string{"sol"}}); err != nil { + t.Fatal(err) + } + const ref = "My Relay/sol" + if err := SetUpstreamName(ref, "vendor-c/sol"); err != nil { + t.Fatalf("a ref spelled with the name the provider is shown by was refused: %v", err) + } + // the setters beside this one take the very same ref + if err := SetModelName(ref, "My Sol"); err != nil { + t.Fatalf("%q names a model of that provider, as the others have it: %v", ref, err) + } + // and both names are kept under the id the provider has, which is what + // every lookup goes by + if got := settings.Load().ModelWires; len(got) != 1 || got["relay/sol"] != "vendor-c/sol" { + t.Fatalf("the settings hold %v", got) + } + if got := settings.Load().ModelNames["relay/sol"]; got != "My Sol" { + t.Fatalf("the settings hold the name %q", got) + } + p, err := Find("relay") + if err != nil { + t.Fatal(err) + } + if got := UpstreamName(*p, "sol"); got != "vendor-c/sol" { + t.Fatalf("the provider is asked for %q", got) + } + if n, ok := ModelName("relay", "sol"); !ok || n != "My Sol" { + t.Fatalf("the model is called %q (%t)", n, ok) + } +} + +// The key is checked where the name is given, and what it says names the +// setting being checked: the wire name a user cannot give, and not the +// price or the limit of the same model. +func TestSetUpstreamNameSaysWhichSettingItChecks(t *testing.T) { + prefsHome(t) + // a providers file magpie did not write: "a" under an id that is not + // spelled the way a provider's is, which is what the check refuses + f := load() + for i := range f.Providers { + if f.Providers[i].ID == "a" { + f.Providers[i].ID = "my_relay" + } + } + if err := store(f); err != nil { + t.Fatal(err) + } + err := SetUpstreamName("my_relay/sol", "vendor-c/sol") + if err == nil { + t.Fatal("a wire name was given for a key the settings do not keep models under") + } + if !strings.Contains(err.Error(), "wire name") { + t.Fatalf("the error does not say which setting it is about: %v", err) + } + if got := settings.Load().ModelWires; len(got) != 0 { + t.Fatalf("a refused name was stored anyway: %v", got) + } +} + +// A row of the ledger is judged by the name in force when the ledger is +// read: a record keeps the name the vendor's reply gave and not the name +// the request went out under, so this is the only name there is to judge by, +// and giving a name after a call re-judges that call — which is what +// usage.Ledger's comment sets out as the trade. What makes it so is that the +// lookup takes the names as the settings stand at the time of reading them, +// with nothing in it that remembers what they held when the call went out, +// and this is that side of it. +func TestUpstreamNameInFollowsTheNamesAsTheyAreNow(t *testing.T) { + prefsHome(t) + // "a" serves "sol" (prefsHome) + if got := UpstreamNameIn(settings.Load().ModelWires, "a", "sol"); got != "sol" { + t.Fatalf("with no name given, a/sol is judged as %q; want sol", got) + } + // a call made and answered under the name magpie knows the model by, + // before any name is given for it + if err := SetUpstreamName("a/sol", "vendor-c/sol"); err != nil { + t.Fatal(err) + } + // the same call read afterwards is judged by the name given since, so + // the vendor's answer naming the very model that was asked for reads + // as another one + if got := UpstreamNameIn(settings.Load().ModelWires, "a", "sol"); got != "vendor-c/sol" { + t.Errorf("after a name is given, a/sol is judged as %q; want vendor-c/sol", got) + } + // and a model with no name of its own is judged by the provider's, the + // same way round: whatever was in force when the call went out is not + // what a later reading of it is measured against + if err := SetUpstreamName("a/*", "vendor-c/one"); err != nil { + t.Fatal(err) + } + if got := UpstreamNameIn(settings.Load().ModelWires, "a", "other"); got != "vendor-c/one" { + t.Errorf("a model with no name of its own is judged as %q; want the provider's", got) + } + if got := UpstreamNameIn(settings.Load().ModelWires, "a", "sol"); got != "vendor-c/sol" { + t.Errorf("with both names in force, a/sol is judged as %q; want its own over the provider's", got) + } +} + +// A ref is taken at the key its name is really stored at, and the spellings +// are tried in one order: the ref as the user spelled it, the id the provider +// has or had, and last the name it is shown by — a spelling another provider +// may be shown by as well, so it is only trusted where nothing is kept under +// either of the other two. A name outlives the provider it was given for, so +// the first is where it is when that provider is gone and its id is free +// again, and it is the one a removal has to reach. +func TestDropUpstreamNameTakesTheKeyTheNameIsStoredAt(t *testing.T) { + prefsHome(t) + // the name a provider is shown by, under the id it has + if err := Save(Provider{ID: "relay", Name: "My Relay", Catalog: "a", Key: "k", + Chat: "http://127.0.0.1:1/v1", Models: []string{"sol"}}); err != nil { + t.Fatal(err) + } + if err := SetUpstreamName("My Relay/sol", "vendor-c/sol"); err != nil { + t.Fatal(err) + } + if dropped, err := DropUpstreamName("My Relay/sol"); err != nil || !dropped { + t.Fatalf("DropUpstreamName = %v, %v; want the name taken off by the name its provider is shown by", dropped, err) + } + // an id a provider had before a rename is the id every lookup still + // goes by, and so is the one a name is taken off at + if err := SetUpstreamName("a/sol", "vendor-c/sol"); err != nil { + t.Fatal(err) + } + if err := Rename("a", "renamed"); err != nil { + t.Fatal(err) + } + if dropped, err := DropUpstreamName("a/sol"); err != nil || !dropped { + t.Fatalf("DropUpstreamName = %v, %v; want the name taken off by the id its provider had", dropped, err) + } + // an id no provider is called by and no name is kept under is a mistyped + // one: there is nothing to take off, which is not an error, and it + // takes nothing else away on the way to saying so + if dropped, err := DropUpstreamName("nope/sol"); dropped || err != nil { + t.Fatalf("DropUpstreamName on an id no provider is called by = %v, %v; want no name taken and no error", dropped, err) + } + if got := settings.Load().ModelWires; len(got) != 0 { + t.Errorf("the settings hold %v; a ref naming no provider took nothing off, and nothing else either", got) + } +} + +// The id a wire name was given under is free again once its provider is +// deleted, and another provider can be shown by it — the name that provider +// is shown by is a spelling any other provider may answer to, so `b/sol` +// names a provider that is not the one a name under `b/sol` is for. Taking +// the name off through it would take away that provider's own name instead, +// a name that is really in force, and leave the one that renames a +// provider's requests for a vendor that never heard of them in the file. +func TestDropUpstreamNameWorksOffTheKeyRatherThanTheProvidersName(t *testing.T) { + prefsHome(t) + if err := SetUpstreamName("b/sol", "vendor-c/sol"); err != nil { + t.Fatal(err) + } + if err := Delete("b"); err != nil { + t.Fatal(err) + } + // a provider of an id of its own, c, shown by the id b that is now + // nobody's, and with a name of its own to be left alone + if err := Save(Provider{ID: "c", Name: "B", Catalog: "b", Key: "k", + Chat: "http://127.0.0.1:1/v1", Models: []string{"sol"}}); err != nil { + t.Fatal(err) + } + if err := SetUpstreamName("c/sol", "vendor-d/sol"); err != nil { + t.Fatal(err) + } + if dropped, err := DropUpstreamName("b/sol"); err != nil || !dropped { + t.Fatalf("DropUpstreamName = %v, %v; want the name under b/sol taken off by that key", dropped, err) + } + wires := settings.Load().ModelWires + if _, ok := wires["b/sol"]; ok { + t.Errorf("b/sol still has a name, %v; it is in force for whichever provider takes that id next", wires) + } + if got := wires["c/sol"]; got != "vendor-d/sol" { + t.Errorf("c's own name is %q; the removal went through the name c is shown by", got) + } + // which is the whole of what taking the name off buys: a provider + // taking the id b is asked for its model by the name magpie knows it by + if err := Save(Provider{ID: "b", Name: "B", Catalog: "b", Key: "k", + Chat: "http://127.0.0.1:1/v1", Models: []string{"sol"}}); err != nil { + t.Fatal(err) + } + p, err := Find("b") + if err != nil { + t.Fatal(err) + } + if got := UpstreamName(*p, "sol"); got != "sol" { + t.Errorf("the provider taking the id b is asked for %q; want sol", got) + } +} + +// HasUpstreamName answers for the key a name is really kept under, however +// it is spelled: a name that is there is not "not kept" because the ref +// that named it carried a magpie/ in front of it, which is the spelling +// every agent uses and the one the removal standing in front of it is asked +// for by. +func TestHasUpstreamNameReadsTheKeyHoweverItIsSpelled(t *testing.T) { + prefsHome(t) + if err := SetUpstreamName("b/sol", "vendor-c/sol"); err != nil { + t.Fatal(err) + } + if err := SetUpstreamName("b/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + for _, key := range []string{"b/sol", "magpie/b/sol", " b/sol ", "magpie/b/*"} { + if !HasUpstreamName(key) { + t.Errorf("a name is kept under %q; HasUpstreamName says none is", key) + } + } + for _, key := range []string{"b/other", "magpie/b/other", "magpie/a/sol", ""} { + if HasUpstreamName(key) { + t.Errorf("no name is kept under %q; HasUpstreamName says one is", key) + } + } +} + +// A name given for one model is that model's at every level, so the vendor +// is asked for the model by the name its owner gave it whichever variant +// answers for it — the name the user is told that model goes out under, and +// never a "*" the vendor never heard of. A name given for the whole provider +// is a different thing: a "*" in it is whatever a request goes out under, so +// each level is asked for by its own id and the effort keeps choosing. +func TestUpstreamNameAsGivesAModelTheNameItWasGiven(t *testing.T) { + prefsHome(t) + a, err := Find("a") + if err != nil { + t.Fatal(err) + } + if err := SetUpstreamName("a/sol", "vendor-c/*"); err != nil { + t.Fatal(err) + } + if got, want := UpstreamName(*a, "sol"), "vendor-c/sol"; got != want { + t.Fatalf("a/sol is asked for as %q; want %q", got, want) + } + if got := UpstreamNameAs(*a, "sol", "sol-high"); got != "vendor-c/sol" { + t.Errorf("at the sol-high variant it is asked for as %q; want the model's own name, vendor-c/sol", got) + } + + b, err := Find("b") + if err != nil { + t.Fatal(err) + } + if err := SetUpstreamName("b/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ sent, want string }{ + {"sol-high", "vendor-c/sol-high"}, + {"sol", "vendor-c/sol"}, + } { + if got := UpstreamNameAs(*b, "sol", tc.sent); got != tc.want { + t.Errorf("with a name for the whole provider, asking for %q gives %q; want %q", tc.sent, got, tc.want) + } + } +} + +// The name a call goes out under is the one the vendor is asked for, and on +// an Antigravity account that is not the model magpie knows: the family is +// asked for as the variant the effort picks, each by its own name. A ledger +// reading the record back therefore judges the reply against the id that +// really went out — read against the family, every level of it reads as +// another model answering. +func TestSentNameInAsksForTheVariantTheEffortPicks(t *testing.T) { + prefsHome(t) + // Antigravity's ids of one model at three levels, as its fetch left them + if err := os.MkdirAll(filepath.Dir(catalog.LivePath("antigravity")), 0o755); err != nil { + t.Fatal(err) + } + live := `{"models":[{"id":"flash-9-low"},{"id":"flash-9-medium"},{"id":"flash-9-high"}]}` + if err := os.WriteFile(catalog.LivePath("antigravity"), []byte(live), 0o644); err != nil { + t.Fatal(err) + } + catalog.Reset() + wires := map[string]string{"antigravity/*": "vendor-c/*"} + for _, tc := range []struct{ effort, want string }{ + {"high", "vendor-c/flash-9-high"}, + {"medium", "vendor-c/flash-9-medium"}, + {"low", "vendor-c/flash-9-low"}, + // no level asked for is the level Antigravity thinks at when it is + // not told one, which is the highest of the family + {"", "vendor-c/flash-9-high"}, + } { + if got := SentNameIn(wires, "antigravity", "flash-9", tc.effort); got != tc.want { + t.Errorf("at %q the vendor is asked for %q; want %q", tc.effort, got, tc.want) + } + } + // with no name given, it is the id itself: a reply naming the level the + // call went out at is that model, whatever family it is a level of + if got := SentNameIn(nil, "antigravity", "flash-9", "medium"); got != "flash-9-medium" { + t.Errorf("with no upstream name the vendor is asked for %q; want the variant", got) + } + // another provider has no variants of its own, so the effort is no part + // of the name and a star given for the provider stands for the model + if got := SentNameIn(map[string]string{"a/*": "vendor-d/*"}, "a", "sol", "high"); got != "vendor-d/sol" { + t.Errorf("provider a at high is asked for %q; want vendor-d/sol", got) + } +} + +// The id a call goes out under is one id, whether the request named the +// family or one of its own variant ids: an agent's model list and a pick +// saved from before the families were one model still name those, and +// Antigravity serves each of them as it stands. So an effort asked for +// picks the level of the family in either case, and no effort asked for +// leaves the id that was named alone — the rule the gateway's envelope is +// built by (codeAssistID) and the rule a ledger reads a record back against +// are one rule, and these are the two ends of it. +func TestSentNameInAsksAVariantIdForWhatTheEffortPicks(t *testing.T) { + prefsHome(t) + // one model at three levels, and one at two, so a family's default is + // the top of it in one case and not the other + if err := os.MkdirAll(filepath.Dir(catalog.LivePath("antigravity")), 0o755); err != nil { + t.Fatal(err) + } + live := `{"models":[{"id":"flash-9-low"},{"id":"flash-9-medium"},{"id":"flash-9-high"},` + + `{"id":"note-4-low"},{"id":"note-4-medium"}]}` + if err := os.WriteFile(catalog.LivePath("antigravity"), []byte(live), 0o644); err != nil { + t.Fatal(err) + } + catalog.Reset() + t.Cleanup(catalog.Reset) + for _, tc := range []struct{ model, effort, want string }{ + // the effort the client asks for wins, as it does for the family + {"flash-9-high", "low", "flash-9-low"}, + {"flash-9-low", "medium", "flash-9-medium"}, + // with none asked for, the id that was named goes as it is + {"flash-9-high", "", "flash-9-high"}, + {"flash-9-low", "", "flash-9-low"}, + // an effort the family does not take is asked at the level of its + // own nearest, not at the one the family thinks at when told none + {"flash-9-high", "minimal", "flash-9-low"}, + {"flash-9", "xhigh", "flash-9-high"}, + // the same for a family whose default is not its top: the id asked + // for is the one that goes out, and an effort moves it to the level + // it picks + {"note-4-low", "", "note-4-low"}, + {"note-4-medium", "", "note-4-medium"}, + {"note-4-low", "high", "note-4-medium"}, + {"note-4-low", "minimal", "note-4-low"}, + } { + if got := SentNameIn(nil, "antigravity", tc.model, tc.effort); got != tc.want { + t.Errorf("%s at %q goes out as %q; want %q", tc.model, tc.effort, got, tc.want) + } + if got := AntigravitySentID(tc.model, tc.effort); got != tc.want { + t.Errorf("%s at %q: the id itself is %q, want %q", tc.model, tc.effort, got, tc.want) + } + } + // a model of no family goes as it is, whatever effort is asked for + if got := SentNameIn(nil, "antigravity", "solo", "high"); got != "solo" { + t.Errorf("a model of no family is asked for as %q; want solo", got) + } +} + +// The variants belong to the account and not to the provider: a model is +// asked for one of its levels only where the account behind the provider is +// the one serving that family. The two are separate questions wherever a +// provider answers to an id that is not its account's — an account provider is +// built with its account's own id today, so reading one for the other agrees +// and says nothing about the rule. So an account the user is signed in to +// under a provider of its own id is asked for the variant, and a relay added +// under a built-in account's id is that relay and is asked for the model +// magpie knows, which is what the gateway's own dispatch (codeAssistID) reads +// the same way. +func TestSentNameIsJudgedByTheAccountAndNotByTheProviderId(t *testing.T) { + prefsHome(t) + if err := os.MkdirAll(filepath.Dir(catalog.LivePath("antigravity")), 0o755); err != nil { + t.Fatal(err) + } + live := `{"models":[{"id":"flash-9-low"},{"id":"flash-9-high"}]}` + if err := os.WriteFile(catalog.LivePath("antigravity"), []byte(live), 0o644); err != nil { + t.Fatal(err) + } + catalog.Reset() + t.Cleanup(catalog.Reset) + wires := map[string]string{"relay-1/*": "vendor-c/*", "antigravity/*": "vendor-d/*"} + + // the account is the one serving the family, whatever id the provider + // it is on answers to + if got := SentNameOnIn(wires, "relay-1", "antigravity", "flash-9", "low"); got != "vendor-c/flash-9-low" { + t.Errorf("an Antigravity account on a provider of another id, at low: the vendor is asked for %q; want vendor-c/flash-9-low", got) + } + // and the provider's id is no part of the question: a relay the user + // added under a built-in account's id is that relay, and no level of + // the family is what a call on it goes out under + if got := SentNameOnIn(wires, "antigravity", "", "flash-9", "low"); got != "vendor-d/flash-9" { + t.Errorf("a relay under the account's own id, at low: the vendor is asked for %q; want vendor-d/flash-9", got) + } + // another account is asked for the model magpie knows as well + if got := SentNameOnIn(wires, "relay-1", "gemini", "flash-9", "low"); got != "vendor-c/flash-9" { + t.Errorf("another account on the same provider, at low: the vendor is asked for %q; want vendor-c/flash-9", got) + } + // a record keeps the id the call went out on and no account, and that id + // is the account's own, so the ledger reads it back the same way — and a + // record of a relay, whose id is no account's, by that id + if got := SentNameIn(wires, "antigravity", "flash-9", "low"); got != "vendor-d/flash-9-low" { + t.Errorf("a record of a call on the account's own provider, at low, reads back as %q; want vendor-d/flash-9-low", got) + } + if got := SentNameIn(wires, "relay-1", "flash-9", "low"); got != "vendor-c/flash-9" { + t.Errorf("a record of a call on a relay, at low, reads back as %q; want vendor-c/flash-9", got) + } +} + +// A name that is only whitespace is no name at either level of the lookup, +// as it is in UpstreamNameIn: the model's own falls through to the +// provider's, and both falling through leaves the request under the id it +// was built with. A blank name read as a name would send a relay an id of +// spaces. +func TestUpstreamNameAsInIgnoresANameOfOnlyWhitespace(t *testing.T) { + for _, tc := range []struct { + what string + wires map[string]string + sent, want string + }{ + {"the model's own name is blank, the provider's is not", + map[string]string{"relay/sol": " ", "relay/*": "vendor-c/*"}, "sol-high", "vendor-c/sol-high"}, + {"both are blank", + map[string]string{"relay/sol": " ", "relay/*": "\t"}, "sol-high", "sol-high"}, + {"the provider's is blank and no other model has one", + map[string]string{"relay/sol": " ", "relay/*": " "}, "sol-high", "sol-high"}, + {"the request goes out under the model itself", + map[string]string{"relay/sol": " ", "relay/*": "vendor-c/*"}, "sol", "vendor-c/sol"}, + } { + if got := UpstreamNameAsIn(tc.wires, "relay", "sol", tc.sent); got != tc.want { + t.Errorf("%s: asking for %q gives %q; want %q", tc.what, tc.sent, got, tc.want) + } + } +} + +// The gateway decides where a request goes, and for an account it hands to +// the agent's own backend the model it knows is what goes out — so a wire +// name given for one of those accounts is accepted, never sent, and read at +// the ledger as every call through it a swap. asksOwnBackend refuses them +// where the name is given, and the two have to be the same set: an account +// the gateway starts serving its own way is refused a name the day it +// lands. So the set is read out of the dispatch itself rather than kept a +// second time beside it, which is the copy that goes on saying what the +// dispatch stopped saying. +func TestAsksOwnBackendAgreesWithTheGatewayDispatch(t *testing.T) { + d := &dispatch{t: t, seen: map[string]bool{}} + dispatched := d.accountBackends(filepath.Join("..", "gateway")) + if len(dispatched) == 0 { + t.Fatal("no account is read as one the gateway asks through its own backend; its dispatch is not being read") + } + for _, agent := range dispatched { + if !(Provider{Account: &Account{Agent: agent}}).asksOwnBackend() { + t.Errorf("the gateway asks a %s account through its own backend, where the model id the agent knows is what goes out, and a wire name is accepted for one of its models", agent) + } + } + // and nothing beyond them: an account the gateway asks over a vendor's + // API does send the name, so refusing one there refuses a name that + // would have worked + for _, agent := range accountIDs { + if slices.Contains(dispatched, agent) { + continue + } + if (Provider{Account: &Account{Agent: agent}}).asksOwnBackend() { + t.Errorf("a %s account is asked over the vendor's own API, where a wire name is sent, and one is refused for its models", agent) + } + } +} + +// dispatch reads an account dispatch out of the sources it is written in: +// which package a name is a constant of, and which constants are on their +// way to themselves. A module's own name is the prefix every import path in +// it has, which is how one package's constant is looked up in another. +type dispatch struct { + t *testing.T + root string // the tree the module is checked out in + mod string // what its go.mod calls the module + seen map[string]bool +} + +// accountBackends are the agents the gateway serves a request itself for, +// where the account belongs to the agent whose backend it is: a branch that +// asks whether the provider's account is one of them and then reaches that +// backend's own serve, as against one that sends the request to a vendor. +// A test file is not read: what a test says the dispatch does is no part of +// what it does. +func (d *dispatch) accountBackends(dir string) []string { + d.t.Helper() + var out []string + for _, name := range d.files(dir) { + ast.Inspect(d.parse(name), func(n ast.Node) bool { + ifs, ok := n.(*ast.IfStmt) + if !ok || !d.servesItself(ifs) { + return true + } + for _, agent := range d.agents(ifs.Cond, dir) { + if !slices.Contains(out, agent) { + out = append(out, agent) + } + } + return true + }) + } + slices.Sort(out) + return out +} + +// servesItself says whether a branch is one where the request is answered for +// the agent that owns the account instead of sent to a vendor: it ends in +// one of the serves, each named for the backend it answers for and reached +// by every account of that agent. A branch that only reads the account — what +// a request needs from it, which model it counts tokens for — is not one, +// and neither is a call that begins with serve and serves nothing: +// servesElsewhere answers whether a provider serves something. +func (d *dispatch) servesItself(ifs *ast.IfStmt) bool { + d.t.Helper() + for _, stmt := range ifs.Body.List { + own := false + ast.Inspect(stmt, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + if sel, ok := call.Fun.(*ast.SelectorExpr); ok { + if name := sel.Sel.Name; len(name) > len("serve") && name[len("serve")] >= 'A' && name[len("serve")] <= 'Z' { + own = true + } + } + return !own + }) + if own { + return true + } + } + return false +} + +// agents are the ones a condition compares the provider's account against, +// and nothing at all unless it asks the account is there: the same +// comparison is made elsewhere in the gateway to tell what a request needs +// rather than where it goes. +func (d *dispatch) agents(cond ast.Expr, dir string) []string { + d.t.Helper() + var out []string + account := false + ast.Inspect(cond, func(n ast.Node) bool { + b, ok := n.(*ast.BinaryExpr) + if !ok { + return true + } + switch b.Op { + case token.NEQ: + if d.isField(b.X, "Account") && d.isIdent(b.Y, "nil") { + account = true + } + case token.EQL: + sel, ok := b.X.(*ast.SelectorExpr) + if !ok || sel.Sel.Name != "Agent" || !d.isField(sel.X, "Account") { + return true + } + agent := d.id(dir, b.Y) + if agent == "" { + d.t.Errorf("the dispatch compares an account's agent against %s, which is not an id this test can read: a wire name for that account would be accepted and never sent", + types.ExprString(b.Y)) + return true + } + out = append(out, agent) + } + return true + }) + if !account { + return nil + } + return out +} + +// isField says whether the expression is a field of that name, and isIdent +// whether it is a name standing for itself. +func (d *dispatch) isField(e ast.Expr, name string) bool { + sel, ok := e.(*ast.SelectorExpr) + return ok && sel.Sel.Name == name +} + +func (d *dispatch) isIdent(e ast.Expr, name string) bool { + id, ok := e.(*ast.Ident) + return ok && id.Name == name +} + +// id is the agent id a comparison is against, read as far as it is written: +// a string as it stands, or the constant it names — of the package being +// read, as the gateway writes its own, or of one that package imports, as +// Qoder's CN subscription is named. A constant written in terms of itself +// is no id, and says so rather than standing in for one. +func (d *dispatch) id(dir string, e ast.Expr) string { + d.t.Helper() + switch x := e.(type) { + case *ast.BasicLit: + if x.Kind == token.STRING { + s, err := strconv.Unquote(x.Value) + if err != nil { + d.t.Errorf("%s is not a string this test can read: %v", x.Value, err) + return "" + } + return s + } + case *ast.Ident: + return d.constant(dir, x.Name) + case *ast.SelectorExpr: + if pkg, ok := x.X.(*ast.Ident); ok { + return d.constant(d.pkgDir(dir, pkg.Name), x.Sel.Name) + } + } + return "" +} + +// constant is what a constant of the package in dir is written as, in the +// first file that declares it, or the empty string where none does. +func (d *dispatch) constant(dir, name string) string { + d.t.Helper() + key := filepath.Join(dir, name) + if d.seen[key] { + return "" + } + d.seen[key] = true + for _, file := range d.files(dir) { + var id string + ast.Inspect(d.parse(file), func(n ast.Node) bool { + spec, ok := n.(*ast.ValueSpec) + if !ok || id != "" || len(spec.Names) != 1 || spec.Names[0].Name != name || len(spec.Values) != 1 { + return true + } + id = d.id(dir, spec.Values[0]) + return true + }) + if id != "" { + return id + } + } + return "" +} + +// pkgDir is where the package in dir imports `pkg` from keeps its sources: +// an import path is the module's own name and a path under it, so the path +// under it is where those sources are in the same tree. +func (d *dispatch) pkgDir(dir, pkg string) string { + d.t.Helper() + if d.root == "" { + d.root, d.mod = d.module() + } + mod := d.mod + "/" + for _, name := range d.files(dir) { + for _, im := range d.parse(name).Imports { + path, err := strconv.Unquote(im.Path.Value) + if err != nil || !strings.HasPrefix(path, mod) { + continue + } + as := filepath.Base(path) + if im.Name != nil { + as = im.Name.Name + } + if as == pkg { + return filepath.Join(d.root, filepath.FromSlash(strings.TrimPrefix(path, mod))) + } + } + } + d.t.Fatalf("no package is imported as %q from %s; the dispatch cannot be read", pkg, dir) + return "" +} + +// module is the tree the sources are in and what its go.mod calls the +// module, found by walking up to the go.mod that names it. +func (d *dispatch) module() (string, string) { + d.t.Helper() + root, err := filepath.Abs(".") + if err != nil { + d.t.Fatalf("the tree the sources are in: %v", err) + } + for { + if b, err := os.ReadFile(filepath.Join(root, "go.mod")); err == nil { + for _, l := range strings.Split(string(b), "\n") { + if m, ok := strings.CutPrefix(l, "module "); ok { + return root, strings.TrimSpace(m) + } + } + } + if up := filepath.Dir(root); up != root { + root = up + continue + } + break + } + d.t.Fatal("no go.mod above the sources says what the module is called; the dispatch cannot be read") + return "", "" +} + +// files are the sources of a package, its tests left out. +func (d *dispatch) files(dir string) []string { + d.t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + d.t.Fatalf("the sources in %s: %v", dir, err) + } + var out []string + for _, e := range entries { + if n := e.Name(); !e.IsDir() && strings.HasSuffix(n, ".go") && !strings.HasSuffix(n, "_test.go") { + out = append(out, filepath.Join(dir, n)) + } + } + if len(out) == 0 { + d.t.Fatalf("no sources in %s", dir) + } + slices.Sort(out) + return out +} + +func (d *dispatch) parse(name string) *ast.File { + d.t.Helper() + f, err := parser.ParseFile(token.NewFileSet(), name, nil, 0) + if err != nil { + d.t.Fatalf("%s: %v", name, err) + } + return f +} diff --git a/internal/provider/test.go b/internal/provider/test.go index 19b9e0d1..ce579d32 100644 --- a/internal/provider/test.go +++ b/internal/provider/test.go @@ -48,7 +48,7 @@ func (p Provider) Test(ctx context.Context) []Result { out = append(out, Result{Protocol: proto, Model: model, Error: "no key is on for this endpoint"}) continue } - url, body := tiny(q, proto, model) + url, body := tiny(q, proto, UpstreamName(p, model)) out = append(out, probe(ctx, q, proto, url, q.Prepare([]byte(body)), model, testWait)) } return out @@ -172,7 +172,12 @@ func (p Provider) testOne(ctx context.Context, model string) Result { } if draws { // as the gateway does, one the images API doesn't serve is tried - // in chat, and only its answer said when that fails too + // in chat, and only its answer said when that fails too. Both go + // out under the name magpie knows the model by: the gateway builds + // those bodies itself and an upstream name is never written into + // one, so asking for the model's own here is what tests a drawing + // the way a real one is made — a name the vendor serves drawings + // by is not one magpie sends them as url, body := tinyDrawing(q, model) r := probe(ctx, q, proto, url, []byte(body), model, drawWait) if !r.OK && (r.Status == 404 || r.Status == 405) { @@ -183,7 +188,7 @@ func (p Provider) testOne(ctx context.Context, model string) Result { } return r } - url, body := tiny(q, proto, model) + url, body := tiny(q, proto, UpstreamName(p, model)) return probe(ctx, q, proto, url, q.Prepare([]byte(body)), model, testWait) } diff --git a/internal/settings/settings.go b/internal/settings/settings.go index 5711a9d2..04fd4b18 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -197,6 +197,15 @@ type Settings struct { // provider's, and the agents' own files are told of either // (see provider.SetModelOutput). ModelOutputs map[string]int `json:"modelOutputs,omitempty"` + // ModelWires is the name to send a vendor for a model magpie knows by + // another, by "/", and "*" for every model of that + // provider. A "*" in the name is the model itself, so one name covers a + // relay that namespaces its models — vendor-c/* asks for model-3 as + // vendor-c/model-3 — while a name with no "*" in it sends every model of + // that key under that one name. Everything else — the catalog agents see, + // the routing groups, the usage records and what a call is priced at — + // keeps the name magpie knows the model by. + ModelWires map[string]string `json:"modelWires,omitempty"` // The main window's size when it was last resized, width and height, // so it opens at it again after a restart. Window []int `json:"window,omitempty"` diff --git a/internal/usage/ledger.go b/internal/usage/ledger.go index cbecb49d..de159948 100644 --- a/internal/usage/ledger.go +++ b/internal/usage/ledger.go @@ -14,6 +14,7 @@ import ( "github.com/yetone/magpie/internal/catalog" "github.com/yetone/magpie/internal/provider" + "github.com/yetone/magpie/internal/settings" ) // Row is one call as the ledger lists it. @@ -67,12 +68,47 @@ func (p Period) Since(now time.Time) time.Time { // Ledger is the calls of a period that the filter keeps, newest first, // with their sum, and the agents that made any call in the period (their // ids, for a filter to offer). +// +// A row's swapped is judged by the upstream names in force now, not by the +// one its call went out under: the names are read from the settings here, +// once, so a name given or changed since a call was recorded re-judges that +// call — one recorded under the name magpie knows the model by, and read +// after a name was set for it, reads as a swap although the vendor answered +// with the very model that was asked for. A record keeps the canonical id +// and the name the vendor's own reply gave (served_model) and not the name +// the request was sent under, so this is the only name there is to judge by +// — read as the name that went out, which the effort a record keeps of +// makes exact: an Antigravity account is asked for the variant of the model +// that level picks, so a reply naming that one is the model that was asked +// for and not another, whatever the family it is a level of. +// It is how the loop below already treats the rest of a row: a provider +// renamed since is put back to the id it has now, and the names in force now +// are what the catalog lists the models under as well. +// +// Storing the name that was sent would judge each call by what it was +// really sent under, and it is the better answer, but it is not this +// function's to make: it is a field on every record the gateway writes, and +// so a change to the file's format — a column a row written by an older +// magpie has no value for, and a decision about whether an empty one means +// "no upstream name" or "unknown" — rather than a row of a report. The rows +// here are what a reader sets beside a bill, and a name in force today is +// the one whose reply a bill's model column is read against. +// +// The names are read here, once for the lot, rather than asked for a row +// at a time, so a caller already holding them — a report over the +// provider/model it was given — reads them the same way and reaches the +// same judgement: what a row says is the same however the rows were asked +// for. func Ledger(p Period, f Filter) (rows []Row, sum Totals, agents []string) { return ledger(p.Since(time.Now()), f, Load(time.Time{})) } func ledger(since time.Time, f Filter, recs []Record) (rows []Row, sum Totals, agents []string) { renamed := provider.Renamed() + // the upstream names in force now, read once for the lot: a row is + // judged by the names standing today, which is what Ledger says, and + // the rows here are what a reader compares a bill against + wires := settings.Load().ModelWires priceOf := pricer() rows = []Row{} agents = []string{} @@ -92,7 +128,7 @@ func ledger(since time.Time, f Filter, recs []Record) (rows []Row, sum Totals, a } pr := priceOf(r) sum.add(r, pr) - row := Row{Record: r, Swapped: r.Served != "" && Swapped(r.Model, r.Served)} + row := Row{Record: r, Swapped: r.Served != "" && Swapped(provider.SentNameIn(wires, r.Provider, r.Model, r.Effort), r.Served)} if pr != nil && r.Input+r.Output > 0 { row.Cost, row.Priced = pr.Cost(r.Input, r.Output, r.CacheRead, r.CacheWrite), true } diff --git a/internal/usage/ledger_test.go b/internal/usage/ledger_test.go index 0f3d3aba..08077499 100644 --- a/internal/usage/ledger_test.go +++ b/internal/usage/ledger_test.go @@ -193,3 +193,60 @@ func TestLedgerUsesTheStatedPrice(t *testing.T) { t.Fatalf("row cost %v, total %v; want %v", rows[0].Cost, sum.Cost, want) } } + +// A row is judged against the id the call went out under, which on an +// Antigravity account is the variant of the model the effort picks and not +// the family it is one level of: a reply naming that variant is the model +// that was asked for, and a name given for the whole provider asks each +// level by its own id, which is the very name the reply names back. Read +// against the family instead, every level of it reads as another model +// having swapped in — in a report a user sets beside a bill. +func TestLedgerJudgesAnAntigravityCallByTheVariantItWentOutUnder(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) + t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache")) + // Antigravity's ids of one model at three levels, as its fetch left them + var raw []catalog.Model + for _, id := range []string{"flash-9-low", "flash-9-medium", "flash-9-high"} { + raw = append(raw, catalog.Model{ID: id}) + } + if err := catalog.SaveLive("antigravity", "", raw); err != nil { + t.Fatal(err) + } + catalog.Reset() + t.Cleanup(catalog.Reset) + if err := settings.Save(settings.Settings{ModelWires: map[string]string{"antigravity/*": "vendor-c/*"}}); err != nil { + t.Fatal(err) + } + os.MkdirAll(filepath.Dir(Path()), 0o755) + now := time.Now() + for i, c := range []struct{ effort, level string }{ + {"high", "high"}, {"low", "low"}, + // with no effort asked for the family thinks at its default, which is + // the highest of it + {"", "high"}, + } { + Append(Record{Time: now.Add(time.Duration(i) * time.Minute), Provider: "antigravity", + Host: "antigravity.example", Model: "flash-9", Effort: c.effort, + Served: "vendor-c/flash-9-" + c.level, Input: 10, Output: 1, Status: 200}) + } + // a level the call did not go out at is another model answering + Append(Record{Time: now.Add(4 * time.Minute), Provider: "antigravity", + Host: "antigravity.example", Model: "flash-9", Effort: "high", + Served: "vendor-c/flash-9-low", Input: 10, Output: 1, Status: 200}) + + rows, sum, _ := Ledger(All, Filter{}) + if len(rows) != 4 || sum.Calls != 4 { + t.Fatalf("rows %d, %+v", len(rows), rows) + } + // newest first: the swap, then the three levels + if !rows[0].Swapped { + t.Errorf("a relay that answered with another level's id is read as no swap: %+v", rows[0]) + } + for _, row := range rows[1:] { + if row.Swapped { + t.Errorf("the call went out at %q and was answered with %q, which is that very model: %+v", row.Effort, row.Served, row) + } + } +} diff --git a/model_cli.go b/model_cli.go index dfbf4a37..30f4cfdd 100644 --- a/model_cli.go +++ b/model_cli.go @@ -35,6 +35,15 @@ const modelUsage = `usage: magpie model output the most a reply of it may hold, and what you said magpie model output say the most, as 128000 or 128k; '/*' is every model magpie model output --reset take your limit off this model + magpie model wire the name the vendor is asked for, and the one you gave + magpie model wire ask for the model by this name, for a relay that serves it + magpie model wire '/*' ask for every model of that provider by this name; a * in + it is the model, so vendor-c/* asks for model-3 as + vendor-c/model-3. Quote it: a shell reads a bare * as a glob + magpie model wire --reset ask for it by the name magpie knows it by again, under an + id of its own for a provider that has since been deleted, + and says when there was no name of its own to take away + magpie model wires the names your vendors are asked for models by magpie model names the models you named or narrowed magpie model suffix [on|off] whether the agents' lists name each model with its provider (or "routing group") after it: on, as by default, "Sol · OpenAI"; @@ -71,12 +80,16 @@ func modelCmd(args []string) error { return modelPrice(args[1:]) case "prices": return modelPrices() + case "wires": + return modelWires() case "suffix", "suffixes": return modelSuffix(args[1:]) case "context", "ctx": return modelContext(args[1:]) case "output", "max-output": return modelOutput(args[1:]) + case "wire", "upstream": + return modelWire(args[1:]) case "help", "-h", "--help": fmt.Println(modelUsage) return nil @@ -449,6 +462,42 @@ func modelNames() error { return nil } +// modelWires lists the names the user gave the vendors their models are asked +// for, and which provider and model each is for. It is `magpie model prices` +// for the wire names: those are what a provider is asked for, and no other +// command says what is kept for a provider as a whole, so a name given for +// every model of one was written with no way to read it back. +// +// A name kept for a provider that has since been deleted is listed like any +// other: the name outlives the provider it was given for, and is in force for +// whichever provider takes that id next, so it is read off the file rather +// than off a provider that may not be there. +// +// A name that is only whitespace is no name and is not listed, as it is +// nowhere else a name is read: UpstreamNames leaves it out and a request +// under it goes out by the name magpie knows the model by. Listing it would +// print a line with nothing on it for a name no vendor is asked for, which +// is the one thing a listing read back off the file must not say. +func modelWires() error { + wires := settings.Load().ModelWires + rows := make([][2]string, 0, len(wires)) + w := 0 + for _, k := range slices.Sorted(maps.Keys(wires)) { + if n := strings.TrimSpace(wires[k]); n != "" { + rows = append(rows, [2]string{k, n}) + w = max(w, len(k)) + } + } + if len(rows) == 0 { + fmt.Println(muted.Render("no vendor is asked for a model by another name yet · magpie model wire ")) + return nil + } + for _, r := range rows { + fmt.Println(" " + pad(r[0], w) + " " + r[1]) + } + return nil +} + func modelContext(args []string) error { if len(args) == 0 { return fmt.Errorf("%s", modelUsage) @@ -681,3 +730,282 @@ func tokenCount(n int) string { return fmt.Sprintf("%d", n) } } + +func modelWire(args []string) error { + if len(args) == 0 { + return fmt.Errorf("%s", modelUsage) + } + // --reset comes before the provider is looked for, and works off the key + // the name is stored at: a name outlives the provider it was given for, + // and one whose provider has since been deleted is in force for + // whichever provider takes that id next, its requests going out renamed + // to a vendor that never heard of it. So it is the one name --reset + // most has to reach, and the only one a reset that resolved the + // provider first could not. + // + // What decides this is the key, not the lookup: a name under an id no + // provider has now is in force for whoever takes that id, and so is one + // under an id another provider is shown by — the provider such a ref + // resolves to holding a name of its own, which is the one a removal + // taken through it would take away instead. A ref that names its + // provider by that provider's own id keeps to the path below, which + // reports the removal as one. + if len(args) > 1 && isReset(args[1:]) { + pid, model, err := splitModelRef(args[0]) + if err != nil { + return err + } + p, ferr := provider.Find(pid) + if ferr != nil || (p.ID != pid && provider.HasUpstreamName(pid+"/"+model)) { + dropped, err := provider.DropUpstreamName(pid + "/" + model) + if err != nil { + return err + } + if dropped { + fmt.Println(green.Render("✓"), pid+"/"+model, + muted.Render("no longer has a name of its own; a provider taking that id later is asked for it by the name magpie knows it by")) + return nil + } + // no name is kept under that key, so the id is a mistyped one + // and not a deleted provider's; the lookup below says which + // provider is not there, as it does for every other command + } + } + p, model, err := modelRef(args[0]) + if err != nil { + return err + } + id := p.ID + "/" + model + every, shown := "'"+p.ID+"/*'", p.ID+"/"+model // quoted: a shell reads a bare * as a glob + if model == "*" { + shown = every + } + names := p.UpstreamNames() + _, own := names[model] + _, one := names["*"] + rest := args[1:] + asked := provider.UpstreamName(*p, model) + if len(rest) == 0 { + fmt.Println(bold.Render(asked), muted.Render("· what "+p.Name+" is asked for, for "+shown)) + switch { + case asked == model: + fmt.Println(faint.Render(" · the same name magpie knows it by · magpie model wire " + shown + " to change it")) + case model == "*": + fmt.Println(faint.Render(" · " + exampleOf(p, asked))) + fmt.Println(faint.Render(" · --reset takes it away · magpie model wire " + p.ID + "/ gives a single model one of its own")) + case own: + fmt.Println(faint.Render(" · magpie and the agents still know it as " + model + " · --reset asks for that again")) + case one: + fmt.Println(faint.Render(" · from the name given for every model of " + p.Name + " (" + every + " " + names["*"] + ")" + + " · --reset on it leaves that one in force")) + } + return nil + } + name := strings.Join(rest, " ") + if isReset(rest) { + name = "" + } + // A blank name is a removal, --reset or a name of only whitespace + // alike, and it is DropUpstreamName that makes it: it says whether + // there was a name there to take away, which with the provider's own + // list is what tells a model already asked for by the name magpie + // knows it by from one the provider does not serve at all, for which + // no name could have been kept. A ✓ over a key holding no name would + // read as a name being gone that never was there, and under a + // provider-wide name as the model's own name being the one in force, + // which it is not. So the removal is asked for rather than made blind, + // the way `magpie model price --reset` asks DropModelPrice. + if strings.TrimSpace(name) == "" { + return resetWireName(*p, model, id, shown) + } + if err := provider.SetUpstreamName(id, name); err != nil { + return err + } + now := provider.UpstreamName(*p, model) + if model == "*" { + // the same precedence the reset above is honest about: a model's + // own key wins over the provider's, so the models that keep one + // are not asked for under this, and a ✓ claiming every model of + // the provider is would be false wherever there is one. The + // example stands on a model this reaches, so the ones it does + // not are named here rather than left to be found one at a time. + scope, kept := "every model of "+p.Name, ownKeptSaid(p) + if kept != "" { + scope += " with no name of its own" + } + fmt.Println(green.Render("✓"), shown, muted.Render("· "+scope+" is asked for as"), bold.Render(now)) + fmt.Println(faint.Render(" · " + exampleOf(p, now))) + if kept != "" { + fmt.Println(faint.Render(" · " + kept)) + } + return nil + } + fmt.Println(green.Render("✓"), shown, muted.Render("is asked for as"), bold.Render(now), + faint.Render("· magpie and the agents still know it as "+model)) + return nil +} + +// resetWireName takes a name off under the key it is stored at and says what +// the model is asked for now, which is the name a wildcard entry leaves in +// force where one is in force at all. +// +// A key holding no name is an error rather than a tick, as it is for a price: +// a removal over a name that is not there has taken nothing away, and the +// tick would say the model is asked for by the name magpie knows it by — a +// claim nothing stands behind, and one that is false wherever a name given +// for the whole provider is in force. So there are three answers, as there +// are for a price: the model is not one the provider serves, so none could +// have been kept for it and none is in force; nothing was kept under that key +// of a model it does serve; or what is in force is the provider's own name +// and that is the one --reset reaches. +func resetWireName(p provider.Provider, model, id, shown string) error { + dropped, err := provider.DropUpstreamName(id) + if err != nil { + return err + } + now := provider.UpstreamName(p, model) + if !dropped { + if model != "*" && !provider.ServesModel(p, model) { + // none was kept and none could have been: a model the + // provider has nothing like is never asked for by any name, + // so naming the one in force would claim a request magpie + // does not make + return fmt.Errorf("%s is not a model of %s magpie knows of, so there is nothing to take away and none in force: a name is only kept for a model the provider serves, and this one is asked for by no name", id, p.Name) + } + every := "'" + p.ID + "/*'" + if provider.HasUpstreamName(p.ID + "/*") { + return fmt.Errorf("%s has no name of its own; what it is asked for is %s, the name given for every model of %s, which magpie model wire %s --reset takes away", + id, now, p.Name, every) + } + if model == "*" { + // the same precedence the success below is honest about: a + // model's own key wins over the provider's, so one that keeps + // a name is asked for under it whatever this pattern said, and + // "every model …" is false the moment there is one — so the + // models that keep a name are named here as they are there + if kept := ownKeptSaid(&p); kept != "" { + return fmt.Errorf("%s has no name of its own to reset, so there was nothing to take away: %s", id, kept) + } + return fmt.Errorf("%s has no name of its own to reset, so there was nothing to take away: every model of %s is already asked for by the name magpie knows it by", id, p.Name) + } + return fmt.Errorf("%s has no name of its own to reset, so there was nothing to take away: it is already asked for as %s", id, now) + } + if now != model { + fmt.Println(green.Render("✓"), shown, muted.Render("has no name of its own again · it is still asked for as"), + bold.Render(now), faint.Render("· that is the name given for '"+p.ID+"/*'")) + return nil + } + if model == "*" { + // the provider's own name is gone, but a model's own beats it + // while it is there, so the models that keep one are still asked + // for by it — and saying every model of the provider is asked + // for by the name magpie knows it by is false the moment one + // does, so they are named instead + if kept := ownKeptSaid(&p); kept != "" { + fmt.Println(green.Render("✓"), shown, muted.Render("has no name of its own again ·"), kept) + return nil + } + fmt.Println(green.Render("✓"), shown, muted.Render("· every model of "+p.Name+" is asked for by the name magpie knows it by again")) + return nil + } + fmt.Println(green.Render("✓"), shown, muted.Render("is asked for as"), bold.Render(model), muted.Render("again")) + return nil +} + +// ownKept are the models of a provider that keep a name of their own, in the +// order they are shown. A name given for every model of a provider does not +// reach them: a model's own key wins over the provider's, so each is asked +// for by its own whatever the provider's entry says, and a line about every +// model of that provider is true only where there is none to leave out of it. +// Naming them is what says which of the two is in force, which the example +// such a name is shown with cannot: that one stands on a model with no name +// of its own, or on the pattern alone, and names no model the provider's entry +// does not reach. +// +// A name kept for a model the provider does not serve is not one of these: +// magpie asks for no such a model, so nothing goes out under that name, and +// saying it did would claim a request that is not made. +func ownKept(p *provider.Provider) []string { + names := p.UpstreamNames() + kept := make([]string, 0, len(names)) + for _, m := range slices.Sorted(maps.Keys(names)) { + if m != "*" && provider.ServesModel(*p, m) { + kept = append(kept, m) + } + } + return kept +} + +// ownKeptSaid is ownKept said as a sentence, and is "" where there is none. +// +// A model is named with the name it goes out under, not with the one as it +// is stored: a star in a model's own name is that model, as it is in one +// given for every model of a provider (UpstreamNameIn), so it is the model +// magpie knows the model by that stands there. The whole claim of the +// sentence is that this is the name in force, so a star left standing in it +// would name an id no vendor is ever asked for. +func ownKeptSaid(p *provider.Provider) string { + names, kept := p.UpstreamNames(), ownKept(p) + askedFor := func(m string) string { return strings.ReplaceAll(names[m], "*", m) } + switch len(kept) { + case 0: + return "" + case 1: + return kept[0] + " is asked for as " + askedFor(kept[0]) + ", a name of its own" + } + // a relay with a name of its own for every model it serves would + // otherwise answer in a line as long as its list, so a few are named + // and the rest counted + const room = 3 + shown, more := kept, "" + if len(shown) > room { + shown, more = shown[:room], ", … and "+strconv.Itoa(len(kept)-room)+" more" + } + asked := make([]string, 0, len(shown)) + for _, m := range shown { + asked = append(asked, askedFor(m)) + } + return strings.Join(shown, ", ") + " are asked for as " + strings.Join(asked, ", ") + ", names of their own" + more +} + +// exampleOf is what a name given for a provider's every model asks for: a +// model of that provider for it to stand on, and what that model goes out as. +// The star in such a name is that model, so it says what it stands for. +// +// The model is one of Exposed, the list the user is shown, and not of +// Available, which is every model the vendor is known to serve: a relay +// fronting one vendor under ids of its own shares that vendor's catalogue, +// so its first model is one nobody is offered, and an example standing on it +// names a model the user cannot pick and the name was not given for. Exposed +// is the user's own list where the file has one, so that is what it is read +// from there. +// +// It is the first of them with no name of its own, and that is what the +// provider's own name is really in force for: a model's own key wins over the +// provider's, so a model given a name of its own is asked for by that one +// whatever is given for every model, and an example standing on it would say +// a name no request goes out under, which reads as the one thing the +// provider's own entry does not reach. +// +// A provider with nothing to stand one on — no catalogue, nothing fetched, +// nothing in the file, or a name of its own for every model it shows — is +// left with the pattern alone, which is all a relay naming none of its own +// can be told. +func exampleOf(p *provider.Provider, name string) string { + if !strings.Contains(name, "*") { + return p.Name + " is asked for every one of its models as " + name + } + const star = " · * is the model magpie knows each of them by" + names := p.UpstreamNames() + model := "" + for _, m := range p.Exposed() { + if _, own := names[m.ID]; !own { + model = m.ID + break + } + } + if model == "" { + return p.Name + " is asked for every one of its models as " + name + star + } + return p.ID + "/" + model + " is asked for as " + strings.ReplaceAll(name, "*", model) + star +} diff --git a/model_cli_test.go b/model_cli_test.go index 0da7913c..7fef675b 100644 --- a/model_cli_test.go +++ b/model_cli_test.go @@ -14,6 +14,58 @@ import ( "github.com/yetone/magpie/internal/settings" ) +// saidArgs is what a command printed, which is where the wire name is +// reported and what the messages about it have to be read from. It is named +// for its arguments rather than said so it does not collide with the said of +// a per-model limits or a price command, which runs a closure instead of a +// command line: they are three separate pull requests and the three of them +// in one tree must still be one package. +func saidArgs(t *testing.T, args ...string) string { + t.Helper() + out, err := refusedArgs(t, args...) + if err != nil { + t.Fatal(err) + } + return out +} + +// refusedArgs is what a command printed and the error it refused with, which +// is the pair a command that answers with an error instead of a line has to +// be read as: whether a tick was printed anyway is half of what a user sees +// of a refused command, and the other half is the exit code the error gives +// it, which is 1 for anything this returns (main). +func refusedArgs(t *testing.T, args ...string) (string, error) { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + old := os.Stdout + os.Stdout = w + var out strings.Builder + done := make(chan struct{}) + go func() { + b, _ := io.ReadAll(r) + out.Write(b) + close(done) + }() + err = modelCmd(args) + w.Close() + os.Stdout = old + <-done + r.Close() + return out.String(), err +} + +func mustFind(t *testing.T, id string) provider.Provider { + t.Helper() + p, err := provider.Find(id) + if err != nil { + t.Fatal(err) + } + return *p +} + // magpie model name and --reset keep and drop the user's name for a // provider's model, a name of several words included. func TestModelNameCmd(t *testing.T) { @@ -1123,3 +1175,911 @@ func TestModelLimitCmdSaysWhatALimitIsInTheVerbOfItsOwnCommand(t *testing.T) { } } } + +// magpie model wire keeps and drops the name a provider's models are asked +// for, and says what is in force after every change. A provider-wide name +// stands on a model that provider really serves rather than on an id made up +// for it, a name with no "*" in it is not explained with one, and a name of +// only whitespace is a name taken away, which is answered as the removal it +// is rather than as a setting that took. +func TestModelWireCmd(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + wires := func() map[string]string { return settings.Load().ModelWires } + + // a custom provider's every model, with the model in the name: the + // example is one of that provider's own, which is the only kind this + // exists for — a relay — and not an id no provider here serves + if out := saidArgs(t, "wire", "b/*", "vendor-c/*"); !strings.Contains(out, "b/vendor/m is asked for as vendor-c/vendor/m") { + t.Fatalf("naming every model said %q; want an example that is one of B's own", out) + } else if strings.Contains(out, "model-2") { + t.Fatalf("the example is a model no provider here serves: %q", out) + } + if n := wires()["b/*"]; n != "vendor-c/*" { + t.Fatalf("the name given for every model of B is %q; want vendor-c/*", n) + } + if got := provider.UpstreamName(mustFind(t, "b"), "gpt-5.5"); got != "vendor-c/gpt-5.5" { + t.Errorf("every model asked for as %q; want vendor-c/gpt-5.5", got) + } + + // the query explains the "*" only where there is one: a name with none + // asks for every model alike, and a sentence about the star would + // contradict the name printed above it + if out := saidArgs(t, "wire", "b/*", "vendor-c/one"); !strings.Contains(out, "B is asked for every one of its models as vendor-c/one") { + t.Errorf("naming every model alike said %q", out) + } + if out := saidArgs(t, "wire", "b/*"); !strings.Contains(out, "B is asked for every one of its models as vendor-c/one") || + strings.Contains(out, "* is the model magpie knows each of them by") { + t.Errorf("asking about a name with no star in it said %q; want no sentence about the star", out) + } + if out := saidArgs(t, "wire", "b/*", "vendor-c/*"); !strings.Contains(out, "* is the model magpie knows each of them by") { + t.Errorf("naming every model with a star in it said %q; want the star explained", out) + } + if out := saidArgs(t, "wire", "b/*"); !strings.Contains(out, "b/vendor/m is asked for as vendor-c/vendor/m") || + !strings.Contains(out, "* is the model magpie knows each of them by") { + t.Errorf("asking about a name with a star in it said %q", out) + } + + // one model of its own, which the provider's name cannot outrank + if out := saidArgs(t, "wire", "a/m", "vendor-c/m"); !strings.Contains(out, "is asked for as vendor-c/m") || + !strings.Contains(out, "magpie and the agents still know it as m") { + t.Errorf("naming one model said %q", out) + } + if n := wires()["a/m"]; n != "vendor-c/m" { + t.Fatalf("the name given for a/m is %q; want vendor-c/m", n) + } + if out := saidArgs(t, "wire", "a/m"); !strings.Contains(out, "vendor-c/m · what A is asked for, for a/m") || + !strings.Contains(out, "--reset asks for that again") { + t.Errorf("asking about one model said %q", out) + } + // a name of only whitespace takes the one away, and where nothing is in + // force for the model it is asked for by the name magpie knows it by + // again — not read as a name just given + if out := saidArgs(t, "wire", "a/m", " "); !strings.Contains(out, "is asked for as m again") || + strings.Contains(out, "magpie and the agents still know it as m") { + t.Errorf("a name of only whitespace said %q; want the removal it is", out) + } + if _, ok := wires()["a/m"]; ok { + t.Error("a name of only whitespace left one behind for a/m") + } + // a blank name on a model that never had one takes nothing away, and is + // refused as what it is: there is no name there to be taken + if out, err := refusedArgs(t, "wire", "a/claude-opus-5-5", " "); err == nil { + t.Errorf("a blank name on a model that never had one said %q", out) + } + if got := len(wires()); got != 1 { + t.Errorf("the names given are %v; a blank name is none", wires()) + } + + // a name given for every model of the provider, which a model with no + // name of its own falls back to and a blank name leaves in force + if err := provider.SetUpstreamName("a/*", "vendor-c/one"); err != nil { + t.Fatal(err) + } + if out := saidArgs(t, "wire", "a/only-a"); !strings.Contains(out, "from the name given for every model of A") { + t.Errorf("a model with no name of its own said %q; want the provider's", out) + } + if got := provider.UpstreamName(mustFind(t, "a"), "m"); got != "vendor-c/one" { + t.Errorf("with a/m taken away, m is asked for as %q; want the provider's vendor-c/one", got) + } + + // a name of only whitespace under the provider's is the removal of a + // name a/m never had of its own, and is refused — and what it says is + // the name that *is* in force, the provider's, which a tick over a key + // holding nothing would have left the user with the wrong idea of + out, err := refusedArgs(t, "wire", "a/m", " ") + if err == nil { + t.Fatalf("a name of only whitespace under the provider's said %q", out) + } + if !strings.Contains(err.Error(), "a/m has no name of its own") || + !strings.Contains(err.Error(), "vendor-c/one") { + t.Errorf("the refusal is %q; want a/m's own name missing and the provider's in force", err) + } + if _, ok := wires()["a/m"]; ok { + t.Error("a name of only whitespace left one behind for a/m") + } + + // --reset takes away only the model's own, leaving the provider's + if out := saidArgs(t, "wire", "a/m", "vendor-c/m"); !strings.Contains(out, "is asked for as vendor-c/m") { + t.Fatalf("naming a/m again said %q", out) + } + if out := saidArgs(t, "wire", "a/m", "--reset"); !strings.Contains(out, "has no name of its own again") { + t.Errorf("--reset said %q", out) + } + if _, ok := wires()["a/m"]; ok { + t.Error("--reset left a name for a/m") + } + if n := wires()["a/*"]; n != "vendor-c/one" { + t.Errorf("--reset on a model took the provider's %q away too", n) + } + if out := saidArgs(t, "wire", "a/*", "--reset"); !strings.Contains(out, "every model of A is asked for by the name magpie knows it by again") { + t.Errorf("--reset on every model said %q", out) + } + if got := len(wires()); got != 1 { + t.Errorf("the names given are %v; want only the ones this left", wires()) + } + + if err := modelCmd([]string{"wire", "m", "x"}); err == nil { + t.Error("named a model without its provider") + } +} + +// The example a name given for a provider's every model stands on is a +// model of that provider the user can actually see. Available is every +// model the vendor is known to serve, and a relay fronting one vendor under +// ids of its own shares that vendor's catalogue, so the first of those is a +// model nobody is offered: an example on it names a model that cannot be +// picked and shows the name standing for a model it was not given for. +func TestModelWireExampleIsAModelTheUserIsShown(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + // C lists two models of its own while its fetched list has a third + // first: the one only the fetched list knows is the one a user never + // sees, and the one an example read off that list would name. + if err := provider.Save(provider.Provider{ + ID: "c", Name: "C", Key: "kc", Models: []string{"shown-1", "shown-2"}, Chat: "http://127.0.0.1:1/v1", + }); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(catalog.LivePath("c")), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(catalog.LivePath("c"), []byte(`{"models":[ + {"id":"only-in-the-fetched-list"},{"id":"shown-1"},{"id":"shown-2"}]}`), 0o644); err != nil { + t.Fatal(err) + } + catalog.Reset() + t.Cleanup(catalog.Reset) + if ms := mustFind(t, "c").Available(); len(ms) != 3 || ms[0].ID != "only-in-the-fetched-list" { + t.Fatalf("what C is known to serve is %+v; want the fetched list's three, the unknown one first", ms) + } + if ms := mustFind(t, "c").Exposed(); len(ms) != 2 || ms[0].ID != "shown-1" { + t.Fatalf("what C shows is %+v; want the two of its own file", ms) + } + // both the line that sets the name and the one that asks about it stand + // a model on, so both have to be one the user is shown + for _, args := range [][]string{{"wire", "c/*", "vendor-c/*"}, {"wire", "c/*"}} { + out := saidArgs(t, args...) + if !strings.Contains(out, "c/shown-1 is asked for as vendor-c/shown-1") { + t.Errorf("%v said %q; want the example to stand on shown-1", args, out) + } + if strings.Contains(out, "only-in-the-fetched-list") { + t.Errorf("%v said %q; the example stands on a model C does not show", args, out) + } + } +} + +// The model a provider-wide name is shown with is one that name is really in +// force for. A model's own key wins over the provider's, so a model already +// given a name is asked for by that one however the provider's entry reads, +// and an example standing on it named a name no request goes out under — and +// named it right after the CLI had set one, which is what the user is +// reading about. So the example stands on a model with no name of its own, +// and a provider whose every model has one is left with the pattern alone. +func TestModelWireExampleStandsOnAModelWithNoNameOfItsOwn(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + // a relay of three models, of which the first is given a name of its + // own before the provider's own name is set + if err := provider.Save(provider.Provider{ + ID: "relay-a", Name: "Relay A", Key: "k", Chat: "http://127.0.0.1:1/v1", + Models: []string{"sol", "sol-preview", "sol-low"}, + }); err != nil { + t.Fatal(err) + } + if err := modelCmd([]string{"wire", "relay-a/sol", "vendor-c/sol-preview"}); err != nil { + t.Fatal(err) + } + // the name in force for sol is the one given for sol itself, and the + // example stands for the provider's + if out := saidArgs(t, "wire", "relay-a/sol"); !strings.Contains(out, "vendor-c/sol-preview") { + t.Fatalf("magpie model wire relay-a/sol said %q; want the name given for sol itself to be the one in force", out) + } + for _, args := range [][]string{{"wire", "relay-a/*", "vendor-c/*"}, {"wire", "relay-a/*"}} { + out := saidArgs(t, args...) + if !strings.Contains(out, "relay-a/sol-preview is asked for as vendor-c/sol-preview") { + t.Errorf("%v said %q; want the example to stand on sol-preview, the first model given no name of its own", args, out) + } + if strings.Contains(out, "relay-a/sol is asked for as ") { + t.Errorf("%v said %q; the example stands on sol, which is asked for by the name given for sol itself", args, out) + } + } + // a name of its own for every model the relay shows leaves nothing to + // stand one on, and the pattern is all that can be said of it + for _, m := range []string{"sol-preview", "sol-low"} { + if err := modelCmd([]string{"wire", "relay-a/" + m, "vendor-c/" + m}); err != nil { + t.Fatal(err) + } + } + out := saidArgs(t, "wire", "relay-a/*") + if want := "Relay A is asked for every one of its models as vendor-c/*"; !strings.Contains(out, want) { + t.Errorf("magpie model wire relay-a/* said %q; want the pattern alone, %q", out, want) + } + if strings.Contains(out, "is asked for as ") { + t.Errorf("magpie model wire relay-a/* said %q; every model it shows has a name of its own, so no model stands for the provider's", out) + } +} + +// --reset on a name given for every model of a provider takes that one away +// and nothing else: a model's own key wins over the provider's, so a model +// given a name of its own is asked for by that one however the provider's +// entry reads. A tick saying every model of the provider is asked for by the +// name magpie knows it by is therefore false the moment one model keeps a +// name, and leaves the reader with the opposite of what the command did — so +// the models that keep one are named with the name in force for each of them, +// and the claim is made only where there are none. +func TestModelWireResetWildcardNamesTheModelsThatKeepAName(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + + if err := modelCmd([]string{"wire", "a/m", "vendor-c/m"}); err != nil { + t.Fatal(err) + } + if err := modelCmd([]string{"wire", "a/*", "vendor-c/one"}); err != nil { + t.Fatal(err) + } + out := saidArgs(t, "wire", "a/*", "--reset") + if strings.Contains(out, "every model of A is asked for by the name magpie knows it by again") { + t.Errorf("magpie model wire a/* --reset said %q; a/m is asked for as vendor-c/m, not by the name magpie knows it by", out) + } + if !strings.Contains(out, "m is asked for as vendor-c/m, a name of its own") { + t.Errorf("magpie model wire a/* --reset said %q; want the model that keeps a name named with the one in force for it", out) + } + // and what it says is what the provider is asked for + if got := provider.UpstreamName(mustFind(t, "a"), "m"); got != "vendor-c/m" { + t.Errorf("after --reset on every model, m is asked for as %q; want the name given for m itself", got) + } + if got := provider.UpstreamName(mustFind(t, "a"), "only-a"); got != "only-a" { + t.Errorf("after --reset on every model, only-a is asked for as %q; want the name magpie knows it by", got) + } + if provider.HasUpstreamName("a/*") { + t.Error("--reset on every model left the provider's own name behind") + } + + // two models keeping one, where the answer is the plural and has to + // name both of them and not one of them + if err := modelCmd([]string{"wire", "a/only-a", "vendor-c/only-a"}); err != nil { + t.Fatal(err) + } + if err := modelCmd([]string{"wire", "a/*", "vendor-c/two"}); err != nil { + t.Fatal(err) + } + out = saidArgs(t, "wire", "a/*", "--reset") + if !strings.Contains(out, "m, only-a are asked for as vendor-c/m, vendor-c/only-a, names of their own") { + t.Errorf("magpie model wire a/* --reset said %q; want both the models that keep a name, each with its own", out) + } + + // a name kept for a model the provider does not serve is not one of + // them: magpie asks for no such a model, so nothing goes out under + // that name, and saying it did would claim a request that is not made + if err := provider.Save(provider.Provider{ + ID: "a", Name: "A", Key: "ka", Models: []string{"claude-opus-5-5"}, Chat: "http://127.0.0.1:1/v1", + }); err != nil { + t.Fatal(err) + } + if err := modelCmd([]string{"wire", "a/*", "vendor-c/three"}); err != nil { + t.Fatal(err) + } + out = saidArgs(t, "wire", "a/*", "--reset") + if !strings.Contains(out, "every model of A is asked for by the name magpie knows it by again") { + t.Errorf("magpie model wire a/* --reset said %q; the names left are for models A no longer serves, and it is asked for none of them", out) + } + if strings.Contains(out, "vendor-c/m") || strings.Contains(out, "vendor-c/only-a") { + t.Errorf("magpie model wire a/* --reset said %q; it names models A no longer serves, which magpie never asks for", out) + } +} + +// The refusal for a pattern with nothing left under it says every model of +// the provider is asked for by the name magpie knows it by, and that is false +// the moment one model keeps a name of its own: the model's own key wins over +// the provider's, so it is asked for under that one however the provider's +// entry reads. The second --reset over a provider-wide name is exactly where +// a reader is left with the opposite of what magpie did, so the models that +// keep a name are named with the name in force for each of them, as the tick +// over the same removal already does. +func TestModelWireResetWildcardRefusalNamesTheModelsThatKeepAName(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + wires := func() map[string]string { return settings.Load().ModelWires } + + if out := saidArgs(t, "wire", "a/*", "vendor-c/*"); !strings.Contains(out, "every model of A is asked for as") { + t.Fatalf("naming every model of A said %q", out) + } + if out := saidArgs(t, "wire", "a/m", "own-m"); !strings.Contains(out, "is asked for as own-m") { + t.Fatalf("naming a/m said %q", out) + } + if out := saidArgs(t, "wire", "a/*", "--reset"); !strings.Contains(out, "m is asked for as own-m, a name of its own") { + t.Fatalf("the first --reset on every model said %q", out) + } + + out, err := refusedArgs(t, "wire", "a/*", "--reset") + if err == nil { + t.Fatalf("the second --reset on every model said %q; the provider's own name was already taken away", out) + } + if strings.Contains(out, "✓") { + t.Errorf("a --reset with nothing under the key printed a tick: %q", out) + } + if strings.Contains(err.Error(), "every model of A is already asked for by the name magpie knows it by") { + t.Errorf("the second --reset says %q; m is asked for as own-m, a name of its own", err) + } + if !strings.Contains(err.Error(), "m is asked for as own-m, a name of its own") { + t.Errorf("the second --reset says %q; want the model that keeps a name named with the one in force for it", err) + } + // and the name the refusal is about is still standing, which is the + // whole of what makes the claim it does not make false + if n := wires()["a/m"]; n != "own-m" { + t.Errorf("m's name is %q after the second --reset; want own-m", n) + } + if provider.HasUpstreamName("a/*") { + t.Error("the second --reset put a name back for every model of A") + } +} + +// A name given for a model may stand for that model with a star in it, the +// way one given for every model of a provider does. So the sentence naming +// the models a provider's own entry does not reach has to say the name the +// request goes out under, and the star in it is the model magpie knows it +// by — a star left standing there would name an id no vendor is ever asked +// for, in the one line whose whole claim is that this is the name in force. +func TestModelWireNamesTheModelsWithAStarInTheirOwnName(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + + if err := modelCmd([]string{"wire", "a/m", "vendor-c/*"}); err != nil { + t.Fatal(err) + } + if got := provider.UpstreamName(mustFind(t, "a"), "m"); got != "vendor-c/m" { + t.Fatalf("m is asked for as %q; the test is not standing on a name with a star in it", got) + } + out := saidArgs(t, "wire", "a/*", "vendor-d/*") + if !strings.Contains(out, "m is asked for as vendor-c/m, a name of its own") { + t.Errorf("magpie model wire a/* vendor-d/* said %q; want m named with the name the request goes out under", out) + } + if strings.Contains(out, "vendor-c/*") { + t.Errorf("magpie model wire a/* vendor-d/* said %q; the star in a model's own name stands for the model, and no request goes out under it", out) + } + // and the same is said of a removal, which is the other way the + // sentence is reached + out = saidArgs(t, "wire", "a/*", "--reset") + if !strings.Contains(out, "m is asked for as vendor-c/m, a name of its own") { + t.Errorf("magpie model wire a/* --reset said %q; want m named with the name the request goes out under", out) + } + if strings.Contains(out, "vendor-c/*") { + t.Errorf("magpie model wire a/* --reset said %q; the star in a model's own name stands for the model, and no request goes out under it", out) + } + + // two models keeping one, each with a star of its own, so the plural + // expands each for the model it was given for rather than one for all + if err := modelCmd([]string{"wire", "a/only-a", "vendor-e/*"}); err != nil { + t.Fatal(err) + } + out = saidArgs(t, "wire", "a/*", "vendor-d/*") + if !strings.Contains(out, "m, only-a are asked for as vendor-c/m, vendor-e/only-a, names of their own") { + t.Errorf("magpie model wire a/* vendor-d/* said %q; want both names, each with the star stood for by its own model", out) + } +} + +// A name given for every model of a provider does not reach a model that has +// one of its own, and the example such a name is shown with deliberately +// stands on a model it does reach — which is what a ✓ claiming every model of +// that provider is asked for under it leaves out, in the one case where the +// precedence between the two is the thing the user needs told. So the line +// narrows to the models with no name of their own, and the ones left alone +// are named with the name in force for each. +func TestModelWireWildcardSaysTheModelsItDoesNotReach(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + + if err := modelCmd([]string{"wire", "a/m", "vendor-c/m"}); err != nil { + t.Fatal(err) + } + out := saidArgs(t, "wire", "a/*", "vendor-c/*") + if strings.Contains(out, "· every model of A is asked for as") { + t.Errorf("magpie model wire a/* vendor-c/* said %q; a/m keeps a name of its own and is not asked for under that one", out) + } + if !strings.Contains(out, "with no name of its own is asked for as vendor-c/*") { + t.Errorf("magpie model wire a/* vendor-c/* said %q; want the claim narrowed to the models it reaches", out) + } + if !strings.Contains(out, "m is asked for as vendor-c/m, a name of its own") { + t.Errorf("magpie model wire a/* vendor-c/* said %q; want the model it does not reach named with the name in force for it", out) + } + // the example still stands on a model the name does reach, so the two + // lines are about different models and neither contradicts the other + if !strings.Contains(out, "a/only-a is asked for as vendor-c/only-a") { + t.Errorf("magpie model wire a/* vendor-c/* said %q; want the example on a model the name does reach", out) + } + // and where no model of the provider keeps a name, the claim is the + // whole of it + if out := saidArgs(t, "wire", "b/*", "vendor-c/*"); !strings.Contains(out, "every model of B is asked for as vendor-c/*") { + t.Errorf("magpie model wire b/* vendor-c/* said %q; no model of B keeps a name, so every one of them is asked for under it", out) + } +} + +// The wire names are the one thing a listing of what the user has said does +// not show: `magpie model names` is a list of what the models are called and +// of the levels they offer, both of which are magpie's own, and a relay +// serving a model under an id of its own changes neither — so a name given +// for every model of one was written with no command that read it back. +// `magpie model wires` is that listing, and it is read off the file rather +// than off a provider, since a name outlives the provider it was given for +// and is in force for whichever provider takes that id next. +func TestModelWiresCmd(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + wires := func() map[string]string { return settings.Load().ModelWires } + listed := func(t *testing.T) string { + t.Helper() + out, err := said(t, func() error { return modelCmd([]string{"wires"}) }) + if err != nil { + t.Fatal(err) + } + return out + } + + if out := listed(t); !strings.Contains(out, "no vendor is asked for a model by another name yet") { + t.Errorf("magpie model wires with nothing given said %q; want it to say there is none", out) + } + for _, ref := range [][2]string{{"a/m", "vendor-c/m"}, {"b/*", "vendor-c/one"}} { + if err := modelCmd([]string{"wire", ref[0], ref[1]}); err != nil { + t.Fatal(err) + } + } + out := listed(t) + // a name is listed with the key it is stored at, on a line of its own: + // a model's own name and the provider's are two entries, and merging + // them would read as the one in force for a model that has its own + for _, ref := range [][2]string{{"a/m", "vendor-c/m"}, {"b/*", "vendor-c/one"}} { + lines := 0 + for _, l := range strings.Split(out, "\n") { + if !strings.Contains(l, ref[0]) { + continue + } + lines++ + if !strings.Contains(l, ref[1]) { + t.Errorf("magpie model wires put %q on a line of its own, %q: %q; want the name with the key it is stored at", ref[0], ref[1], out) + } + } + if lines != 1 { + t.Errorf("magpie model wires listed %q on %d lines, %q; want it once", ref[0], lines, out) + } + } + // a name outlives the provider it was given for, so it is still listed + // once that provider is gone — it is in force for whichever provider + // takes that id next + if err := provider.Delete("b"); err != nil { + t.Fatal(err) + } + if out := listed(t); !strings.Contains(out, "b/*") || !strings.Contains(out, "vendor-c/one") { + t.Errorf("magpie model wires after b was deleted said %q; want the name it outlived listed still", out) + } + if len(wires()) != 2 { + t.Errorf("the names kept are %v; a listing takes none away", wires()) + } +} + +// A name that is only whitespace is no name, as it is where one is given and +// where one is looked up: UpstreamNames leaves it out and a request under it +// goes out by the name magpie knows the model by. So the listing reads it the +// way the file means it, one an older magpie stored or one hand-edited in, +// rather than printing a line with nothing on it — a listing whose whole +// purpose is reading the names back cannot be the one place a name shows up +// that no vendor is ever asked for by. +func TestModelWiresCmdSkipsANameOfOnlyWhitespace(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + // magpie will not write one itself — a name of only whitespace is + // taken as a removal — so it is the file that is read, as one left by + // an older magpie or edited by hand is + s := settings.Load() + s.ModelWires = map[string]string{"a/m": " vendor-c/m ", "a/only-a": " "} + if err := settings.Save(s); err != nil { + t.Fatal(err) + } + out := saidArgs(t, "wires") + listed := false + for _, l := range strings.Split(out, "\n") { + if strings.Contains(l, "only-a") { + t.Errorf("magpie model wires listed %q; a name of only whitespace is no name, and no request goes out under one", l) + } + if strings.Contains(l, "a/m") { + listed = true + // the name ends the line, so what the file stored around + // it is not in force and is not what a request goes out as + if !strings.HasSuffix(l, "vendor-c/m") { + t.Errorf("magpie model wires said %q; want the name listed as it will be sent, trimmed", out) + } + } + } + if !listed { + t.Errorf("magpie model wires said %q; want the name that is in force listed", out) + } + + // and where every name in the file is one of those, the listing says + // there is none rather than printing nothing at all + s.ModelWires = map[string]string{"a/m": " "} + if err := settings.Save(s); err != nil { + t.Fatal(err) + } + if out := saidArgs(t, "wires"); !strings.Contains(out, "no vendor is asked for a model by another name yet") { + t.Errorf("magpie model wires with nothing but whitespace names said %q; want it to say no vendor is asked for one by another name", out) + } +} + +// --reset takes a name away, and where there is no name to take away the +// model is already asked for by the name magpie knows it by, or by the name +// its provider's own entry puts in force. A tick over either would report a +// removal that did not happen — and under a provider-wide name it would +// leave the reader with the one thing that is false, that the model's own +// name is the one in force. So a removal with nothing under the key is +// refused, as `magpie model price --reset` refuses a price that was never +// set, and the refusal says which of the two the model is asked for by. An +// error here is a command that exits 1. +func TestModelWireResetRefusesANameThereIsNot(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + wires := func() map[string]string { return settings.Load().ModelWires } + + // the second --reset of a name that is gone: the first took it, so this + // one has nothing left to do + if out := saidArgs(t, "wire", "a/m", "vendor-c/m"); !strings.Contains(out, "is asked for as vendor-c/m") { + t.Fatalf("naming a/m said %q", out) + } + if out := saidArgs(t, "wire", "a/m", "--reset"); !strings.Contains(out, "is asked for as m again") { + t.Fatalf("the first --reset said %q", out) + } + out, err := refusedArgs(t, "wire", "a/m", "--reset") + if err == nil { + t.Fatalf("--reset over a name already taken away said %q; there was none to take", out) + } + if strings.Contains(out, "✓") { + t.Errorf("--reset over no name at all printed a tick: %q", out) + } + if !strings.Contains(err.Error(), "a/m has no name of its own to reset") { + t.Errorf("the second --reset says %q; want it to say a/m had no name of its own", err) + } + + // A model the provider does not serve is one magpie never asks for by + // any name, so there is none to take away — and saying which name it is + // asked for under would claim a request magpie does not make, which is + // the one thing a mistyped id must not be answered with. It is refused + // as not one of the provider's models, and the same id is refused when + // a name is given for it, which is what the README says a name is + // given only for. + out, err = refusedArgs(t, "wire", "a/nosuch", "--reset") + if err == nil { + t.Fatalf("--reset on a model A does not serve said %q; A is never asked for a nosuch", out) + } + if strings.Contains(out, "✓") { + t.Errorf("--reset on a model A does not serve printed a tick: %q", out) + } + if !strings.Contains(err.Error(), "a/nosuch is not a model of A magpie knows of") { + t.Errorf("--reset on a model A does not serve says %q; want it to say magpie knows no such model", err) + } + if strings.Contains(err.Error(), "already asked for as") { + t.Errorf("--reset on a model A does not serve says %q; that model is asked for by no name at all", err) + } + if _, ok := wires()["a/nosuch"]; ok { + t.Error("a refused --reset left a name behind") + } + if out, err := refusedArgs(t, "wire", "a/nosuch", "vendor-c/nosuch"); err == nil { + t.Errorf("a name was given for a model A does not serve: %q", out) + } + if _, ok := wires()["a/nosuch"]; ok { + t.Error("a name for a model A does not serve was stored anyway") + } + + // the same over the provider's own name, which is a name of its own + if out := saidArgs(t, "wire", "a/*", "vendor-c/one"); !strings.Contains(out, "every model of A is asked for as vendor-c/one") { + t.Fatalf("naming every model of A said %q", out) + } + if out := saidArgs(t, "wire", "a/*", "--reset"); !strings.Contains(out, "every model of A is asked for by the name magpie knows it by again") { + t.Fatalf("the first --reset on every model said %q", out) + } + out, err = refusedArgs(t, "wire", "a/*", "--reset") + if err == nil { + t.Fatalf("--reset over the provider's own name, twice, said %q", out) + } + if strings.Contains(out, "✓") { + t.Errorf("--reset over no name at all printed a tick: %q", out) + } + if !strings.Contains(err.Error(), "a/* has no name of its own to reset") || + !strings.Contains(err.Error(), "every model of A is already asked for by the name magpie knows it by") { + t.Errorf("the second --reset on every model says %q", err) + } + + // a model with only the provider's name over it: --reset reaches the + // model's own and nothing else, so the one that takes this away is the + // 'a/*' one, and it is named rather than left to a tick over a key that + // was never set + if err := provider.SetUpstreamName("a/*", "vendor-c/one"); err != nil { + t.Fatal(err) + } + out, err = refusedArgs(t, "wire", "a/m", "--reset") + if err == nil { + t.Fatalf("--reset on a model with no name of its own said %q", out) + } + if !strings.Contains(err.Error(), "a/m has no name of its own") || + !strings.Contains(err.Error(), "magpie model wire 'a/*' --reset") { + t.Errorf("the refusal is %q; want the 'a/*' --reset that takes the name in force away", err) + } + if strings.Contains(out, "✓") { + t.Errorf("--reset with only the provider's name in force printed a tick: %q", out) + } + if n := wires()["a/*"]; n != "vendor-c/one" { + t.Errorf("the provider's own name is %q; a refused --reset took it away", n) + } + // and what is in force for that model is what the refusal says, so a + // user who meant to take that name away is sent to the one command + // that does take it + if got := provider.UpstreamName(mustFind(t, "a"), "m"); got != "vendor-c/one" { + t.Errorf("with only the provider's name in force, m is asked for as %q", got) + } + // a model A does not serve is outside all of that: the provider's own + // name reaches every model of A, and a model that is not one of them is + // asked for by it no more than by a name of its own, so the answer is + // the one about magpie not knowing the model rather than the name in + // force for the others + out, err = refusedArgs(t, "wire", "a/nosuch", "--reset") + if err == nil { + t.Fatalf("--reset on a model A does not serve, under the provider's own name, said %q", out) + } + if !strings.Contains(err.Error(), "a/nosuch is not a model of A magpie knows of") || + strings.Contains(err.Error(), "vendor-c/one") { + t.Errorf("the refusal is %q; want magpie knowing no such model, not the name in force", err) + } + if out := saidArgs(t, "wire", "a/*", "--reset"); !strings.Contains(out, "every model of A is asked for by the name magpie knows it by again") { + t.Errorf("--reset on every model said %q", out) + } + if got := len(wires()); got != 0 { + t.Errorf("the names left are %v; every one of them was taken away", wires()) + } +} + +// A wire name outlives the provider it was given for: that provider can be +// deleted, and the name is then in force for whichever provider takes that +// id next, its models going out renamed to a vendor that never heard of +// them, with nothing saying so. --reset therefore works off the key the +// name is stored at rather than off a provider that is not there any more — +// the one name most in need of taking off is the one a reset that resolved +// the provider first could not take off at all. +func TestModelWireResetAfterTheProviderIsGone(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + wires := func() map[string]string { return settings.Load().ModelWires } + + if out := saidArgs(t, "wire", "b/vendor/m", "vendor-c/m"); !strings.Contains(out, "is asked for as vendor-c/m") { + t.Fatalf("naming one of b's models said %q", out) + } + if err := provider.Delete("b"); err != nil { + t.Fatal(err) + } + // deleting the provider takes the models out of the catalog, not the + // name out of the file: the id is free again, and whoever takes it next + // would be asked for its models under the old vendor's name + if n := wires()["b/vendor/m"]; n != "vendor-c/m" { + t.Fatalf("with b deleted the name under b/vendor/m is %q; the point is that one is still there", n) + } + + if out := saidArgs(t, "wire", "b/vendor/m", "--reset"); !strings.Contains(out, "b/vendor/m") || + !strings.Contains(out, "no longer has a name of its own") { + t.Errorf("--reset with the provider gone said %q", out) + } + if left := wires(); len(left) != 0 { + t.Errorf("--reset left the names behind: %v", left) + } + + // a provider that takes the id afterwards is asked for its models by + // the names magpie knows them by, which is the whole of what clearing + // the name buys + if err := provider.Save(provider.Provider{ID: "b", Name: "B", Key: "kb", + Models: []string{"vendor/m"}, Chat: "http://127.0.0.1:1/v1"}); err != nil { + t.Fatal(err) + } + if got := provider.UpstreamName(mustFind(t, "b"), "vendor/m"); got != "vendor/m" { + t.Errorf("the new b is asked for its model as %q; want vendor/m", got) + } + + // an id that is mistyped rather than a deleted provider's is still the + // error it was: there is no name under it to take away, and the user is + // told which provider is missing rather than that nothing was there + if err := modelCmd([]string{"wire", "bb/vendor/m", "--reset"}); err == nil { + t.Error("--reset on a provider that never existed was taken as a removal") + } +} + +// The id a wire name is stored under is free again once its provider is +// deleted, and another provider can be shown by it — the name a provider is +// shown by is a spelling any other provider answers to as well. So with b's +// provider gone and a provider c shown as B, `wire b/vendor/m --reset` still +// has to take the name off b/vendor/m, which is in force for whichever +// provider takes that id next, going out renamed to a vendor that never +// heard of it. Resolving the ref first lands on c, where what a reset would +// remove is c's own name — one that is really in force — and the name left +// behind is the one doing the damage, with nothing having said so. +func TestModelWireResetUnderAProviderShownAsTheGoneId(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + wires := func() map[string]string { return settings.Load().ModelWires } + + if out := saidArgs(t, "wire", "b/vendor/m", "vendor-c/m"); !strings.Contains(out, "is asked for as vendor-c/m") { + t.Fatalf("naming one of b's models said %q", out) + } + if err := provider.Delete("b"); err != nil { + t.Fatal(err) + } + // a provider whose own id is c, shown by the id b that is now nobody's, + // with a name of its own that a reset has to leave in force + if err := provider.Save(provider.Provider{ID: "c", Name: "B", Key: "kc", + Models: []string{"vendor/m"}, Chat: "http://127.0.0.1:1/v1"}); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("c/vendor/m", "vendor-d/m"); err != nil { + t.Fatal(err) + } + + if out := saidArgs(t, "wire", "b/vendor/m", "--reset"); !strings.Contains(out, "b/vendor/m") || + !strings.Contains(out, "no longer has a name of its own") { + t.Errorf("--reset with the id b shown by another provider said %q", out) + } + if _, ok := wires()["b/vendor/m"]; ok { + t.Errorf("b/vendor/m still has a name, %v; it is in force for whichever provider takes that id next", wires()) + } + if got := wires()["c/vendor/m"]; got != "vendor-d/m" { + t.Errorf("c's own name is %q; --reset took away the name c's model is asked for", got) + } +} + +// The help is two columns, and every description starts in the second of +// them — a continuation line included. A description a column early lines +// its text up under the line above rather than under the command it belongs +// to, which is how the wire lines read until they were put in line. +func TestModelUsageLinesUpItsDescriptions(t *testing.T) { + const cmd = " magpie model " + // where a description starts: past the command and the run of spaces + // after it, or — for a wrapped description — at its own first letter + startsAt := func(l string) (int, bool) { + if !strings.HasPrefix(l, cmd) { + return 0, false + } + rest := l[len(cmd):] + i := strings.Index(rest, " ") + if i < 0 { + return 0, false + } + return len(cmd) + i + len(rest[i:]) - len(strings.TrimLeft(rest[i:], " ")), true + } + // a line that is neither a command nor prose is a description wrapped + // onto a line of its own, and starts where those start + wrappedAt := func(l string) (int, bool) { + lead := len(l) - len(strings.TrimLeft(l, " ")) + return lead, strings.TrimSpace(l) != "" && lead > len(cmd) + } + + want, laid := -1, 0 + for _, l := range strings.Split(modelUsage, "\n") { + at, ok := startsAt(l) + if !ok { + at, ok = wrappedAt(l) + } + if !ok { + continue // a paragraph of prose, which is in no column + } + if want < 0 { + want = at + } + if at != want { + t.Errorf("the description on %q starts at column %d; the rest start at %d", l, at, want) + } + laid++ + } + if laid < 11 { + t.Fatalf("only %d lines of the help were laid out; the check is not reading it", laid) + } +} + +// The columns line up whatever a description belongs to, so a command with a +// description another command's line finishes is laid out perfectly and +// reads as though the two of them said the same thing: the wire lines all +// carried on into the stack under them, and which of the three each +// sentence went with was anybody's guess. What says who a wrapped line +// belongs to is that it follows its own command, so this reads the block in +// the groups a reader takes it in — a command and the lines wrapped under +// it — and fails on a group whose description stops on a word no sentence +// ends on, which is what a description whose continuation went to another +// command looks like. +// +// The gap after a command is not always two spaces: a command as long as the +// column the descriptions are laid out in leaves one, and to a check that +// counts spaces that reads as a command with no description at all. So the +// column is read off the block rather than written down, and such a line's +// description is taken from the first space past it — which is where the +// layout puts it, and which a reader reads as readily as any other line. +func TestModelUsageGivesEveryCommandItsOwnDescription(t *testing.T) { + const cmd = " magpie model " + // words a sentence cannot end on, which is what a description is left + // reading as when the line it carried on into was somebody else's. A + // preposition like "by" is not one of them: a description can end on it + // and does — the name a model goes by. + dangling := map[string]bool{ + "and": true, "or": true, "but": true, "in": true, "of": true, + "to": true, "for": true, "as": true, "with": true, "that": true, + "which": true, "a": true, "an": true, "the": true, + } + lines := strings.Split(modelUsage, "\n") + // the column the descriptions are laid out in: the leftmost one a + // command with room for a gap reaches. It is read off the block rather + // than written down, so the layout can move and still be read. + col := -1 + gapAt := func(l string) int { + i := strings.Index(l[len(cmd):], " ") + if i < 0 { + return -1 + } + k := len(cmd) + i + for k < len(l) && l[k] == ' ' { + k++ + } + return k + } + for _, l := range lines { + if !strings.HasPrefix(l, cmd) { + continue + } + if k := gapAt(l); k >= 0 && (col < 0 || k < col) { + col = k + } + } + // a command with no description beside it carries its words onto the + // next line, and that is what tells such a line from one whose command + // is as long as the column and its description is pushed past it + wrapped := func(i int) bool { + return col >= 0 && i+1 < len(lines) && strings.TrimSpace(lines[i+1]) != "" && + len(lines[i+1])-len(strings.TrimLeft(lines[i+1], " ")) == col + } + type group struct{ command, said string } + var groups []group + laid := 0 + for i, l := range lines { + switch { + case strings.HasPrefix(l, cmd): + // the command, and the description that starts past the run + // of spaces after it — the column the block is laid out in, + // or, for a command that fills that column, the one space it + // leaves before the description + g := group{command: strings.TrimSpace(l)} + rest := l[len(cmd):] + switch k := gapAt(l); { + case k >= 0: + g.command, g.said = strings.TrimSpace(rest[:k-len(cmd)]), strings.TrimSpace(l[k:]) + case col >= 0 && col < len(l) && !wrapped(i): + if k := col + strings.Index(l[col:], " "); k >= 0 { + g.command, g.said = strings.TrimSpace(rest[:k-len(cmd)]), strings.TrimSpace(l[k+1:]) + } + } + groups = append(groups, g) + laid++ + + case strings.HasPrefix(l, strings.Repeat(" ", len(cmd)+1)): + // a description wrapped onto a line of its own, which belongs + // to the command above it and to no other + if len(groups) == 0 { + t.Fatalf("a description is wrapped onto a line of its own with no command above it: %q", l) + } + groups[len(groups)-1].said += " " + strings.TrimSpace(l) + laid++ + } + } + if laid < 11 { + t.Fatalf("only %d lines of the help were read; the check is not reading it", laid) + } + for _, g := range groups { + if g.said == "" { + t.Errorf("%s is a command with no description of its own", g.command) + continue + } + last := g.said[strings.LastIndex(g.said, " ")+1:] + if dangling[last] { + t.Errorf("%s is described as %q, which stops on %q: its continuation is under another command", g.command, g.said, last) + } + } +}