From 024f494bc4e89f5c4f2cfb6b3882a343b0997fbe Mon Sep 17 00:00:00 2001 From: Comic Chang <1257169+comicchang@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:03:57 +0800 Subject: [PATCH] Allow provider-specific upstream model names (#280) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A provider may serve a model under an id of its own: namespaced with a prefix, dated, a `-latest` that is not what models.dev calls it, or simply a different spelling. magpie sends the name it knows the model by, and the provider answers that it has no such model. magpie model wire relay-b/model-2 vendor-c/model-2-preview magpie model wire 'relay-b/*' 'vendor-c/*' magpie model wire relay-b/model-2 --reset A `*` in the **value** stands for the model, so `relay-b/*` set to `vendor-c/*` sends `vendor-c/model-3` for model-3. A value with no `*` sends every model of that provider under one name, which is the case where a relay really does serve them alike. The CLI shows the pattern the way it was given, with a concrete line for one of the models, so what is in force is not something you have to imagine. An exact key wins over the provider's. The canonical id stays canonical. `Entry.Model` is not rewritten: it is the key automatic routing groups are built from, what `Resolve` returns as the model's identity, what `GET /models` lists, what an agent asks for, and what a call is recorded and priced as. The alias goes into the request's model field and nowhere else — which is the part that was wrong at first. Handing it to the serializer gave the provider's name to the code that decides what a request should look like: `buildChat` checks the model for Gemini compatibility, `buildAnthropic` matches it for adaptive-only and effort handling, `buildCodeAssist` maps by model. A provider serving a model under a namespaced id got a request shaped as if it were that other model. So the request is shaped from the model magpie knows and the field is rewritten once at the end, with `rewriteModel` — which already existed to put one provider's own model name in a body and keeps every other byte as it was. It has to reach every path. A request the provider understands as it came is relayed by `passthrough` without being rebuilt, and the Anthropic token count is asked of the provider directly; both rewrite the field too, so on a provider speaking the client's own protocol — the common case for a relay fronting one vendor — the alias still applies. "Swapped" compared the canonical id with what the vendor's reply named, so an alias that is not a prefix or a dated variant marked every call swapped. It compares the name that was actually sent, in the gateway and in the ledger alike; the ledger reads the map once for the lot, since its rows are what a reader sets beside a bill. The Codex backend path is left alone: it sends magpie's own pick and never an alias. `provider.Test` asked with the canonical id, so a provider that only accepts the alias reported a failure while its real requests worked. It asks with the alias now and still reports the canonical id, since that is the one the user picked. A blank name was a hole: the setter trimmed what it was given, so a value of only whitespace was removed on the way in, while the read path would have used it had anything else written it — a hand-edited file, or a build from before the trim. Both ends treat a blank as no name now. The key is checked where it is written rather than in Save, so one hand-edited entry cannot block every save, and #288's walk moves it on a provider rename without a line here. Co-authored-by: yetone --- README.md | 66 ++ internal/gateway/codeassist.go | 100 +- internal/gateway/gateway.go | 53 +- internal/gateway/wire_test.go | 1254 ++++++++++++++++++++++++++ internal/gui/settings_keep_test.go | 1 + internal/provider/modelprefs.go | 392 ++++++++ internal/provider/modelprefs_test.go | 883 ++++++++++++++++++ internal/provider/test.go | 11 +- internal/settings/settings.go | 9 + internal/usage/ledger.go | 38 +- internal/usage/ledger_test.go | 57 ++ model_cli.go | 328 +++++++ model_cli_test.go | 960 ++++++++++++++++++++ 13 files changed, 4103 insertions(+), 49 deletions(-) create mode 100644 internal/gateway/wire_test.go diff --git a/README.md b/README.md index 06da8c23..c2d226e0 100644 --- a/README.md +++ b/README.md @@ -283,6 +283,72 @@ at start-up. A session already running therefore keeps the window it began with, while the gateway's own `/models` and every request from then on are right at once. +### The name a vendor knows a model by + +A relay often serves a model under an id of its own — a prefix it namespaces +with, a dated name, a `-latest` that is not what models.dev calls it. Say +which name to ask for: + +```sh +magpie model wire relay-b/model-2 # what the vendor is asked for +magpie model wire relay-b/model-2 vendor-c/model-2-preview # ask for it by this +magpie model wire 'relay-b/*' 'vendor-c/*' # every model, * being the model +magpie model wire relay-b/model-2 --reset # ask for it by its own name again +magpie model wires # every name your vendors are asked for models by +``` + +Quote the arguments with a `*` in them: zsh reads a bare `*` as a glob and +answers `no matches found`. In the name, `*` stands for the model itself, so +`'relay-b/*' 'vendor-c/*'` sends `vendor-c/model-3` for `model-3` and +`vendor-c/model-2` for `model-2` — one name for a relay that namespaces its +models, each still asked for by its own. A name with no `*` in it sends every +model of that key under that one name, which is the right answer only for a +relay that does serve them all alike. The model's own key wins over the +provider's, and `--reset` takes away only the one it is given, so resetting +`relay-b/model-2` while `'relay-b/*'` is set leaves the provider's name in +force — the CLI says which of the two is in force after every change, naming +the models a name for the whole provider leaves to their own, and +`--reset` over a name that was never given says so rather than ticking a +removal that took nothing away. + +Only the request that goes out carries that name, in the model field of a +chat, Responses or Anthropic Messages request and of an Anthropic token +count; image requests are left with the name magpie knows the model by. +Gemini CLI and Antigravity sign-ins go on Code Assist, and there the model +an effort picks is a variant of the model magpie knows — `gemini-3.7-flash` +at `high` is sent as `gemini-3.7-flash-high` — so a `*` in the name is that +variant: `'antigravity/*' 'vendor-c/*'` asks for `vendor-c/gemini-3.7-flash-high` +and `vendor-c/gemini-3.7-flash-low` each by its own, as it does everywhere +else. A name with no `*` in it, or one given for `antigravity/gemini-3.7-flash` +itself, is that one name at every level. + +Everything else keeps the name magpie knows the model by: the catalog agents +pick from, the routing groups' membership, `GET /models`, and what a call is +recorded and priced as. What the vendor's own reply said answered is kept +beside that, in the ledger's `served_model`, which is what it is for, and it +is compared with the name the vendor was asked for — so a model answered +under the relay's own id is not read as a swap. That is against the names +in force when the ledger is read, since a record keeps what the vendor +answered and not the name the request went out under: naming a model after +the call re-judges that call, which is then left reading as a swap. What a +provider *supports* — whether it takes a temperature, which reasoning levels +fit — is still asked about the model magpie knows, so a rename upstream does +not change how magpie behaves towards the model. + +An upstream name is **one provider's**, not the model's: another provider +serving the same id is asked for it under its own name, or this one. The +model it is given for has to be one that provider serves — `/*` +is the way to say one for every model — because a name for a model magpie +would never ask the provider for is not a name of its own: the model is +asked for by the name magpie knows it by, and nothing anywhere would say the +name given for it is not the one in force. A relay that serves a model under +an id of its own *and* lists it does get the name; what is refused is a +model the provider's list does not have at all. The model test uses the name +too, so a relay that only knows its own ids does not report a working model +as broken. An image model is the exception that follows the rule above: it is +tested by the name magpie knows it by, on the images API and on the chat it +falls back to, because that is how a drawing is asked for. + ### Routing groups A routing group is several models, from one provider or many, that an agent diff --git a/internal/gateway/codeassist.go b/internal/gateway/codeassist.go index e4594eb6..80dba831 100644 --- a/internal/gateway/codeassist.go +++ b/internal/gateway/codeassist.go @@ -23,59 +23,77 @@ const skipSignature = "skip_thought_signature_validator" var unsafeToolID = regexp.MustCompile(`[^a-zA-Z0-9_-]`) -// antigravityVariants are the ids of Antigravity's a model magpie offers -// stands for, by level, and antigravityBaseOf the model and level one of -// those ids is; vars so tests can stand in for them. -var ( - antigravityVariants = provider.AntigravityVariants - antigravityBaseOf = provider.AntigravityBase -) +// antigravityBaseOf is the model an id of Antigravity's own is one level +// of; a var so tests can stand in for it. +var antigravityBaseOf = provider.AntigravityBase -// antigravityModelID is Antigravity's id for a model magpie offers, at -// the effort asked for: the family's variant at it, or at the nearest -// level it has; with none asked, the family's default. One of -// Antigravity's own ids at a level (gemini-3.7-flash-low, one an agent -// was set to before) goes as it is when no effort is asked, and else is -// the family's variant at the effort asked for: the effort the client -// picks wins, as it does for the family. Any other id goes as it is. -func antigravityModelID(model, effort string) string { - if base, _, ok := antigravityBaseOf(model); ok { - if effort == "" { - return model - } - model = base +// accountAgent is the agent behind p's account, and "" where p is one of the +// user's own and has none: the question a rule keyed by the account has to be +// able to answer for, and one a provider's id cannot answer — a relay added +// under a built-in's id is that relay and not that account. +func accountAgent(p provider.Provider) string { + if p.Account == nil { + return "" } - vs, ok := antigravityVariants(model) - if !ok { - return model + return p.Account.Agent +} + +// codeAssistID is the id a request on the account's app goes out under: on +// Antigravity the variant the effort picks for a model that is a family of +// levels, and elsewhere the model magpie knows. It is keyed by the account +// (accountAgent) rather than by the provider's id, since that is the account +// the variants belong to, and it is the id a ledger judges what the vendor +// answered against (provider.AntigravitySentID), so the gateway and the +// ledger read one rule rather than one each. +// The envelope is built under it and named with it, so it is worked out +// once where both of those are and handed down. +func codeAssistID(model, effort, agent string) string { + if agent == "antigravity" { + return provider.AntigravitySentID(model, effort) } - if effort == "" { - return vs[""] + return model +} + +// codeAssistBody is the envelope for a request on p's account, with the +// vendor's own name for the model in its model field, over the names in +// force (wires). It is named with the id the envelope carries, which on +// Antigravity is the variant the effort picked (codeAssistID): a name given +// for the whole provider then asks for each level by its own id instead of +// one id for every level, so the effort keeps choosing and the relay is +// asked for the model it serves at that level. +// +// The id is worked out once here and handed to the builder, since the name +// in force is looked up against the very id the envelope is addressed to: +// asking twice reads the vendor's catalogue twice over for one answer. +func codeAssistBody(p provider.Provider, r *Request, model string, wires map[string]string) []byte { + sent := codeAssistID(model, r.Effort, p.Account.Agent) + body := buildCodeAssistSent(r, sent, p.Account.Agent) + if wire := provider.UpstreamNameAsIn(wires, p.ID, model, sent); wire != sent { + body = rewriteModel(body, wire) } - var levels []string - for _, l := range effortRank { - if vs[l] != "" { - levels = append(levels, l) - } - } - if id := vs[fitEffort(effort, levels)]; id != "" { - return id - } - return vs[""] + return body } // buildCodeAssist builds the Code Assist envelope for a request, for the // app the account belongs to: "gemini" or "antigravity". On Antigravity a // model that is a family of levels is asked for as the variant the effort -// picks (antigravityModelID). +// picks (codeAssistID). func buildCodeAssist(r *Request, model, agent string) []byte { + return buildCodeAssistSent(r, codeAssistID(model, r.Effort, agent), agent) +} + +// buildCodeAssistSent is buildCodeAssist with the id the envelope is +// addressed to already worked out, for the caller that names the envelope +// with that same id and would otherwise work it out a second time. There is +// no model but that id in here: on Antigravity the two differ, and the one +// the request goes out under is the one everything below is shaped from. +func buildCodeAssistSent(r *Request, sent, agent string) []byte { ag := agent == "antigravity" fixed := false // the id says the level it thinks at if ag { - model = antigravityModelID(model, r.Effort) - _, _, fixed = antigravityBaseOf(model) + _, _, fixed = antigravityBaseOf(sent) } - claude := strings.Contains(strings.ToLower(model), "claude") + claude := strings.Contains(strings.ToLower(sent), "claude") toolID := func(id string) string { if !ag || id == "" { return id @@ -201,7 +219,7 @@ func buildCodeAssist(r *Request, model, agent string) []byte { if len(r.Stop) > 0 { gen["stopSequences"] = r.Stop } - if tc := thinkingConfig(r, model, claude, fixed); tc != nil { + if tc := thinkingConfig(r, sent, claude, fixed); tc != nil { gen["thinkingConfig"] = tc // Claude's answer has to have room past its thinking if b, ok := tc["thinkingBudget"].(int); ok && claude && gen["maxOutputTokens"] == nil { @@ -211,7 +229,7 @@ func buildCodeAssist(r *Request, model, agent string) []byte { if len(gen) > 0 { req["generationConfig"] = gen } - b, _ := json.Marshal(map[string]any{"model": model, "request": req}) + b, _ := json.Marshal(map[string]any{"model": sent, "request": req}) return b } diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index b79c5faf..27b65be0 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -63,6 +63,33 @@ func agentOf(r *http.Request) string { // "" leaves the model as asked. Set by main. var StandIn func(agent, model string) string +// wiresKey carries the upstream names in force on a request's context: read +// once where the request comes in rather than once per place a name is +// looked up, since settings.Load reads and parses the whole file every call +// and one chat request looks names up in several places — the id a try went +// out under, the body the vendor is sent, the token count asked of it. The +// names of one request are then one set, the way the ledger reads them once +// for a whole report. +type wiresKey struct{} + +// withWires carries the names in force on r's context, read at most once and +// only where something asks for one: a request refused before any name is +// looked up pays for nothing. +func withWires(r *http.Request) *http.Request { + load := sync.OnceValue(func() map[string]string { return settings.Load().ModelWires }) + return r.WithContext(context.WithValue(r.Context(), wiresKey{}, load)) +} + +// wiresOf is the names in force for ctx: the ones withWires read, or the +// ones in force now for a request magpie made for itself and never carried +// them on. +func wiresOf(ctx context.Context) map[string]string { + if load, ok := ctx.Value(wiresKey{}).(func() map[string]string); ok { + return load() + } + return settings.Load().ModelWires +} + // standIn is StandIn's model for one magpie shows no entry for: not a // catalog id or model, nor a ready provider's "provider/model". func standIn(agent, asked string) string { @@ -563,8 +590,11 @@ func (s *Server) countTokens(w http.ResponseWriter, r *http.Request) { } } } + // the names in force, read at most once however many candidates are + // counted, and not at all where there are none + wires := sync.OnceValue(func() map[string]string { return settings.Load().ModelWires }) for i, c := range counts { - res, err := s.forward(r.Context(), c.p, provider.Anthropic, "/v1/messages/count_tokens", rewriteModel(body, model), r.Header) + res, err := s.forward(r.Context(), c.p, provider.Anthropic, "/v1/messages/count_tokens", rewriteModel(body, provider.UpstreamNameIn(wires(), c.p.ID, model)), r.Header) if err != nil { if r.Context().Err() == nil { s.restAfter(c, http.StatusBadGateway, nil, []byte(err.Error())) @@ -731,6 +761,9 @@ func estimate(req *Request) int { // serve routes one parsed-enough request to its provider. func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Protocol, body []byte) { start := time.Now() + // the upstream names in force for this request, read once here rather + // than once per place a name is looked up below + r = withWires(r) // secrets go as placeholders and come back as they were; the log has // what the vendor saw and said w, body, unmask := redacted(w, body) @@ -1077,7 +1110,7 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro call.Error = refusedError(c.p, c.model, hw.failMsg) } try := Try{ID: c.rest, Model: c.model, Effort: sent, Picked: picked, Fixed: c.effort, Start: began, Done: true, Status: call.Status, Millis: time.Since(began).Milliseconds(), Error: call.Error, - Served: call.Usage.Served, Swapped: swapped(c.model, call.Usage.Served)} + Served: call.Usage.Served, Swapped: swapped(provider.SentNameOnIn(wiresOf(r.Context()), c.p.ID, accountAgent(c.p), c.model, sent), call.Usage.Served)} try.TTFT, try.FirstText = hw.first.ms() // the request's, from when it came as its ms are: the time before // this try, the ones that failed first, is in it @@ -1566,7 +1599,7 @@ func codexClientHeader(k string) bool { // written, when the provider serves the model on another of its endpoints // but not this one. func (s *Server) passthrough(w http.ResponseWriter, r *http.Request, p provider.Provider, proto provider.Protocol, model string, body []byte, u *Usage) (status int, msg string, done bool) { - body = rewriteModel(body, model) + body = rewriteModel(body, provider.UpstreamNameIn(wiresOf(r.Context()), p.ID, model)) searchFn := false // Codex's tool search sent as a function switch proto { case provider.Responses: @@ -1864,6 +1897,9 @@ func (s *Server) forwardTranslated(ctx context.Context, p provider.Provider, to r.Effort, req = e, &r } } + // the names in force, read once however many endpoints the request is + // built for below + wires := wiresOf(ctx) web := req.WebSearch // the cache key was left out to see if it was what the upstream refused dropped := false @@ -1891,7 +1927,16 @@ func (s *Server) forwardTranslated(ctx context.Context, p provider.Provider, to body = asCompletionTokens(body) } if to == provider.CodeAssist && p.Account != nil { - body = buildCodeAssist(req, model, p.Account.Agent) + // the envelope is named in there, where the id it carries is + // known: on Antigravity that is the variant the effort picked, + // not the model magpie knows + body = codeAssistBody(p, req, model, wires) + } else if wire := provider.UpstreamNameIn(wires, p.ID, model); wire != model { + // the vendor's own name for the model goes in the model field + // and nowhere else: everything above shaped the request from + // the model magpie knows, which is what those decisions are + // about + body = rewriteModel(body, wire) } res, err := s.forward(ctx, p, to, pathOf(to), p.Prepare(body), in) if err != nil || res.StatusCode < 400 { diff --git a/internal/gateway/wire_test.go b/internal/gateway/wire_test.go new file mode 100644 index 00000000..a57f9738 --- /dev/null +++ b/internal/gateway/wire_test.go @@ -0,0 +1,1254 @@ +package gateway + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/yetone/magpie/internal/catalog" + "github.com/yetone/magpie/internal/provider" + "github.com/yetone/magpie/internal/settings" + "github.com/yetone/magpie/internal/usage" +) + +// asked is what a vendor was asked for, in order, from the several +// goroutines a model test runs at once. +type asked struct { + mu sync.Mutex + models []string +} + +func (a *asked) add(model string) { + a.mu.Lock() + a.models = append(a.models, model) + a.mu.Unlock() +} + +func (a *asked) all() []string { + a.mu.Lock() + defer a.mu.Unlock() + return slices.Clone(a.models) +} + +// wire takes what the vendor was asked for since the last call and says +// whether every one of those requests named a model the way a turn does: +// by the name on the wire, and nothing else. Taking it empties the record, +// so a probe that went out under the name magpie knows can't be vouched +// for by a later probe that went out under the right one. +func (a *asked) wire(t *testing.T, what string, want []string) []string { + t.Helper() + a.mu.Lock() + got := slices.Clone(a.models) + a.models = nil + a.mu.Unlock() + if len(got) == 0 { + t.Errorf("%s asked the vendor for no model at all", what) + } + for _, m := range got { + if !slices.Contains(want, m) { + t.Errorf("%s asked the vendor for %q; a turn only ever carries %v", what, m, want) + } + } + return got +} + +// asking is a vendor that answers with the model it was asked for, as a +// relay does when it serves models under ids of its own, and records what it +// was asked for. +func asking(t *testing.T, a *asked) *httptest.Server { + t.Helper() + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/models") { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"data":[{"id":"model-2"},{"id":"model-3"}]}`) + return + } + var b struct { + Model string `json:"model"` + } + json.NewDecoder(r.Body).Decode(&b) + a.add(b.Model) + w.Header().Set("Content-Type", "text/event-stream") + io.WriteString(w, `data: {"id":"x","model":`+strconv.Quote(b.Model)+ + `,"choices":[{"index":0,"delta":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],`+ + `"usage":{"prompt_tokens":5,"completion_tokens":3}}`+"\n\n") + io.WriteString(w, "data: [DONE]\n\n") + })) + t.Cleanup(s.Close) + return s +} + +// putProvider adds a provider to the file the configured and the signed-in +// ones are merged from, for a provider whose id Save reserves and a test +// still needs magpie to be able to find. +func putProvider(t *testing.T, p map[string]any) { + t.Helper() + b, err := json.Marshal(map[string]any{"providers": []map[string]any{p}}) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(provider.Path()), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(provider.Path(), b, 0o600); err != nil { + t.Fatal(err) + } +} + +// A relay that serves a model under an id of its own is asked for it by that +// id, and by nothing else: the catalog agents see, the routing groups and the +// usage records keep the name magpie knows the model by, so a rename upstream +// is not a rename here. +func TestUpstreamNameReachesTheVendorOnly(t *testing.T) { + fresh(t) + + var seen asked + relay, other := asking(t, &seen), asking(t, &seen) + + for _, p := range []provider.Provider{ + {ID: "relay-b", Name: "Relay B", Key: "k", Models: []string{"model-2"}, Chat: relay.URL}, + {ID: "other", Name: "Other", Key: "k", Models: []string{"model-2"}, Chat: other.URL}, + } { + if err := provider.Save(p); err != nil { + t.Fatal(err) + } + } + if err := provider.SetUpstreamName("relay-b/model-2", "vendor-c/model-2-preview"); err != nil { + t.Fatal(err) + } + + s := New() + ask := func(model string) { + rec := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/v1/messages", strings.NewReader( + `{"model":"`+model+`","max_tokens":1024,"stream":true,`+ + `"messages":[{"role":"user","content":"hi"}]}`)) + s.Handler().ServeHTTP(rec, req) + if rec.Code != 200 { + t.Fatalf("%s: status %d: %s", model, rec.Code, rec.Body.String()) + } + } + + // the relay is asked for the model by the name it serves it under + ask("relay-b/model-2") + if got := seen.all(); len(got) != 1 || got[0] != "vendor-c/model-2-preview" { + t.Fatalf("the relay was asked for %v; want the upstream name", got) + } + + // another provider serving the same model id is left alone: an upstream + // name is one provider's, not the model's + ask("other/model-2") + if got := seen.all(); len(got) != 2 || got[1] != "model-2" { + t.Fatalf("the other provider was asked for %v", got[1:]) + } + + // the catalog still knows the model by magpie's name for it + found := false + for _, e := range provider.Catalog() { + if e.ID == "relay-b/model-2" { + found = true + if e.Model != "model-2" { + t.Errorf("the catalog shows the model as %q; the canonical name is what agents see", e.Model) + } + } + } + if !found { + t.Fatal("relay-b/model-2 is not in the catalog") + } + if _, model, ok := provider.Resolve("relay-b/model-2"); !ok || model != "model-2" { + t.Errorf("Resolve gave %q; the wire name is not the model's identity", model) + } + + // a call is recorded under the canonical name; what the vendor's own + // answer said it was is kept beside it, which is what that field is for + rows := usage.Load(time.Time{}) + if len(rows) == 0 { + t.Fatal("no call was recorded") + } + for _, r := range rows { + if r.Model != "model-2" { + t.Errorf("a call was recorded as %q; the vendor's name is not the model's", r.Model) + } + } + if rows[0].Served != "vendor-c/model-2-preview" { + t.Errorf("the record says the vendor served %q", rows[0].Served) + } + + // taking the user's name away asks for the model's own again + if err := provider.SetUpstreamName("relay-b/model-2", ""); err != nil { + t.Fatal(err) + } + ask("relay-b/model-2") + if got := seen.all(); got[len(got)-1] != "model-2" { + t.Fatalf("after --reset the relay was asked for %q", got[len(got)-1]) + } +} + +// A name given for a provider outlives that provider, and it outlives it by +// the id: the provider can be deleted and another take the same id, and the +// name goes on asking that one's models for theirs. So a removal has to work +// off the key the name is stored at — which is all `magpie model wire +// relay-b/model-2 --reset` has left to work off once the provider is gone — +// and here it is: with the name off, the relay that took the id is asked for +// its own model by its own id. +func TestUpstreamNameDroppedWithItsProvider(t *testing.T) { + fresh(t) + + var gone, here asked + was, now := asking(t, &gone), asking(t, &here) + + for _, p := range []provider.Provider{ + {ID: "relay-b", Name: "Relay B", Key: "k", Models: []string{"model-2"}, Chat: was.URL}, + } { + if err := provider.Save(p); err != nil { + t.Fatal(err) + } + } + if err := provider.SetUpstreamName("relay-b/model-2", "vendor-c/model-2-preview"); err != nil { + t.Fatal(err) + } + if err := provider.Delete("relay-b"); err != nil { + t.Fatal(err) + } + // the name outlived the provider that had it, and the id is free: a + // relay taking it is asked for the model under the deleted one's + // vendor's name, which is the hazard the removal is for + if err := provider.Save(provider.Provider{ID: "relay-b", Name: "Relay B", Key: "k", + Models: []string{"model-2"}, Chat: now.URL}); err != nil { + t.Fatal(err) + } + + s := New() + ask := func(model string) { + rec := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/v1/messages", strings.NewReader( + `{"model":"`+model+`","max_tokens":1024,"stream":true,`+ + `"messages":[{"role":"user","content":"hi"}]}`)) + s.Handler().ServeHTTP(rec, req) + if rec.Code != 200 { + t.Fatalf("%s: status %d: %s", model, rec.Code, rec.Body.String()) + } + } + + ask("relay-b/model-2") + if got := here.all(); len(got) != 1 || got[0] != "vendor-c/model-2-preview" { + t.Fatalf("the relay that took the id was asked for %v; want the name the deleted one had", got) + } + + if dropped, err := provider.DropUpstreamName("relay-b/model-2"); err != nil || !dropped { + t.Fatalf("DropUpstreamName = %v, %v; want the name taken off by its key", dropped, err) + } + ask("relay-b/model-2") + if got := here.all(); len(got) != 2 || got[1] != "model-2" { + t.Fatalf("with the name off the relay that took the id was asked for %v; want model-2", got[1:]) + } + if got := gone.all(); len(got) != 0 { + t.Errorf("the deleted relay's was asked for %v; nothing sends to it now", got) + } +} + +// The same removal, with a provider shown by the id whose name is gone: the +// name a provider is shown by is a spelling any other provider answers to as +// well, so a ref spelled with the id b's provider had names that provider +// rather than the one the name under it is for. Taken through that name, the +// removal would take away the other provider's own name — a name that is +// really in force — and leave the one that has this relay's models going out +// renamed to a vendor that never heard of them, for whoever takes the id. +func TestUpstreamNameDroppedUnderAProviderShownAsTheGoneId(t *testing.T) { + fresh(t) + + var here, that asked + now, other := asking(t, &here), asking(t, &that) + + if err := provider.Save(provider.Provider{ID: "b", Name: "B", Key: "k", + Models: []string{"model-2"}, Chat: now.URL}); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("b/model-2", "vendor-c/model-2-preview"); err != nil { + t.Fatal(err) + } + if err := provider.Delete("b"); err != nil { + t.Fatal(err) + } + if err := provider.Save(provider.Provider{ID: "b", Name: "B Two", Key: "k", + Models: []string{"model-2"}, Chat: now.URL}); err != nil { + t.Fatal(err) + } + + s := New() + ask := func(model string) { + rec := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/v1/messages", strings.NewReader( + `{"model":"`+model+`","max_tokens":1024,"stream":true,`+ + `"messages":[{"role":"user","content":"hi"}]}`)) + s.Handler().ServeHTTP(rec, req) + if rec.Code != 200 { + t.Fatalf("%s: status %d: %s", model, rec.Code, rec.Body.String()) + } + } + + // the id b is free again, and the relay that took it is asked for its + // model under the deleted one's vendor's name + ask("b/model-2") + if got := here.all(); len(got) != 1 || got[0] != "vendor-c/model-2-preview" { + t.Fatalf("the relay that took the id was asked for %v; want the name the deleted one had", got) + } + + // that relay goes too, leaving the name in force for the id b still, and + // a relay of an id of its own is now shown by the id b — with a name of + // its own, which is the one a removal taken through that name would take + // away instead + if err := provider.Delete("b"); err != nil { + t.Fatal(err) + } + if err := provider.Save(provider.Provider{ID: "c", Name: "B", Key: "k", + Models: []string{"model-2"}, Chat: other.URL}); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("B/model-2", "vendor-c/model-2"); err != nil { + t.Fatal(err) + } + ask("c/model-2") + if got := that.all(); len(got) != 1 || got[0] != "vendor-c/model-2" { + t.Fatalf("the relay shown as B was asked for %v; want the name given for it", got) + } + + if dropped, err := provider.DropUpstreamName("b/model-2"); err != nil || !dropped { + t.Fatalf("DropUpstreamName = %v, %v; want the name taken off by its key", dropped, err) + } + if got := settings.Load().ModelWires; len(got) != 1 || got["c/model-2"] != "vendor-c/model-2" { + t.Fatalf("the settings hold %v; want the name the deleted relay had gone, and the other left in force", got) + } + + // which is all taking that name off does to what goes out: the relay + // shown as B is still asked for its model by the name given for it, and + // the one that took the freed id b is asked for its own + ask("c/model-2") + if got := that.all(); len(got) != 2 || got[1] != "vendor-c/model-2" { + t.Fatalf("with the other name off the relay shown as B was asked for %v; want the name given for it", got[1:]) + } + if got := here.all(); len(got) != 1 { + t.Errorf("the relay that took the id was asked for %v again; nothing sends to it now", got) + } + if err := provider.Save(provider.Provider{ID: "b", Name: "B Two", Key: "k", + Models: []string{"model-2"}, Chat: now.URL}); err != nil { + t.Fatal(err) + } + ask("b/model-2") + if got := here.all(); len(got) != 2 || got[1] != "model-2" { + t.Fatalf("with the name off the relay that took the id was asked for %v; want model-2", got[1:]) + } +} + +// A name given for a provider's every model has the model in it, so each +// model is asked for by its own: one name covers a relay that namespaces its +// models, and none of them collapses into another's — which is what the +// record, and the bill read from it, would otherwise say answered. +func TestUpstreamWildcardNamesEachModel(t *testing.T) { + fresh(t) + + var seen asked + relay := asking(t, &seen) + if err := provider.Save(provider.Provider{ID: "relay-b", Name: "Relay B", Key: "k", + Models: []string{"model-2", "model-3"}, Chat: relay.URL}); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("relay-b/*", "vendor-c/sol-*"); err != nil { + t.Fatal(err) + } + + s := New() + for _, model := range []string{"model-2", "model-3"} { + rec := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/v1/messages", strings.NewReader( + `{"model":"relay-b/`+model+`","max_tokens":1024,"stream":true,`+ + `"messages":[{"role":"user","content":"hi"}]}`)) + s.Handler().ServeHTTP(rec, req) + if rec.Code != 200 { + t.Fatalf("%s: status %d: %s", model, rec.Code, rec.Body.String()) + } + } + want := []string{"vendor-c/sol-model-2", "vendor-c/sol-model-3"} + if got := seen.all(); len(got) != 2 || got[0] != want[0] || got[1] != want[1] { + t.Fatalf("the relay was asked for %v; want %v — each model by its own name", got, want) + } + + // the reply naming the model it was asked for is that model answering: + // "vendor-c/sol-model-3" is not another model than the one asked for + r := lastRoute(s) + if r.Tries[0].Model != "model-3" { + t.Errorf("the try is for %q; a wire name is not the model's identity", r.Tries[0].Model) + } + if r.Served != "vendor-c/sol-model-3" { + t.Errorf("the route says %q served", r.Served) + } + if r.Swapped || r.Tries[0].Swapped { + t.Errorf("the model answered under the relay's own id is marked swapped: %+v", r.Tries[0]) + } + // and the ledger, which reads the records back, marks it the same way + rows, _, _ := usage.Ledger(usage.Month, usage.Filter{}) + if len(rows) == 0 { + t.Fatal("the ledger has no rows") + } + for _, row := range rows { + if row.Model != "model-2" && row.Model != "model-3" { + t.Errorf("a row is for %q; a wire name is not the model's", row.Model) + } + if row.Swapped { + t.Errorf("a row is marked swapped though the vendor served what it was asked for: %+v", row) + } + } +} + +// The first test drives a request that has to be translated, because the +// relay only speaks Chat Completions. These two cover the paths that test +// cannot reach: a request the provider understands as it came, which magpie +// relays without rebuilding, and the token count asked of the vendor. +func TestUpstreamNameOnNativeAndCountPaths(t *testing.T) { + for _, tc := range []struct { + name string + path string + }{ + {"a request relayed as it came", "/v1/messages"}, + {"a token count asked of the vendor", "/v1/messages/count_tokens"}, + } { + t.Run(tc.name, func(t *testing.T) { + fresh(t) + var seen asked + // the vendor's own reply, naming the model it was asked for + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var b struct { + Model string `json:"model"` + } + json.NewDecoder(r.Body).Decode(&b) + seen.add(b.Model) + w.Header().Set("Content-Type", "application/json") + if strings.HasSuffix(r.URL.Path, "count_tokens") { + io.WriteString(w, `{"input_tokens":7}`) + return + } + io.WriteString(w, `{"id":"x","model":`+strconv.Quote(b.Model)+ + `,"content":[{"type":"text","text":"hi"}],`+ + `"usage":{"input_tokens":5,"output_tokens":3}}`) + })) + t.Cleanup(up.Close) + // an Anthropic base, so /v1/messages is the provider's own + // protocol and the request is relayed rather than rebuilt + if err := provider.Save(provider.Provider{ + ID: "relay", Name: "Relay", Key: "k", + Models: []string{"sol"}, + Anthropic: up.URL, + }); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("relay/sol", "vendor-c/sol-2"); err != nil { + t.Fatal(err) + } + + s := New() + rec := httptest.NewRecorder() + req := httptest.NewRequest("POST", tc.path, strings.NewReader( + `{"model":"relay/sol","max_tokens":64,`+ + `"messages":[{"role":"user","content":"hi"}]}`)) + s.Handler().ServeHTTP(rec, req) + if rec.Code != 200 { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + if got := seen.all(); len(got) != 1 || got[0] != "vendor-c/sol-2" { + t.Fatalf("the vendor was asked for %v; want the upstream name", got) + } + // "vendor-c/sol-2" is the same model as "sol", and a name that + // is not a prefix or a dated version of it is exactly what a + // comparison with magpie's own name would get wrong + if tc.path == "/v1/messages" { + if r := lastRoute(s); r.Served != "vendor-c/sol-2" || r.Swapped { + t.Errorf("the model answered under the relay's own id is marked swapped: %+v", r) + } + } + }) + } +} + +// The lookup is the model's own name, then the provider's "*", then the +// canonical id — and --reset removes only the first, leaving a provider-wide +// name in force. The CLI says so; this is where it is true. +func TestUpstreamNamePrecedence(t *testing.T) { + fresh(t) + if err := provider.Save(provider.Provider{ID: "relay", Name: "Relay", Key: "k", + Chat: "https://relay.example/v1", Models: []string{"sol", "other"}}); err != nil { + t.Fatal(err) + } + p, err := provider.Find("relay") + if err != nil { + t.Fatal(err) + } + + // with nothing set, the vendor is asked for the model magpie knows + if got := provider.UpstreamName(*p, "sol"); got != "sol" { + t.Errorf("unset: %q; want the canonical id", got) + } + + // a name given for every model of the provider covers a model that has + // none of its own, with each model in it + if err := provider.SetUpstreamName("relay/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + for _, m := range []string{"sol", "other"} { + if got := provider.UpstreamName(*p, m); got != "vendor-c/"+m { + t.Errorf("%s under the provider's *: %q; want vendor-c/%s", m, got, m) + } + } + + // a name with no model in it is every model's own name under that one + if err := provider.SetUpstreamName("relay/*", "vendor-c/one"); err != nil { + t.Fatal(err) + } + for _, m := range []string{"sol", "other"} { + if got := provider.UpstreamName(*p, m); got != "vendor-c/one" { + t.Errorf("%s under a name with no *: %q; want vendor-c/one", m, got) + } + } + + // the model's own wins + if err := provider.SetUpstreamName("relay/sol", "vendor-c/sol-exact"); err != nil { + t.Fatal(err) + } + if got := provider.UpstreamName(*p, "sol"); got != "vendor-c/sol-exact" { + t.Errorf("exact: %q; want vendor-c/sol-exact", got) + } + if got := provider.UpstreamName(*p, "other"); got != "vendor-c/one" { + t.Errorf("other: %q; want the provider's vendor-c/one", got) + } + + // taking the model's away leaves the provider's in force + if err := provider.SetUpstreamName("relay/sol", ""); err != nil { + t.Fatal(err) + } + if got := provider.UpstreamName(*p, "sol"); got != "vendor-c/one" { + t.Errorf("after --reset: %q; want the provider's vendor-c/one still in force", got) + } + if err := provider.SetUpstreamName("relay/*", ""); err != nil { + t.Fatal(err) + } + if got := provider.UpstreamName(*p, "sol"); got != "sol" { + t.Errorf("after both are reset: %q; want the canonical id", got) + } + // what was given is kept as it was given, the "*" and all, for a list of + // them to show + if err := provider.SetUpstreamName("relay/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + if got := p.UpstreamNames()["*"]; got != "vendor-c/*" { + t.Errorf("the name given is %q; want it as it was given", got) + } +} + +// A wire name keyed by something no lookup will match is refused where it is +// given rather than saved as a silent no-op, and one that is only whitespace +// is the user taking their name away, not a stored empty one. +func TestUpstreamNameRejectsBadKeys(t *testing.T) { + fresh(t) + if err := provider.Save(provider.Provider{ID: "relay", Name: "Relay", Key: "k", + Chat: "https://relay.example/v1", Models: []string{"sol"}}); err != nil { + t.Fatal(err) + } + for _, key := range []string{"noslash", "/sol", "Not A Provider/sol", "relay/"} { + if err := provider.SetUpstreamName(key, "vendor-c/sol"); err == nil { + t.Errorf("the key %q was accepted", key) + } + } + // a key hand-edited into the file is that entry's own no-op, and must + // not keep every other setting from being saved + s := settings.Load() + if s.ModelWires == nil { + s.ModelWires = map[string]string{} + } + s.ModelWires["noslash"] = "vendor-c/sol" + if err := settings.Save(s); err != nil { + t.Fatalf("a wire name keyed badly by hand stopped the settings being saved: %v", err) + } + if err := provider.SetUpstreamName("relay/sol", " "); err != nil { + t.Errorf("a whitespace-only name should remove the override, not fail: %v", err) + } +} + +// A provider given another id takes the names the user gave its models with +// it: a model asked for by the relay's own id would be asked for by magpie's +// again, and the relay would be asked for a model it does not have. +func TestUpstreamNamesFollowTheProviderRename(t *testing.T) { + fresh(t) + if err := provider.Save(provider.Provider{ID: "relay-b", Name: "Relay B", Key: "k", + Chat: "https://relay.example/v1", Models: []string{"model-2", "model-3"}}); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("relay-b/model-2", "vendor-c/model-2-preview"); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("relay-b/*", "vendor-c/sol-*"); err != nil { + t.Fatal(err) + } + if err := provider.Rename("relay-b", "relay-c"); err != nil { + t.Fatal(err) + } + p, err := provider.Find("relay-c") + if err != nil { + t.Fatal(err) + } + if got := provider.UpstreamName(*p, "model-2"); got != "vendor-c/model-2-preview" { + t.Errorf("the model's own name is %q after the rename", got) + } + if got := provider.UpstreamName(*p, "model-3"); got != "vendor-c/sol-model-3" { + t.Errorf("the provider's name is %q after the rename; it follows the id", got) + } +} + +// A relay that only knows its own ids would fail a test naming the model +// magpie knows it by, though every real turn reaches it: the provider's own +// tests ask for the model the way a request does. Each of the two probes is +// held to that on its own, on what it asked the relay for while it was the +// only one running — a provider test asking for magpie's name and a model +// test asking for the relay's, read together, both look right. +func TestProviderTestsAskTheWireName(t *testing.T) { + fresh(t) + var seen asked + relay := asking(t, &seen) + if err := provider.Save(provider.Provider{ID: "relay-b", Name: "Relay B", Key: "k", + Models: []string{"model-2", "model-3"}, Chat: relay.URL}); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("relay-b/*", "vendor-c/sol-*"); err != nil { + t.Fatal(err) + } + p, err := provider.Find("relay-b") + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + // the two names the relay serves these models by + wire := []string{"vendor-c/sol-model-2", "vendor-c/sol-model-3"} + + results := p.Test(ctx) + if len(results) == 0 { + t.Fatal("the provider was not tested at all") + } + for _, r := range results { + if !r.OK { + t.Errorf("the provider's own test failed: %+v", r) + } + } + seen.wire(t, "the provider's own test", wire) + + for _, r := range p.TestModels(ctx, []string{"model-2", "model-3"}) { + if !r.OK { + t.Errorf("testing %q failed: %+v", r.Model, r) + } + } + // each model by its own wire name, and both of them asked for + if got := seen.wire(t, "testing the models", wire); len(got) != len(wire) { + t.Errorf("testing the models asked the relay for %v; want one request per model", got) + } + // the model reported is the one magpie knows, not the relay's own id + for _, r := range p.TestModels(ctx, []string{"model-2"}) { + if r.Model != "model-2" { + t.Errorf("a test reports the model as %q; the canonical id is what the user picked", r.Model) + } + } +} + +// A drawing is the one request that keeps the name magpie knows the model by: +// the gateway builds those bodies itself, on the images API or on chat with +// modalities, and no upstream name is written into one. A provider test that +// asked for the vendor's own name would be testing a request no turn ever +// makes, and would report a working drawing model as broken. +func TestProviderDrawingTestsAskTheCanonicalName(t *testing.T) { + fresh(t) + var seen asked + relay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var b struct { + Model string `json:"model"` + } + json.NewDecoder(r.Body).Decode(&b) + seen.add(b.Model) + if strings.HasSuffix(r.URL.Path, "/images/generations") { + // not served here, so the chat fallback is taken as the + // gateway takes it + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.Header().Set("Content-Type", "text/event-stream") + io.WriteString(w, `data: {"id":"x","model":`+strconv.Quote(b.Model)+ + `,"choices":[{"index":0,"delta":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],`+ + `"usage":{"prompt_tokens":5,"completion_tokens":3}}`+"\n\n") + io.WriteString(w, "data: [DONE]\n\n") + })) + t.Cleanup(relay.Close) + if err := provider.Save(provider.Provider{ID: "relay-b", Name: "Relay B", Key: "k", + Models: []string{"imagine-2"}, Chat: relay.URL}); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("relay-b/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + p, err := provider.Find("relay-b") + if err != nil { + t.Fatal(err) + } + for _, r := range p.TestModels(context.Background(), []string{"imagine-2"}) { + if !r.OK { + t.Fatalf("testing the drawing model failed: %+v", r) + } + } + // the images request and the chat one it fell back to, neither carrying + // the upstream name + if got := seen.all(); len(got) != 2 || got[0] != "imagine-2" || got[1] != "imagine-2" { + t.Fatalf("the relay was asked for %v; a drawing goes out under the name magpie knows the model by", got) + } +} + +// A name that is only whitespace is the user taking their name away, not one +// to be kept: it is a legal removal, so it must not fail, and it must leave +// nothing behind — no key in the settings, and the model asked for by the +// name magpie knows it by. A wire name that is a space in the file would be +// sent to the vendor as the model, and a list of them would show it as one +// the user gave. +func TestUpstreamNameBlankIsAReset(t *testing.T) { + fresh(t) + if err := provider.Save(provider.Provider{ID: "relay", Name: "Relay", Key: "k", + Chat: "https://relay.example/v1", Models: []string{"sol", "other"}}); err != nil { + t.Fatal(err) + } + p, err := provider.Find("relay") + if err != nil { + t.Fatal(err) + } + + if err := provider.SetUpstreamName("relay/sol", "vendor-c/sol"); err != nil { + t.Fatal(err) + } + if got := provider.UpstreamName(*p, "sol"); got != "vendor-c/sol" { + t.Fatalf("before the reset: %q; want vendor-c/sol", got) + } + + // blank takes a name that was given away, and is not an error + if err := provider.SetUpstreamName("relay/sol", " "); err != nil { + t.Fatalf("a whitespace-only name is the user taking theirs away, not a bad input: %v", err) + } + if n, ok := settings.Load().ModelWires["relay/sol"]; ok { + t.Errorf("the settings kept a wire name of %q for relay/sol", n) + } + if got := provider.UpstreamName(*p, "sol"); got != "sol" { + t.Errorf("after a blank name: %q; want the canonical id", got) + } + if _, ok := p.UpstreamNames()["sol"]; ok { + t.Error("a name taken away is still listed as one the user gave") + } + + // and blank for a key there is nothing under writes no key at all + if err := provider.SetUpstreamName("relay/other", "\t\n "); err != nil { + t.Fatalf("a whitespace-only name is the user taking theirs away, not a bad input: %v", err) + } + if n, ok := settings.Load().ModelWires["relay/other"]; ok { + t.Errorf("a blank name wrote %q for relay/other", n) + } + if got := provider.UpstreamName(*p, "other"); got != "other" { + t.Errorf("after a blank name: %q; want the canonical id", got) + } + if got := p.UpstreamNames(); len(got) != 0 { + t.Errorf("the names given are %v; a blank one is none", got) + } + + // a provider-wide name in force is what a model's own blank name falls + // back to, not a blank name of its own + if err := provider.SetUpstreamName("relay/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("relay/sol", " "); err != nil { + t.Fatalf("a whitespace-only name is the user taking theirs away, not a bad input: %v", err) + } + if got := provider.UpstreamName(*p, "sol"); got != "vendor-c/sol" { + t.Errorf("with the provider's name in force: %q; want vendor-c/sol", got) + } +} + +// A name left blank in the file — by an older magpie, or by hand — is no +// name: the model is asked for by the name magpie knows it by, and a list of +// them does not show a blank as one the user gave. It is the same thing a +// blank name is where it is set, and the file is no more exempt. +func TestUpstreamNameBlankInTheFileIsNoName(t *testing.T) { + fresh(t) + if err := provider.Save(provider.Provider{ID: "relay", Name: "Relay", Key: "k", + Chat: "https://relay.example/v1", Models: []string{"sol", "other"}}); err != nil { + t.Fatal(err) + } + p, err := provider.Find("relay") + if err != nil { + t.Fatal(err) + } + + s := settings.Load() + s.ModelWires = map[string]string{ + "relay/sol": " ", + "relay/other": "vendor-c/other", + } + if err := settings.Save(s); err != nil { + t.Fatal(err) + } + + if got := provider.UpstreamName(*p, "sol"); got != "sol" { + t.Errorf("a blank name in the file: %q; want the canonical id", got) + } + if got := provider.UpstreamName(*p, "other"); got != "vendor-c/other" { + t.Errorf("a name beside a blank one: %q; want vendor-c/other", got) + } + names := p.UpstreamNames() + if _, ok := names["sol"]; ok { + t.Errorf("a blank name in the file is listed as one given: %v", names) + } + if names["other"] != "vendor-c/other" { + t.Errorf("the names given are %v; a blank one must not hide the rest", names) + } + + // the same for the provider-wide key, which covers the models a blank + // model's own name does not reach + s = settings.Load() + s.ModelWires["relay/*"] = " " + if err := settings.Save(s); err != nil { + t.Fatal(err) + } + if got := provider.UpstreamName(*p, "sol"); got != "sol" { + t.Errorf("a blank name in the file: %q; want the canonical id", got) + } + if got := p.UpstreamNames(); len(got) != 1 || got["other"] != "vendor-c/other" { + t.Errorf("the names given are %v; a blank one is none", got) + } +} + +// Every "*" in a wire name is the model, wherever it stands and however many +// there are — the rule the settings' own comment, the CLI help and the README +// all give, and what the lookup does. A name with none is that one name for +// every model of the key, which is what a query lists back as it was given, +// "*" and all, rather than a name with one filled in. +func TestUpstreamNameWildcardStandsWhereverItStands(t *testing.T) { + fresh(t) + if err := provider.Save(provider.Provider{ID: "relay", Name: "Relay", Key: "k", + Chat: "https://relay.example/v1", Models: []string{"sol", "other"}}); err != nil { + t.Fatal(err) + } + p, err := provider.Find("relay") + if err != nil { + t.Fatal(err) + } + + for _, tc := range []struct { + wires map[string]string + model string + want string + why string + }{ + {map[string]string{"relay/sol": "vendor-c/*"}, "sol", "vendor-c/sol", "a trailing *"}, + {map[string]string{"relay/sol": "vendor/*/pro"}, "sol", "vendor/sol/pro", "a * in the middle"}, + {map[string]string{"relay/sol": "*-pro"}, "sol", "sol-pro", "a leading *"}, + {map[string]string{"relay/sol": "a/*-b/*"}, "sol", "a/sol-b/sol", "two *"}, + {map[string]string{"relay/sol": "*"}, "sol", "sol", "nothing but a *"}, + {map[string]string{"relay/*": "vendor/*/pro"}, "sol", "vendor/sol/pro", "the provider's, mid-name"}, + {map[string]string{"relay/*": "vendor-c/one"}, "sol", "vendor-c/one", "no * at all"}, + {map[string]string{"relay/*": "vendor-c/one"}, "other", "vendor-c/one", "no *, other model"}, + {map[string]string{"relay/sol": "vendor-c/sol"}, "other", "other", "another provider's model"}, + } { + if got := provider.UpstreamNameIn(tc.wires, "relay", tc.model); got != tc.want { + t.Errorf("%s: %q; want %q", tc.why, got, tc.want) + } + } + + // what a query shows is what was given, "*" and all, so a name given for + // every model can be read as what it sends rather than as one model's + if err := provider.SetUpstreamName("relay/*", "vendor/*/pro"); err != nil { + t.Fatal(err) + } + if got := p.UpstreamNames()["*"]; got != "vendor/*/pro" { + t.Errorf("the name given for every model is listed as %q; want it as it was given", got) + } + for _, m := range []string{"sol", "other"} { + if got := provider.UpstreamName(*p, m); got != "vendor/"+m+"/pro" { + t.Errorf("%s under the provider's name: %q; want vendor/%s/pro", m, got, m) + } + } + + // a name with no "*" in it is that one name for every model of the key, + // and is listed as it was given too + if err := provider.SetUpstreamName("relay/*", "vendor-c/one"); err != nil { + t.Fatal(err) + } + if got := p.UpstreamNames()["*"]; got != "vendor-c/one" { + t.Errorf("the name given for every model is listed as %q; want vendor-c/one", got) + } + for _, m := range []string{"sol", "other"} { + if got := provider.UpstreamName(*p, m); got != "vendor-c/one" { + t.Errorf("%s under a name with no *: %q; want vendor-c/one", m, got) + } + } + + // and a single model's own name, wildcard and all, beside the + // provider's + if err := provider.SetUpstreamName("relay/sol", "vendor-c/sol-*"); err != nil { + t.Fatal(err) + } + names := p.UpstreamNames() + if names["sol"] != "vendor-c/sol-*" || names["*"] != "vendor-c/one" { + t.Errorf("the names given are %v; each is listed as it was given", names) + } + if got := provider.UpstreamName(*p, "sol"); got != "vendor-c/sol-sol" { + t.Errorf("the model's own: %q; want vendor-c/sol-sol", got) + } + if got := provider.UpstreamName(*p, "other"); got != "vendor-c/one" { + t.Errorf("the other model: %q; want the provider's vendor-c/one", got) + } +} + +// A name given for a provider's every model is not one id for all of them: an +// Antigravity account is asked for the variant its effort picks, and the name +// goes with that variant. Filling the "*" in with the model magpie knows +// would send one id whatever level is asked for, so the effort stops choosing +// and a relay that serves gemini-3.7-flash-high is asked for the base id it +// has no answer for. +func TestUpstreamNameFollowsTheAntigravityVariant(t *testing.T) { + fresh(t) + var raw []catalog.Model + for _, l := range []string{"gemini-3.7-flash-high - Gemini 3.7 Flash (High)", "gemini-3.7-flash-low - Gemini 3.7 Flash (Low)", + "gemini-3.7-flash-medium - Gemini 3.7 Flash (Medium)"} { + id, name, _ := strings.Cut(l, " - ") + raw = append(raw, catalog.Model{ID: id, Name: name}) + } + if err := catalog.SaveLive("antigravity", "", raw); err != nil { + t.Fatal(err) + } + // a wire name is given for a model the provider really serves, so + // magpie must be able to find the provider that serves it. The account + // is built at runtime and its id is reserved (Save refuses it), so the + // test registers the provider under that id in the file directly: the + // model a name is keyed by is the family the live list stands for. + putProvider(t, map[string]any{ + "id": "antigravity", "name": "Antigravity", "key": "k", + "chat": "https://antigravity.test/v1", + }) + if err := provider.SetUpstreamName("antigravity/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + ask := func(agent, model, effort string) string { + t.Helper() + p := provider.Provider{ID: "antigravity", Name: "Antigravity", Account: &provider.Account{Agent: agent}} + r := codeAssistRequest(t) + r.Thinking, r.Effort = false, effort + return modelOf(codeAssistBody(p, r, model, settings.Load().ModelWires)) + } + + // the level the effort picked is in the name it is asked for by + for _, c := range []struct{ effort, want string }{ + {"high", "vendor-c/gemini-3.7-flash-high"}, + {"medium", "vendor-c/gemini-3.7-flash-medium"}, + {"low", "vendor-c/gemini-3.7-flash-low"}, + {"", "vendor-c/gemini-3.7-flash-high"}, + } { + if got := ask("antigravity", "gemini-3.7-flash", c.effort); got != c.want { + t.Errorf("at %q the vendor is asked for %q; want %q — each level by its own name", c.effort, got, c.want) + } + } + + // a name given for the model itself is that model's, whichever variant + // answers for it: the exact key still wins over the provider's + if err := provider.SetUpstreamName("antigravity/gemini-3.7-flash", "vendor-c/flash"); err != nil { + t.Fatal(err) + } + for _, effort := range []string{"high", "low"} { + if got := ask("antigravity", "gemini-3.7-flash", effort); got != "vendor-c/flash" { + t.Errorf("the model's own name at %q: %q; want vendor-c/flash", effort, got) + } + } + + // a Gemini CLI account has no variants, so the model magpie knows is + // what is asked for, under the provider's name + if err := provider.SetUpstreamName("antigravity/gemini-3.7-flash", ""); err != nil { + t.Fatal(err) + } + if got := ask("gemini", "gemini-3.7-flash", "high"); got != "vendor-c/gemini-3.7-flash" { + t.Errorf("a Gemini CLI account: %q; want vendor-c/gemini-3.7-flash", got) + } + + // and with no name given, the envelope goes out as it was built + if err := provider.SetUpstreamName("antigravity/*", ""); err != nil { + t.Fatal(err) + } + if got := ask("antigravity", "gemini-3.7-flash", "low"); got != "gemini-3.7-flash-low" { + t.Errorf("with nothing given: %q; want the variant as it is", got) + } +} + +// A turn's route and its ledger row are two views of the one call, so they +// have to judge it against one name: on an Antigravity account the call +// goes out under the variant of the model the effort picks, so a relay that +// answers with that very id has not swapped anything — and one that answers +// with another level's id has, whichever of the two the reader is looking +// at. The route read the model magpie knows where the ledger read the id +// that went out, and told the user two different things about the same +// reply. +// +// The provider here is one of the user's own under an id of its own, which is +// the shape the two views agree on by rule and not by coincidence: a relay's +// id names no account, so neither view reaches for a variant, and both read +// the reply back by the one name that went out. An Antigravity account's +// provider is not a shape a test can build here — it speaks Code Assist, +// which magpie cannot point at a relay — and the rule keyed by the account is +// read in internal/provider, where the account is what a caller hands in. +func TestUpstreamNameReadTheSameByTheRouteAndTheLedger(t *testing.T) { + fresh(t) + var raw []catalog.Model + for _, l := range []string{"gemini-3.7-flash-high - High", "gemini-3.7-flash-low - Low", "gemini-3.7-flash-medium - Medium"} { + id, name, _ := strings.Cut(l, " - ") + raw = append(raw, catalog.Model{ID: id, Name: name}) + } + if err := catalog.SaveLive("antigravity", "", raw); err != nil { + t.Fatal(err) + } + // a relay that answers with an id of its own choosing, the way a vendor + // serves a model under a name of its own and says which answered + answers := "" + relay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var b struct { + Model string `json:"model"` + } + json.NewDecoder(r.Body).Decode(&b) + w.Header().Set("Content-Type", "text/event-stream") + io.WriteString(w, `data: {"id":"x","model":`+strconv.Quote(answers)+ + `,"choices":[{"index":0,"delta":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],`+ + `"usage":{"prompt_tokens":5,"completion_tokens":3}}`+"\n\n") + io.WriteString(w, "data: [DONE]\n\n") + })) + t.Cleanup(relay.Close) + putProvider(t, map[string]any{ + "id": "ag-relay", "name": "AG Relay", "key": "k", "chat": relay.URL, + "models": []string{"gemini-3.7-flash"}, + }) + if err := provider.SetUpstreamName("ag-relay/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + s := New() + turn := func() (Route, usage.Row) { + t.Helper() + rec := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/v1/messages", strings.NewReader( + `{"model":"ag-relay/gemini-3.7-flash","max_tokens":1024,"stream":true,`+ + `"thinking":{"type":"enabled","budget_tokens":24000},`+ + `"messages":[{"role":"user","content":"hi"}]}`)) + s.Handler().ServeHTTP(rec, req) + if rec.Code != 200 { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + rows, _, _ := usage.Ledger(usage.All, usage.Filter{Query: "gemini-3.7-flash"}) + if len(rows) == 0 { + t.Fatal("the ledger holds no row for the turn") + } + return lastRoute(s), rows[0] // newest first: the turn just made + } + + // the id the call went out under is the one the wire gave it, so a relay + // answering with that very id has swapped nothing — in the route and in + // the ledger alike + answers = "vendor-c/gemini-3.7-flash" + route, row := turn() + if route.Swapped || row.Swapped { + t.Errorf("the relay answered with the id the call went out under, which is no swap: the route says %v, the ledger %v", route.Swapped, row.Swapped) + } + + // another model's id is another model, and both views say so + answers = "vendor-c/gemini-3.7-flash-high" + route, row = turn() + if !route.Swapped || !row.Swapped { + t.Errorf("the relay answered with another level's id: route swapped %v, ledger %v", route.Swapped, row.Swapped) + } +} + +// A route reads a turn's reply against the name the call really went out +// under, and on an Antigravity account that name is the variant of the model +// the effort picked — so the account is the question, not the provider's id. +// The two are one id today because an account provider is built with its +// account's own id, and a test that builds a provider of an account's own id +// with no account behind it says nothing about which of the two the rule is +// keyed by: it is the shape where they come apart, and the relay here is +// asked for the model magpie knows, so no level of the family is what the +// route has to read the reply back by. +// +// Only the route is asserted: a record keeps the provider id and no account, +// and reads itself back by that id (SentNameIn), which is the whole of what +// the ledger has to work from. +func TestRouteJudgesTheUpstreamNameByTheAccountAndNotByTheProvidersId(t *testing.T) { + fresh(t) + var raw []catalog.Model + for _, l := range []string{"gemini-3.7-flash-high - High", "gemini-3.7-flash-low - Low", "gemini-3.7-flash-medium - Medium"} { + id, name, _ := strings.Cut(l, " - ") + raw = append(raw, catalog.Model{ID: id, Name: name}) + } + if err := catalog.SaveLive("antigravity", "", raw); err != nil { + t.Fatal(err) + } + answers := "" + relay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + io.WriteString(w, `data: {"id":"x","model":`+strconv.Quote(answers)+ + `,"choices":[{"index":0,"delta":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],`+ + `"usage":{"prompt_tokens":5,"completion_tokens":3}}`+"\n\n") + io.WriteString(w, "data: [DONE]\n\n") + })) + t.Cleanup(relay.Close) + putProvider(t, map[string]any{ + "id": "antigravity", "name": "Antigravity", "key": "k", "chat": relay.URL, + "models": []string{"gemini-3.7-flash"}, + }) + if err := provider.SetUpstreamName("antigravity/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + s := New() + turn := func() Route { + t.Helper() + rec := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/v1/messages", strings.NewReader( + `{"model":"antigravity/gemini-3.7-flash","max_tokens":1024,"stream":true,`+ + `"thinking":{"type":"enabled","budget_tokens":24000},`+ + `"messages":[{"role":"user","content":"hi"}]}`)) + s.Handler().ServeHTTP(rec, req) + if rec.Code != 200 { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + return lastRoute(s) + } + + // the relay was asked for the model under its provider's own name, and + // answered with that id: nothing swapped + answers = "vendor-c/gemini-3.7-flash" + if r := turn(); r.Swapped { + t.Errorf("the relay answered with the id it was asked for, which is no swap: the route says %v", r.Swapped) + } + + // a level of the family is another model, and the route reads it as one + answers = "vendor-c/gemini-3.7-flash-high" + if r := turn(); !r.Swapped { + t.Errorf("the relay answered with a level's id: the route says swapped %v", r.Swapped) + } +} + +// The id an Antigravity envelope is named with is the id a ledger reads the +// vendor's reply back against, so it is one rule and not two: a model of the +// family and one of the family's own variant ids — an id a pick or an +// agent's model list kept from before the families were one model — each go +// out at, and are judged against, the same id at every level. +func TestAntigravityEnvelopeIsNamedAsTheLedgerJudgesIt(t *testing.T) { + fresh(t) + var raw []catalog.Model + for _, l := range []string{"gemini-3.7-flash-high - High", "gemini-3.7-flash-low - Low", "gemini-3.7-flash-medium - Medium"} { + id, name, _ := strings.Cut(l, " - ") + raw = append(raw, catalog.Model{ID: id, Name: name}) + } + if err := catalog.SaveLive("antigravity", "", raw); err != nil { + t.Fatal(err) + } + for _, model := range []string{"gemini-3.7-flash", "gemini-3.7-flash-high", "gemini-3.7-flash-low"} { + for _, effort := range []string{"high", "medium", "low", "minimal", "xhigh", ""} { + sent := codeAssistID(model, effort, "antigravity") + judged := provider.SentNameIn(nil, "antigravity", model, effort) + if sent != judged { + t.Errorf("%s at %q goes out as %q and is judged against %q", model, effort, sent, judged) + } + if usage.Swapped(sent, sent) { + t.Errorf("%s at %q: the id the call goes out under is read as another model answering", model, effort) + } + } + } + // and a Gemini CLI account is asked for the model magpie knows, which is + // what its ledger row reads back too + if got := codeAssistID("gemini-3.7-flash", "high", "gemini"); got != "gemini-3.7-flash" { + t.Errorf("a Gemini CLI account is asked for %q; want the model magpie knows", got) + } +} + +// A turn reads the upstream names once, wherever in it they are looked up: +// the id a try goes out under, the body the vendor is sent, the token count +// asked of it. settings.Load reads and parses the whole file every call, so +// a name looked up per place made every turn pay for the settings once per +// place — a turn that named no model at all paying just the same. Reading +// them once also keeps a turn to the names that stood when it came in, so a +// name given while it is on its way is for the turns after it and not for +// the one in the air. +func TestUpstreamNamesAreReadOnceInATurn(t *testing.T) { + fresh(t) + var seen []string + named := false + // a relay that is given a name for the model it serves while a turn is + // on its way, and answers with the model it was asked for, as a relay + // serving a model under an id of its own does + relay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var b struct { + Model string `json:"model"` + } + json.NewDecoder(r.Body).Decode(&b) + seen = append(seen, b.Model) + if !named { + named = true + st := settings.Load() + st.ModelWires = map[string]string{"relay/model-2": "vendor-c/model-2-renamed"} + if err := settings.Save(st); err != nil { + t.Error(err) + } + } + w.Header().Set("Content-Type", "text/event-stream") + io.WriteString(w, `data: {"id":"x","model":`+strconv.Quote(b.Model)+ + `,"choices":[{"index":0,"delta":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],`+ + `"usage":{"prompt_tokens":5,"completion_tokens":3}}`+"\n\n") + io.WriteString(w, "data: [DONE]\n\n") + })) + t.Cleanup(relay.Close) + if err := provider.Save(provider.Provider{ID: "relay", Name: "Relay", Key: "k", + Models: []string{"model-2"}, Anthropic: relay.URL}); err != nil { + t.Fatal(err) + } + s := New() + turn := func() Route { + t.Helper() + rec := httptest.NewRecorder() + s.Handler().ServeHTTP(rec, httptest.NewRequest("POST", "/v1/messages", strings.NewReader( + `{"model":"relay/model-2","max_tokens":1024,"stream":true,`+ + `"messages":[{"role":"user","content":"hi"}]}`))) + if rec.Code != 200 { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + return lastRoute(s) + } + + // the first turn went out under the names that stood when it came in, and + // is judged against those and not the ones given since: read afresh at + // every place it looks, its own record of the reply would name the model + // the relay never answered for + route := turn() + if got := seen; len(got) != 1 || got[0] != "model-2" { + t.Fatalf("the relay was asked for %v; the first turn goes out under the names it came in with", got) + } + if route.Served != "model-2" || route.Swapped { + t.Errorf("the relay answered with the model the turn went out under, which is no swap: %+v", route) + } + // and the next turn is the one the name given mid-flight is for + route = turn() + if got := seen; len(got) != 2 || got[1] != "vendor-c/model-2-renamed" { + t.Fatalf("the relay was asked for %v; the second turn goes out under the name given", got) + } + if route.Served != "vendor-c/model-2-renamed" || route.Swapped { + t.Errorf("the relay answered with the name the turn went out under, which is no swap: %+v", route) + } +} diff --git a/internal/gui/settings_keep_test.go b/internal/gui/settings_keep_test.go index dc002b7a..a61c5ce8 100644 --- a/internal/gui/settings_keep_test.go +++ b/internal/gui/settings_keep_test.go @@ -74,6 +74,7 @@ func TestSettingsSaveKeepsWhatItDoesNotSend(t *testing.T) { ModelImages: map[string]bool{"p/m": true}, ModelOutputs: map[string]int{"p/m": 131072}, ModelPrices: map[string]settings.ModelPrice{"p/m": {Input: &one, Output: &two}}, + ModelWires: map[string]string{"p/m": "vendor-c/m"}, RedactRules: []redact.Rule{{Kind: "prefix", Prefix: "oc_sk_"}}, LAN: true, LANKey: "sk-lan", diff --git a/internal/provider/modelprefs.go b/internal/provider/modelprefs.go index 7b543e46..bcb357ec 100644 --- a/internal/provider/modelprefs.go +++ b/internal/provider/modelprefs.go @@ -452,3 +452,395 @@ func (p Provider) serves(model string) bool { has := func(m catalog.Model) bool { return m.ID == model } return slices.ContainsFunc(p.Available(), has) || slices.ContainsFunc(p.Exposed(), has) } + +// ServesModel is whether the provider really serves model, listed or +// exposed: the question SetUpstreamName asks before it keeps a name for +// one, and the same one, since a name given for a model the provider does +// not serve is a request magpie would never send it. It is the answer a +// caller told to take a name off needs as well: a model the provider has +// nothing like is not asked for by any name, so reporting which one is in +// force for it would say magpie asks for a model it does not know. +func ServesModel(p Provider, model string) bool { return p.serves(model) } + +// UpstreamName is the name to send a provider for one of its models: the one +// the user gave, for the model or for every model of that provider, else the +// name magpie knows the model by. Every "*" in the name the user gave is that +// model, wherever it stands: "vendor-c/*" for a provider's every model sends +// vendor-c/model-3 for model-3, so one name covers a relay that namespaces +// its models and still asks for each of them by its own, and "vendor/*/pro" +// sends vendor/model-3/pro. A name with no "*" in it is that one name for +// every model of the key, which is the right answer only for a relay that +// does serve them all alike. +// +// The two are deliberately different things. Everything a user or an agent +// sees — the catalog, the routing groups' membership, what a call is recorded +// and priced as — is in magpie's name for the model, so a rename upstream +// never renames a model here. Only the request that goes out carries the +// vendor's name, and what a reply says answered is compared with it, so a +// model asked for by the name the vendor serves it under is not read as +// another one. +func UpstreamName(p Provider, model string) string { + return UpstreamNameIn(settings.Load().ModelWires, p.ID, model) +} + +// UpstreamNameIn is UpstreamName over a settings' ModelWires already read, +// for a caller that walks a whole ledger of models in a row. A name that is +// only whitespace — one an older magpie stored, or one hand-edited into the +// file — is no name, as it is where one is given. +func UpstreamNameIn(wires map[string]string, pid, model string) string { + n := strings.TrimSpace(wires[pid+"/"+model]) + if n == "" { + n = strings.TrimSpace(wires[pid+"/*"]) + } + if n == "" { + return model + } + return strings.ReplaceAll(n, "*", model) +} + +// UpstreamNameAs is UpstreamName for a request that goes out asking for sent, +// which is not always the model magpie knows: an Antigravity account is asked +// for the variant of it the effort picks, and that is the id the body carries. +// The two levels of the lookup are kept apart, since they are about different +// things. A name given for the model itself is the model's, whichever variant +// answers for it, so a "*" in it is that model and not the variant: the vendor +// is asked for the model by the name its owner gave, as it is everywhere else +// in magpie, and not for a "*" it never heard of. A "*" in the provider's own +// name is whatever the request is going out under, so one name for a provider +// asks for gemini-3.7-flash-high and gemini-3.7-flash-low each by its own, +// rather than for the one id whatever level the effort picked. +func UpstreamNameAs(p Provider, model, sent string) string { + return UpstreamNameAsIn(settings.Load().ModelWires, p.ID, model, sent) +} + +// UpstreamNameAsIn is UpstreamNameAs over a settings' ModelWires already +// read. A name that is only whitespace is no name, as it is in +// UpstreamNameIn: the lookup falls through to the level below it, and a +// request with no name at all goes out under the id it was built with. +func UpstreamNameAsIn(wires map[string]string, pid, model, sent string) string { + if sent == model { + return UpstreamNameIn(wires, pid, model) + } + if n := strings.TrimSpace(wires[pid+"/"+model]); n != "" { + return strings.ReplaceAll(n, "*", model) + } + if n := strings.TrimSpace(wires[pid+"/*"]); n != "" { + return strings.ReplaceAll(n, "*", sent) + } + return sent +} + +// SentNameIn is the name a call on the provider's model goes out under at +// that effort: the name the vendor would be asked for now, which is the one +// magpie knows the model by unless the user gave one. It is UpstreamNameAs for +// the id that call asks for, which is not the model magpie knows everywhere: +// on an Antigravity account it is the variant of it that effort picks, so a +// name given for the provider asks for each level by its own id, and a ledger +// reading a record back reads the vendor's reply against the name that +// really went out — the same one the gateway sent — rather than against the +// family it is one level of. +// +// The effort is the one the gateway sent, already fitted to the model's own +// levels, which is what a record keeps of it, so a level of the model's own +// is one of its variants and goes out as that one. An effort that is no +// level of the family names the same id the gateway asked for all the same: +// the level of the family's own nearest (AntigravitySentID), which is where +// the envelope went out. +// +// It is SentNameOnIn for the caller that has nothing but the id — a record in +// the ledger, which keeps the id the call went out on and nothing else. A +// provider that is an account is built with that account's own id as its own +// (googleProvider and the rest of the account providers), so the id says +// which account it was, and a provider of the user's own is no account's +// whatever its id is. A record's provider need not even be signed in now, +// which is why nothing here goes looking for the account behind the id: the +// id a call went out on is all a record has and all it can be read by. +func SentNameIn(wires map[string]string, pid, model, effort string) string { + return SentNameOnIn(wires, pid, pid, model, effort) +} + +// SentNameOnIn is SentNameIn for a caller holding the provider itself, and so +// the account behind it: the call goes out on that account, and it is the +// account the id the effort picks is a question about. A provider's id is +// that account's own id today (googleProvider and the rest of the account +// providers build each one with it), so passing the id for the account reads +// the same — but the two are one id by construction rather than by rule, and +// a provider of the user's own has no account at all whatever its id is. So +// the route, which is holding the provider, asks about the account and lets +// the gateway's dispatch (codeAssistID, on the same agent) be the one rule. +func SentNameOnIn(wires map[string]string, pid, account, model, effort string) string { + return UpstreamNameAsIn(wires, pid, model, antigravitySentID(account, model, effort)) +} + +// antigravitySentID is the id a call on the model goes out under at that +// effort on an Antigravity account. Every other account is asked for the +// model magpie knows, and so is one of a model this account does not serve +// as a family of levels. +func antigravitySentID(account, model, effort string) string { + if account != "antigravity" { + return model + } + return AntigravitySentID(model, effort) +} + +// AntigravitySentID is the id a request for model goes out under at that +// effort on an Antigravity account: the variant of the family the effort +// picks, at the level of that family nearest the effort asked for; with no +// effort asked for, the family's own default. One of the family's own +// variant ids — an id a pick or an agent's model list kept from before the +// families were one model — goes as it is when no effort is asked for, and +// at the level the effort picks otherwise, since the effort the client +// asks for wins, as it does for the family. Any other id goes as it is. +// +// The gateway sends the envelope under this and the ledger judges what the +// vendor answered against it, so there is one of it rather than one each: +// a reply naming the id a call went out under is that model answering, and +// read against the family instead every level of it reads as another +// model having swapped in, in the routing view and in the ledger alike. +func AntigravitySentID(model, effort string) string { + if base, _, ok := AntigravityBase(model); ok { + if effort == "" { + return model + } + model = base + } + vs, ok := AntigravityVariants(model) + if !ok { + return model + } + if effort == "" { + return vs[""] + } + var levels []string + for _, l := range Levels { + if vs[l] != "" { + levels = append(levels, l) + } + } + if id := vs[nearestEffort(effort, levels)]; id != "" { + return id + } + return vs[""] +} + +// nearestEffort is the level of levels nearest want in Levels' order, a tie +// going up, or want itself where the levels are none or hold it: an effort a +// family does not take is asked at the level of its own nearest, the rule +// the gateway fits an effort to a model's levels by on the way out +// (fitEffort), kept here where a ledger reading a record back can reach it +// too. +func nearestEffort(want string, levels []string) string { + at := slices.Index(Levels, want) + if len(levels) == 0 || at < 0 || slices.Contains(levels, want) { + return want + } + best, dist := want, len(Levels) + for _, l := range levels { + i := slices.Index(Levels, l) + if i < 0 || l == "none" { + continue + } + d := i - at + if d < 0 { + d = -d + } + if d < dist || d == dist && i > at { + best, dist = l, d + } + } + return best +} + +// SetUpstreamName is the name to send a provider for one of its models; an +// empty name takes the user's away and sends the one magpie knows. A name is +// only given for a model the provider really serves: a relay lists the model +// under the id magpie knows it by and asks for it under the vendor's own, and +// one it does not list is a request magpie would never send, so naming it is +// refused here, as a price and a limit are for the same id. A "*" stands +// for every model of the provider rather than for one, and a removal is +// exempt, so an entry left for a model that has since gone can still be +// taken away. +// +// A provider magpie asks through an agent's own backend has no model field +// to put a name in — the request goes out carrying the id the agent knows — +// so a name given for one of its models would never be sent (asksOwnBackend). +// +// The ref is taken as the provider it names, by its id now, and the name is +// kept under that id: a ref is as likely to be spelled with an id the +// provider had before a rename (Was) or with the name it is shown by, and +// every lookup of a wire name goes by the id the provider has (UpstreamName). +// A name under any other spelling of the provider is one no lookup will ever +// match — a name the provider is never asked for, and a removal that takes +// away nothing. +// +// The key is checked here, where it is given, and not as the whole settings +// are written: a hand-edited key no lookup will ever match is a no-op of the +// user's own making, and must not keep every other setting from being saved. +// +// Kept in settings, and saved without telling the agents, like a price: which +// model an agent picks, and what it is called, have not changed — only the +// name the request goes out under. +// +// A removal goes to DropUpstreamName, which works off the key rather than +// off this provider, so that taking a name away is the same operation +// whether or not the provider it was given for is still there. +// +// A blank name is a removal, and whether there was a name there to take away +// is DropUpstreamName's to say, not this one's: a caller with something of +// its own to say about a name that was not there — `magpie model wire -- +// reset`, which tells a mistyped id from a model already asked for by the +// name magpie knows it by — asks DropUpstreamName for the removal itself, +// the way `magpie model price --reset` asks DropModelPrice. A caller that +// only wants the model left as it is has nothing to tell the user either +// way, which is why a blank name is an error and not a second answer. +func SetUpstreamName(ref, name string) error { + p, model, err := splitRef(ref) + if err != nil { + return err + } + key := p.ID + "/" + model + if err := settings.CheckModelKey("wire name", key); err != nil { + return err + } + if name = strings.TrimSpace(name); name == "" { + _, err := DropUpstreamName(key) + return err + } + if model != "*" && !p.serves(model) { + return fmt.Errorf("%s has no model %s (magpie provider %s lists them)", p.ID, model, p.ID) + } + if p.asksOwnBackend() { + return fmt.Errorf("%s is %s's own account: magpie asks it for the model by the id it knows, so there is no upstream name to give it", p.Name, p.Account.Agent) + } + s := settings.Load() + if s.ModelWires == nil { + s.ModelWires = map[string]string{} + } + s.ModelWires[key] = name + return settings.Save(s) +} + +// DropUpstreamName takes a name off under the key it is stored at, and says +// whether there was one to take. A name outlives the provider it was given +// for: that provider can be deleted, and the name is then in force for +// whichever provider takes that id next — its requests would go out renamed +// to a vendor that never heard of it, and nothing would say so — so a +// removal that resolved the provider first would leave the user no way to +// take away the one name doing that. A key holding no name is not an error: +// there is nothing there to take away, which is the state it is left in +// either way, and false says so to a caller that has something else to say. +// +// Which key a ref is taken at is wireKey's: the name comes off from where it +// is really kept, rather than from under whichever provider the ref names +// now. +func DropUpstreamName(key string) (bool, error) { + key = strings.TrimPrefix(strings.TrimSpace(key), "magpie/") + if strings.HasPrefix(key, GroupPrefix) { + return false, errors.New("that is a routing group, not a provider's model") + } + pid, model, ok := strings.Cut(key, "/") + if !ok || pid == "" || model == "" { + return false, fmt.Errorf("name a model as provider/model, not %q", key) + } + s := settings.Load() + key = wireKey(s.ModelWires, pid, model) + if err := settings.CheckModelKey("wire name", key); err != nil { + return false, err + } + if _, ok := s.ModelWires[key]; !ok { + return false, nil + } + delete(s.ModelWires, key) + return true, settings.Save(s) +} + +// HasUpstreamName reports whether a name is kept under exactly the key it is +// given, the provider's part spelled as it is. No other spelling of the +// provider is tried: a caller deciding whether to remove a name by this key +// has to know that this key is the one a name is kept under, which is the +// one question wireKey's order is about. +// +// A key spelled as an agent spells one — with magpie/ in front of it, and +// the spaces a ref is typed with — is that same key, as it is for a removal +// (DropUpstreamName): a name kept under it is there whichever way it is +// spelled, and a check that answered for one spelling only would say a name +// was not kept where it is. The one caller has had that prefix taken off +// already (splitModelRef), so nothing depends on it yet. +func HasUpstreamName(key string) bool { + key = strings.TrimPrefix(strings.TrimSpace(key), "magpie/") + _, ok := settings.Load().ModelWires[key] + return ok +} + +// wireKey is the key a name for "pid/model", as the user spelled it, is kept +// under and taken off by, and the three spellings are tried in one order: +// the ref as it is given, the id the provider has or had, the name it is +// shown by. The first is where a name really is once the provider it was +// given for is gone — nothing rewrites the key, so it is still there under +// an id no provider has now, and it is the one a removal has to reach. +// Resolving that id through the name another provider is shown by instead +// would take away that provider's own name instead, and leave the one that +// is in force for whoever takes the id next in the file. +// +// The id a provider has, or had before a rename (Was), is where a name is +// written and read (SetUpstreamName, UpstreamName), and the name a provider +// is shown by is the last of the three, for a ref spelled that way and with +// nothing kept under either of the other two. +func wireKey(wires map[string]string, pid, model string) string { + key := pid + "/" + model + if _, ok := wires[key]; ok { + return key + } + if p, ok := byIDOrWas(pid); ok { + return p.ID + "/" + model + } + if p, err := Find(pid); err == nil { + return p.ID + "/" + model + } + return key +} + +// UpstreamNames are the names the user gave the provider's models, by model +// id, with the one given for all of them under "*" — each as it will be sent, +// trimmed, "*" and all, which is what a list of them shows. A name that is +// only whitespace is none, and is not listed. +func (p Provider) UpstreamNames() map[string]string { + out := map[string]string{} + for k, n := range settings.Load().ModelWires { + if m, ok := strings.CutPrefix(k, p.ID+"/"); ok { + if n = strings.TrimSpace(n); n != "" { + out[m] = n + } + } + } + return out +} + +// asksOwnBackend reports whether p is a signed-in account magpie reaches +// through that agent's own backend rather than through a vendor API: the +// request goes out carrying the model id the agent already knows, with no +// model field for magpie to name differently, so a wire name given for one +// of its models would never be sent. It is the account dispatch in +// internal/gateway that decides which agents those are, so this and that +// have to keep saying the same thing — a name accepted for an account the +// gateway asks its own way is a name never sent — and +// TestAsksOwnBackendAgreesWithTheGatewayDispatch reads the dispatch itself +// rather than a list written out a second time to keep them so. +// +// Codex is deliberately not among them: it is served by the ChatGPT backend +// (codex_backend.go), which has no provider id to key a name by. +func (p Provider) asksOwnBackend() bool { + if p.Account == nil { + return false + } + switch p.Account.Agent { + case "claude", "cursor", "devin", "kiro", "zed", CommandCodePlanID: + return true + } + // Qoder is one backend however many sites it is signed in to, and the + // gateway's dispatch covers the whole family rather than the site the + // account was made on, so the family is asked of rather than one id of + // it: a name refused for the one subscription has to be refused for + // the other too, which is as much magpie's own there. + return slices.Contains(qoderAgents, p.Account.Agent) +} diff --git a/internal/provider/modelprefs_test.go b/internal/provider/modelprefs_test.go index 749069ae..04bb4552 100644 --- a/internal/provider/modelprefs_test.go +++ b/internal/provider/modelprefs_test.go @@ -1,11 +1,16 @@ package provider import ( + "go/ast" + "go/parser" + "go/token" + "go/types" "os" "path/filepath" "reflect" "runtime" "slices" + "strconv" "strings" "testing" @@ -311,3 +316,881 @@ func TestRenameMovesEveryPerModelMap(t *testing.T) { t.Errorf("Visible, which is by agent id, was moved to %v", got) } } + +// A wire name is given only for a model the provider really serves: a relay +// lists the model under the id magpie knows and asks for it under the +// vendor's own, but one the provider does not serve is a request magpie would +// never send, so naming it is refused where it is given, as a price and a +// limit are for the same id. "*" stands for every model of the provider +// rather than for one and is allowed; a removal is exempt, so an entry left +// for a model that has since gone can still be taken away. +func TestSetUpstreamNameRefusesAModelNotServed(t *testing.T) { + prefsHome(t) + // "a" serves "sol" (prefsHome) + if err := SetUpstreamName("a/nope", "vendor-c/sol"); err == nil { + t.Error("a name for a model the provider does not serve was accepted") + } + if _, ok := settings.Load().ModelWires["a/nope"]; ok { + t.Error("a refused name was stored anyway") + } + // the whole-provider name stands on no one model, so it is allowed + if err := SetUpstreamName("a/*", "vendor-c/sol"); err != nil { + t.Errorf("a name for every model of the provider was refused: %v", err) + } + // a model that has since gone off the provider's list can still have the + // name it was given taken away: the removal is what the user is after + if err := SetUpstreamName("a/sol", "vendor-c/sol-2"); err != nil { + t.Fatal(err) + } + if err := SetUpstreamName("a/sol", ""); err != nil { + t.Errorf("taking away the name of a model now gone was refused: %v", err) + } + if _, ok := settings.Load().ModelWires["a/sol"]; ok { + t.Error("the name of a model now gone was not taken away") + } +} + +// A provider magpie reaches through an agent's own backend is asked for the +// model by the id that agent already knows, with no model field for a wire +// name to change; a name given for one of its models would never be sent, and +// comparing the served model against it would read every call as swapped. It +// is refused where it is given instead, so neither the gateway nor the usage +// ledger can read one off it. A key the user gave a provider is not one of +// these and still takes a name. +func TestSetUpstreamNameRefusesAnAccountBackend(t *testing.T) { + prefsHome(t) + // a Kiro account with a saved key is one magpie reaches through Kiro's + // own backend; the live list is the model it serves + if err := Save(Provider{ID: "kiro", Name: "Kiro", Key: "k"}); err != nil { + t.Fatal(err) + } + if err := catalog.SaveLive("kiro", "", []catalog.Model{{ID: "auto", Name: "Auto"}}); err != nil { + t.Fatal(err) + } + p, err := Find("kiro") + if err != nil { + t.Fatal(err) + } + if p.Account == nil || p.Account.Agent != "kiro" { + t.Fatalf("the provider is not the account the test set up: %+v", p.Account) + } + if !p.asksOwnBackend() { + t.Fatal("a signed-in account backend is not recognised as one") + } + if err := SetUpstreamName("kiro/auto", "vendor-c/auto"); err == nil { + t.Error("a name for an account backend's model was accepted") + } + if _, ok := settings.Load().ModelWires["kiro/auto"]; ok { + t.Error("a refused account-backend name was stored anyway") + } + // Codex is not an account backend for this purpose: it is served by the + // ChatGPT backend, which has no provider id to key a name by + if (Provider{Account: &Account{Agent: "codex"}}).asksOwnBackend() { + t.Error("the Codex subscription was taken for an account backend") + } + // a key the user gave a provider is not an account backend and takes one + if err := SetUpstreamName("a/sol", "vendor-c/sol"); err != nil { + t.Errorf("a name for a keyed provider's model was refused: %v", err) + } +} + +// A wire name is the name of a model of a provider, not of a spelling of +// that provider's id: it is kept, looked up and taken away under the id the +// provider has now, so a ref spelled with the id it had before a rename — +// which an agent still running on an older config sends — names that same +// model, and leaves no second key behind that no lookup will ever match. +func TestSetUpstreamNameFollowsARenamedProvider(t *testing.T) { + prefsHome(t) + if err := SetUpstreamName("a/sol", "vendor-c/sol"); err != nil { + t.Fatal(err) + } + if err := Rename("a", "c"); err != nil { + t.Fatal(err) + } + p, err := Find("c") + if err != nil { + t.Fatal(err) + } + // the rename took the name with it + if got := settings.Load().ModelWires["c/sol"]; got != "vendor-c/sol" { + t.Fatalf("the settings hold %q", got) + } + // giving one by the id the provider had is giving one for that model, + // not a name the provider is never asked for + if err := SetUpstreamName("a/sol", "vendor-c/sol-2"); err != nil { + t.Fatal(err) + } + if got := settings.Load().ModelWires; len(got) != 1 || got["c/sol"] != "vendor-c/sol-2" { + t.Fatalf("the settings hold %v", got) + } + if got := UpstreamName(*p, "sol"); got != "vendor-c/sol-2" { + t.Fatalf("the provider is asked for %q", got) + } + // and taking it away by that id takes away the one in force + if err := SetUpstreamName("a/sol", ""); err != nil { + t.Fatal(err) + } + if got := settings.Load().ModelWires; len(got) != 0 { + t.Fatalf("the settings still hold %v", got) + } + if got := UpstreamName(*p, "sol"); got != "sol" { + t.Fatalf("the provider is asked for %q", got) + } +} + +// The same for the one name given for every model of a provider: it is kept +// under the id the provider has now, and a removal by the id it had takes +// it away rather than leaving it in force under a key nothing matches. +func TestSetUpstreamNameWildcardFollowsARenamedProvider(t *testing.T) { + prefsHome(t) + if err := SetUpstreamName("a/*", "vendor-c/sol"); err != nil { + t.Fatal(err) + } + if err := Rename("a", "c"); err != nil { + t.Fatal(err) + } + p, err := Find("c") + if err != nil { + t.Fatal(err) + } + if err := SetUpstreamName("a/*", "vendor-c/sol-2"); err != nil { + t.Fatal(err) + } + if got := settings.Load().ModelWires; len(got) != 1 || got["c/*"] != "vendor-c/sol-2" { + t.Fatalf("the settings hold %v", got) + } + if got := UpstreamName(*p, "sol"); got != "vendor-c/sol-2" { + t.Fatalf("the provider is asked for %q", got) + } + if err := SetUpstreamName("a/*", ""); err != nil { + t.Fatal(err) + } + if got := settings.Load().ModelWires; len(got) != 0 { + t.Fatalf("the settings still hold %v", got) + } + if got := UpstreamName(*p, "sol"); got != "sol" { + t.Fatalf("the provider is asked for %q", got) + } +} + +// The provider of a ref is the provider it names, however it is spelled, as +// it already is for the setters beside this one: a name a user reaches for +// a provider by is that provider, and the wire name is kept under the id +// it has. What the key check is for is a provider id that is not spelled +// the way a provider's is — a file magpie did not write, whose keys no +// lookup goes by — and not the name the provider is shown by. +func TestSetUpstreamNameTakesTheNameAProviderIsCalledBy(t *testing.T) { + prefsHome(t) + // the same list "a" serves sol from in prefsHome + if err := Save(Provider{ID: "relay", Name: "My Relay", Catalog: "a", Key: "k", + Chat: "http://127.0.0.1:1/v1", Models: []string{"sol"}}); err != nil { + t.Fatal(err) + } + const ref = "My Relay/sol" + if err := SetUpstreamName(ref, "vendor-c/sol"); err != nil { + t.Fatalf("a ref spelled with the name the provider is shown by was refused: %v", err) + } + // the setters beside this one take the very same ref + if err := SetModelName(ref, "My Sol"); err != nil { + t.Fatalf("%q names a model of that provider, as the others have it: %v", ref, err) + } + // and both names are kept under the id the provider has, which is what + // every lookup goes by + if got := settings.Load().ModelWires; len(got) != 1 || got["relay/sol"] != "vendor-c/sol" { + t.Fatalf("the settings hold %v", got) + } + if got := settings.Load().ModelNames["relay/sol"]; got != "My Sol" { + t.Fatalf("the settings hold the name %q", got) + } + p, err := Find("relay") + if err != nil { + t.Fatal(err) + } + if got := UpstreamName(*p, "sol"); got != "vendor-c/sol" { + t.Fatalf("the provider is asked for %q", got) + } + if n, ok := ModelName("relay", "sol"); !ok || n != "My Sol" { + t.Fatalf("the model is called %q (%t)", n, ok) + } +} + +// The key is checked where the name is given, and what it says names the +// setting being checked: the wire name a user cannot give, and not the +// price or the limit of the same model. +func TestSetUpstreamNameSaysWhichSettingItChecks(t *testing.T) { + prefsHome(t) + // a providers file magpie did not write: "a" under an id that is not + // spelled the way a provider's is, which is what the check refuses + f := load() + for i := range f.Providers { + if f.Providers[i].ID == "a" { + f.Providers[i].ID = "my_relay" + } + } + if err := store(f); err != nil { + t.Fatal(err) + } + err := SetUpstreamName("my_relay/sol", "vendor-c/sol") + if err == nil { + t.Fatal("a wire name was given for a key the settings do not keep models under") + } + if !strings.Contains(err.Error(), "wire name") { + t.Fatalf("the error does not say which setting it is about: %v", err) + } + if got := settings.Load().ModelWires; len(got) != 0 { + t.Fatalf("a refused name was stored anyway: %v", got) + } +} + +// A row of the ledger is judged by the name in force when the ledger is +// read: a record keeps the name the vendor's reply gave and not the name +// the request went out under, so this is the only name there is to judge by, +// and giving a name after a call re-judges that call — which is what +// usage.Ledger's comment sets out as the trade. What makes it so is that the +// lookup takes the names as the settings stand at the time of reading them, +// with nothing in it that remembers what they held when the call went out, +// and this is that side of it. +func TestUpstreamNameInFollowsTheNamesAsTheyAreNow(t *testing.T) { + prefsHome(t) + // "a" serves "sol" (prefsHome) + if got := UpstreamNameIn(settings.Load().ModelWires, "a", "sol"); got != "sol" { + t.Fatalf("with no name given, a/sol is judged as %q; want sol", got) + } + // a call made and answered under the name magpie knows the model by, + // before any name is given for it + if err := SetUpstreamName("a/sol", "vendor-c/sol"); err != nil { + t.Fatal(err) + } + // the same call read afterwards is judged by the name given since, so + // the vendor's answer naming the very model that was asked for reads + // as another one + if got := UpstreamNameIn(settings.Load().ModelWires, "a", "sol"); got != "vendor-c/sol" { + t.Errorf("after a name is given, a/sol is judged as %q; want vendor-c/sol", got) + } + // and a model with no name of its own is judged by the provider's, the + // same way round: whatever was in force when the call went out is not + // what a later reading of it is measured against + if err := SetUpstreamName("a/*", "vendor-c/one"); err != nil { + t.Fatal(err) + } + if got := UpstreamNameIn(settings.Load().ModelWires, "a", "other"); got != "vendor-c/one" { + t.Errorf("a model with no name of its own is judged as %q; want the provider's", got) + } + if got := UpstreamNameIn(settings.Load().ModelWires, "a", "sol"); got != "vendor-c/sol" { + t.Errorf("with both names in force, a/sol is judged as %q; want its own over the provider's", got) + } +} + +// A ref is taken at the key its name is really stored at, and the spellings +// are tried in one order: the ref as the user spelled it, the id the provider +// has or had, and last the name it is shown by — a spelling another provider +// may be shown by as well, so it is only trusted where nothing is kept under +// either of the other two. A name outlives the provider it was given for, so +// the first is where it is when that provider is gone and its id is free +// again, and it is the one a removal has to reach. +func TestDropUpstreamNameTakesTheKeyTheNameIsStoredAt(t *testing.T) { + prefsHome(t) + // the name a provider is shown by, under the id it has + if err := Save(Provider{ID: "relay", Name: "My Relay", Catalog: "a", Key: "k", + Chat: "http://127.0.0.1:1/v1", Models: []string{"sol"}}); err != nil { + t.Fatal(err) + } + if err := SetUpstreamName("My Relay/sol", "vendor-c/sol"); err != nil { + t.Fatal(err) + } + if dropped, err := DropUpstreamName("My Relay/sol"); err != nil || !dropped { + t.Fatalf("DropUpstreamName = %v, %v; want the name taken off by the name its provider is shown by", dropped, err) + } + // an id a provider had before a rename is the id every lookup still + // goes by, and so is the one a name is taken off at + if err := SetUpstreamName("a/sol", "vendor-c/sol"); err != nil { + t.Fatal(err) + } + if err := Rename("a", "renamed"); err != nil { + t.Fatal(err) + } + if dropped, err := DropUpstreamName("a/sol"); err != nil || !dropped { + t.Fatalf("DropUpstreamName = %v, %v; want the name taken off by the id its provider had", dropped, err) + } + // an id no provider is called by and no name is kept under is a mistyped + // one: there is nothing to take off, which is not an error, and it + // takes nothing else away on the way to saying so + if dropped, err := DropUpstreamName("nope/sol"); dropped || err != nil { + t.Fatalf("DropUpstreamName on an id no provider is called by = %v, %v; want no name taken and no error", dropped, err) + } + if got := settings.Load().ModelWires; len(got) != 0 { + t.Errorf("the settings hold %v; a ref naming no provider took nothing off, and nothing else either", got) + } +} + +// The id a wire name was given under is free again once its provider is +// deleted, and another provider can be shown by it — the name that provider +// is shown by is a spelling any other provider may answer to, so `b/sol` +// names a provider that is not the one a name under `b/sol` is for. Taking +// the name off through it would take away that provider's own name instead, +// a name that is really in force, and leave the one that renames a +// provider's requests for a vendor that never heard of them in the file. +func TestDropUpstreamNameWorksOffTheKeyRatherThanTheProvidersName(t *testing.T) { + prefsHome(t) + if err := SetUpstreamName("b/sol", "vendor-c/sol"); err != nil { + t.Fatal(err) + } + if err := Delete("b"); err != nil { + t.Fatal(err) + } + // a provider of an id of its own, c, shown by the id b that is now + // nobody's, and with a name of its own to be left alone + if err := Save(Provider{ID: "c", Name: "B", Catalog: "b", Key: "k", + Chat: "http://127.0.0.1:1/v1", Models: []string{"sol"}}); err != nil { + t.Fatal(err) + } + if err := SetUpstreamName("c/sol", "vendor-d/sol"); err != nil { + t.Fatal(err) + } + if dropped, err := DropUpstreamName("b/sol"); err != nil || !dropped { + t.Fatalf("DropUpstreamName = %v, %v; want the name under b/sol taken off by that key", dropped, err) + } + wires := settings.Load().ModelWires + if _, ok := wires["b/sol"]; ok { + t.Errorf("b/sol still has a name, %v; it is in force for whichever provider takes that id next", wires) + } + if got := wires["c/sol"]; got != "vendor-d/sol" { + t.Errorf("c's own name is %q; the removal went through the name c is shown by", got) + } + // which is the whole of what taking the name off buys: a provider + // taking the id b is asked for its model by the name magpie knows it by + if err := Save(Provider{ID: "b", Name: "B", Catalog: "b", Key: "k", + Chat: "http://127.0.0.1:1/v1", Models: []string{"sol"}}); err != nil { + t.Fatal(err) + } + p, err := Find("b") + if err != nil { + t.Fatal(err) + } + if got := UpstreamName(*p, "sol"); got != "sol" { + t.Errorf("the provider taking the id b is asked for %q; want sol", got) + } +} + +// HasUpstreamName answers for the key a name is really kept under, however +// it is spelled: a name that is there is not "not kept" because the ref +// that named it carried a magpie/ in front of it, which is the spelling +// every agent uses and the one the removal standing in front of it is asked +// for by. +func TestHasUpstreamNameReadsTheKeyHoweverItIsSpelled(t *testing.T) { + prefsHome(t) + if err := SetUpstreamName("b/sol", "vendor-c/sol"); err != nil { + t.Fatal(err) + } + if err := SetUpstreamName("b/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + for _, key := range []string{"b/sol", "magpie/b/sol", " b/sol ", "magpie/b/*"} { + if !HasUpstreamName(key) { + t.Errorf("a name is kept under %q; HasUpstreamName says none is", key) + } + } + for _, key := range []string{"b/other", "magpie/b/other", "magpie/a/sol", ""} { + if HasUpstreamName(key) { + t.Errorf("no name is kept under %q; HasUpstreamName says one is", key) + } + } +} + +// A name given for one model is that model's at every level, so the vendor +// is asked for the model by the name its owner gave it whichever variant +// answers for it — the name the user is told that model goes out under, and +// never a "*" the vendor never heard of. A name given for the whole provider +// is a different thing: a "*" in it is whatever a request goes out under, so +// each level is asked for by its own id and the effort keeps choosing. +func TestUpstreamNameAsGivesAModelTheNameItWasGiven(t *testing.T) { + prefsHome(t) + a, err := Find("a") + if err != nil { + t.Fatal(err) + } + if err := SetUpstreamName("a/sol", "vendor-c/*"); err != nil { + t.Fatal(err) + } + if got, want := UpstreamName(*a, "sol"), "vendor-c/sol"; got != want { + t.Fatalf("a/sol is asked for as %q; want %q", got, want) + } + if got := UpstreamNameAs(*a, "sol", "sol-high"); got != "vendor-c/sol" { + t.Errorf("at the sol-high variant it is asked for as %q; want the model's own name, vendor-c/sol", got) + } + + b, err := Find("b") + if err != nil { + t.Fatal(err) + } + if err := SetUpstreamName("b/*", "vendor-c/*"); err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ sent, want string }{ + {"sol-high", "vendor-c/sol-high"}, + {"sol", "vendor-c/sol"}, + } { + if got := UpstreamNameAs(*b, "sol", tc.sent); got != tc.want { + t.Errorf("with a name for the whole provider, asking for %q gives %q; want %q", tc.sent, got, tc.want) + } + } +} + +// The name a call goes out under is the one the vendor is asked for, and on +// an Antigravity account that is not the model magpie knows: the family is +// asked for as the variant the effort picks, each by its own name. A ledger +// reading the record back therefore judges the reply against the id that +// really went out — read against the family, every level of it reads as +// another model answering. +func TestSentNameInAsksForTheVariantTheEffortPicks(t *testing.T) { + prefsHome(t) + // Antigravity's ids of one model at three levels, as its fetch left them + if err := os.MkdirAll(filepath.Dir(catalog.LivePath("antigravity")), 0o755); err != nil { + t.Fatal(err) + } + live := `{"models":[{"id":"flash-9-low"},{"id":"flash-9-medium"},{"id":"flash-9-high"}]}` + if err := os.WriteFile(catalog.LivePath("antigravity"), []byte(live), 0o644); err != nil { + t.Fatal(err) + } + catalog.Reset() + wires := map[string]string{"antigravity/*": "vendor-c/*"} + for _, tc := range []struct{ effort, want string }{ + {"high", "vendor-c/flash-9-high"}, + {"medium", "vendor-c/flash-9-medium"}, + {"low", "vendor-c/flash-9-low"}, + // no level asked for is the level Antigravity thinks at when it is + // not told one, which is the highest of the family + {"", "vendor-c/flash-9-high"}, + } { + if got := SentNameIn(wires, "antigravity", "flash-9", tc.effort); got != tc.want { + t.Errorf("at %q the vendor is asked for %q; want %q", tc.effort, got, tc.want) + } + } + // with no name given, it is the id itself: a reply naming the level the + // call went out at is that model, whatever family it is a level of + if got := SentNameIn(nil, "antigravity", "flash-9", "medium"); got != "flash-9-medium" { + t.Errorf("with no upstream name the vendor is asked for %q; want the variant", got) + } + // another provider has no variants of its own, so the effort is no part + // of the name and a star given for the provider stands for the model + if got := SentNameIn(map[string]string{"a/*": "vendor-d/*"}, "a", "sol", "high"); got != "vendor-d/sol" { + t.Errorf("provider a at high is asked for %q; want vendor-d/sol", got) + } +} + +// The id a call goes out under is one id, whether the request named the +// family or one of its own variant ids: an agent's model list and a pick +// saved from before the families were one model still name those, and +// Antigravity serves each of them as it stands. So an effort asked for +// picks the level of the family in either case, and no effort asked for +// leaves the id that was named alone — the rule the gateway's envelope is +// built by (codeAssistID) and the rule a ledger reads a record back against +// are one rule, and these are the two ends of it. +func TestSentNameInAsksAVariantIdForWhatTheEffortPicks(t *testing.T) { + prefsHome(t) + // one model at three levels, and one at two, so a family's default is + // the top of it in one case and not the other + if err := os.MkdirAll(filepath.Dir(catalog.LivePath("antigravity")), 0o755); err != nil { + t.Fatal(err) + } + live := `{"models":[{"id":"flash-9-low"},{"id":"flash-9-medium"},{"id":"flash-9-high"},` + + `{"id":"note-4-low"},{"id":"note-4-medium"}]}` + if err := os.WriteFile(catalog.LivePath("antigravity"), []byte(live), 0o644); err != nil { + t.Fatal(err) + } + catalog.Reset() + t.Cleanup(catalog.Reset) + for _, tc := range []struct{ model, effort, want string }{ + // the effort the client asks for wins, as it does for the family + {"flash-9-high", "low", "flash-9-low"}, + {"flash-9-low", "medium", "flash-9-medium"}, + // with none asked for, the id that was named goes as it is + {"flash-9-high", "", "flash-9-high"}, + {"flash-9-low", "", "flash-9-low"}, + // an effort the family does not take is asked at the level of its + // own nearest, not at the one the family thinks at when told none + {"flash-9-high", "minimal", "flash-9-low"}, + {"flash-9", "xhigh", "flash-9-high"}, + // the same for a family whose default is not its top: the id asked + // for is the one that goes out, and an effort moves it to the level + // it picks + {"note-4-low", "", "note-4-low"}, + {"note-4-medium", "", "note-4-medium"}, + {"note-4-low", "high", "note-4-medium"}, + {"note-4-low", "minimal", "note-4-low"}, + } { + if got := SentNameIn(nil, "antigravity", tc.model, tc.effort); got != tc.want { + t.Errorf("%s at %q goes out as %q; want %q", tc.model, tc.effort, got, tc.want) + } + if got := AntigravitySentID(tc.model, tc.effort); got != tc.want { + t.Errorf("%s at %q: the id itself is %q, want %q", tc.model, tc.effort, got, tc.want) + } + } + // a model of no family goes as it is, whatever effort is asked for + if got := SentNameIn(nil, "antigravity", "solo", "high"); got != "solo" { + t.Errorf("a model of no family is asked for as %q; want solo", got) + } +} + +// The variants belong to the account and not to the provider: a model is +// asked for one of its levels only where the account behind the provider is +// the one serving that family. The two are separate questions wherever a +// provider answers to an id that is not its account's — an account provider is +// built with its account's own id today, so reading one for the other agrees +// and says nothing about the rule. So an account the user is signed in to +// under a provider of its own id is asked for the variant, and a relay added +// under a built-in account's id is that relay and is asked for the model +// magpie knows, which is what the gateway's own dispatch (codeAssistID) reads +// the same way. +func TestSentNameIsJudgedByTheAccountAndNotByTheProviderId(t *testing.T) { + prefsHome(t) + if err := os.MkdirAll(filepath.Dir(catalog.LivePath("antigravity")), 0o755); err != nil { + t.Fatal(err) + } + live := `{"models":[{"id":"flash-9-low"},{"id":"flash-9-high"}]}` + if err := os.WriteFile(catalog.LivePath("antigravity"), []byte(live), 0o644); err != nil { + t.Fatal(err) + } + catalog.Reset() + t.Cleanup(catalog.Reset) + wires := map[string]string{"relay-1/*": "vendor-c/*", "antigravity/*": "vendor-d/*"} + + // the account is the one serving the family, whatever id the provider + // it is on answers to + if got := SentNameOnIn(wires, "relay-1", "antigravity", "flash-9", "low"); got != "vendor-c/flash-9-low" { + t.Errorf("an Antigravity account on a provider of another id, at low: the vendor is asked for %q; want vendor-c/flash-9-low", got) + } + // and the provider's id is no part of the question: a relay the user + // added under a built-in account's id is that relay, and no level of + // the family is what a call on it goes out under + if got := SentNameOnIn(wires, "antigravity", "", "flash-9", "low"); got != "vendor-d/flash-9" { + t.Errorf("a relay under the account's own id, at low: the vendor is asked for %q; want vendor-d/flash-9", got) + } + // another account is asked for the model magpie knows as well + if got := SentNameOnIn(wires, "relay-1", "gemini", "flash-9", "low"); got != "vendor-c/flash-9" { + t.Errorf("another account on the same provider, at low: the vendor is asked for %q; want vendor-c/flash-9", got) + } + // a record keeps the id the call went out on and no account, and that id + // is the account's own, so the ledger reads it back the same way — and a + // record of a relay, whose id is no account's, by that id + if got := SentNameIn(wires, "antigravity", "flash-9", "low"); got != "vendor-d/flash-9-low" { + t.Errorf("a record of a call on the account's own provider, at low, reads back as %q; want vendor-d/flash-9-low", got) + } + if got := SentNameIn(wires, "relay-1", "flash-9", "low"); got != "vendor-c/flash-9" { + t.Errorf("a record of a call on a relay, at low, reads back as %q; want vendor-c/flash-9", got) + } +} + +// A name that is only whitespace is no name at either level of the lookup, +// as it is in UpstreamNameIn: the model's own falls through to the +// provider's, and both falling through leaves the request under the id it +// was built with. A blank name read as a name would send a relay an id of +// spaces. +func TestUpstreamNameAsInIgnoresANameOfOnlyWhitespace(t *testing.T) { + for _, tc := range []struct { + what string + wires map[string]string + sent, want string + }{ + {"the model's own name is blank, the provider's is not", + map[string]string{"relay/sol": " ", "relay/*": "vendor-c/*"}, "sol-high", "vendor-c/sol-high"}, + {"both are blank", + map[string]string{"relay/sol": " ", "relay/*": "\t"}, "sol-high", "sol-high"}, + {"the provider's is blank and no other model has one", + map[string]string{"relay/sol": " ", "relay/*": " "}, "sol-high", "sol-high"}, + {"the request goes out under the model itself", + map[string]string{"relay/sol": " ", "relay/*": "vendor-c/*"}, "sol", "vendor-c/sol"}, + } { + if got := UpstreamNameAsIn(tc.wires, "relay", "sol", tc.sent); got != tc.want { + t.Errorf("%s: asking for %q gives %q; want %q", tc.what, tc.sent, got, tc.want) + } + } +} + +// The gateway decides where a request goes, and for an account it hands to +// the agent's own backend the model it knows is what goes out — so a wire +// name given for one of those accounts is accepted, never sent, and read at +// the ledger as every call through it a swap. asksOwnBackend refuses them +// where the name is given, and the two have to be the same set: an account +// the gateway starts serving its own way is refused a name the day it +// lands. So the set is read out of the dispatch itself rather than kept a +// second time beside it, which is the copy that goes on saying what the +// dispatch stopped saying. +func TestAsksOwnBackendAgreesWithTheGatewayDispatch(t *testing.T) { + d := &dispatch{t: t, seen: map[string]bool{}} + dispatched := d.accountBackends(filepath.Join("..", "gateway")) + if len(dispatched) == 0 { + t.Fatal("no account is read as one the gateway asks through its own backend; its dispatch is not being read") + } + for _, agent := range dispatched { + if !(Provider{Account: &Account{Agent: agent}}).asksOwnBackend() { + t.Errorf("the gateway asks a %s account through its own backend, where the model id the agent knows is what goes out, and a wire name is accepted for one of its models", agent) + } + } + // and nothing beyond them: an account the gateway asks over a vendor's + // API does send the name, so refusing one there refuses a name that + // would have worked + for _, agent := range accountIDs { + if slices.Contains(dispatched, agent) { + continue + } + if (Provider{Account: &Account{Agent: agent}}).asksOwnBackend() { + t.Errorf("a %s account is asked over the vendor's own API, where a wire name is sent, and one is refused for its models", agent) + } + } +} + +// dispatch reads an account dispatch out of the sources it is written in: +// which package a name is a constant of, and which constants are on their +// way to themselves. A module's own name is the prefix every import path in +// it has, which is how one package's constant is looked up in another. +type dispatch struct { + t *testing.T + root string // the tree the module is checked out in + mod string // what its go.mod calls the module + seen map[string]bool +} + +// accountBackends are the agents the gateway serves a request itself for, +// where the account belongs to the agent whose backend it is: a branch that +// asks whether the provider's account is one of them and then reaches that +// backend's own serve, as against one that sends the request to a vendor. +// A test file is not read: what a test says the dispatch does is no part of +// what it does. +func (d *dispatch) accountBackends(dir string) []string { + d.t.Helper() + var out []string + for _, name := range d.files(dir) { + ast.Inspect(d.parse(name), func(n ast.Node) bool { + ifs, ok := n.(*ast.IfStmt) + if !ok || !d.servesItself(ifs) { + return true + } + for _, agent := range d.agents(ifs.Cond, dir) { + if !slices.Contains(out, agent) { + out = append(out, agent) + } + } + return true + }) + } + slices.Sort(out) + return out +} + +// servesItself says whether a branch is one where the request is answered for +// the agent that owns the account instead of sent to a vendor: it ends in +// one of the serves, each named for the backend it answers for and reached +// by every account of that agent. A branch that only reads the account — what +// a request needs from it, which model it counts tokens for — is not one, +// and neither is a call that begins with serve and serves nothing: +// servesElsewhere answers whether a provider serves something. +func (d *dispatch) servesItself(ifs *ast.IfStmt) bool { + d.t.Helper() + for _, stmt := range ifs.Body.List { + own := false + ast.Inspect(stmt, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + if sel, ok := call.Fun.(*ast.SelectorExpr); ok { + if name := sel.Sel.Name; len(name) > len("serve") && name[len("serve")] >= 'A' && name[len("serve")] <= 'Z' { + own = true + } + } + return !own + }) + if own { + return true + } + } + return false +} + +// agents are the ones a condition compares the provider's account against, +// and nothing at all unless it asks the account is there: the same +// comparison is made elsewhere in the gateway to tell what a request needs +// rather than where it goes. +func (d *dispatch) agents(cond ast.Expr, dir string) []string { + d.t.Helper() + var out []string + account := false + ast.Inspect(cond, func(n ast.Node) bool { + b, ok := n.(*ast.BinaryExpr) + if !ok { + return true + } + switch b.Op { + case token.NEQ: + if d.isField(b.X, "Account") && d.isIdent(b.Y, "nil") { + account = true + } + case token.EQL: + sel, ok := b.X.(*ast.SelectorExpr) + if !ok || sel.Sel.Name != "Agent" || !d.isField(sel.X, "Account") { + return true + } + agent := d.id(dir, b.Y) + if agent == "" { + d.t.Errorf("the dispatch compares an account's agent against %s, which is not an id this test can read: a wire name for that account would be accepted and never sent", + types.ExprString(b.Y)) + return true + } + out = append(out, agent) + } + return true + }) + if !account { + return nil + } + return out +} + +// isField says whether the expression is a field of that name, and isIdent +// whether it is a name standing for itself. +func (d *dispatch) isField(e ast.Expr, name string) bool { + sel, ok := e.(*ast.SelectorExpr) + return ok && sel.Sel.Name == name +} + +func (d *dispatch) isIdent(e ast.Expr, name string) bool { + id, ok := e.(*ast.Ident) + return ok && id.Name == name +} + +// id is the agent id a comparison is against, read as far as it is written: +// a string as it stands, or the constant it names — of the package being +// read, as the gateway writes its own, or of one that package imports, as +// Qoder's CN subscription is named. A constant written in terms of itself +// is no id, and says so rather than standing in for one. +func (d *dispatch) id(dir string, e ast.Expr) string { + d.t.Helper() + switch x := e.(type) { + case *ast.BasicLit: + if x.Kind == token.STRING { + s, err := strconv.Unquote(x.Value) + if err != nil { + d.t.Errorf("%s is not a string this test can read: %v", x.Value, err) + return "" + } + return s + } + case *ast.Ident: + return d.constant(dir, x.Name) + case *ast.SelectorExpr: + if pkg, ok := x.X.(*ast.Ident); ok { + return d.constant(d.pkgDir(dir, pkg.Name), x.Sel.Name) + } + } + return "" +} + +// constant is what a constant of the package in dir is written as, in the +// first file that declares it, or the empty string where none does. +func (d *dispatch) constant(dir, name string) string { + d.t.Helper() + key := filepath.Join(dir, name) + if d.seen[key] { + return "" + } + d.seen[key] = true + for _, file := range d.files(dir) { + var id string + ast.Inspect(d.parse(file), func(n ast.Node) bool { + spec, ok := n.(*ast.ValueSpec) + if !ok || id != "" || len(spec.Names) != 1 || spec.Names[0].Name != name || len(spec.Values) != 1 { + return true + } + id = d.id(dir, spec.Values[0]) + return true + }) + if id != "" { + return id + } + } + return "" +} + +// pkgDir is where the package in dir imports `pkg` from keeps its sources: +// an import path is the module's own name and a path under it, so the path +// under it is where those sources are in the same tree. +func (d *dispatch) pkgDir(dir, pkg string) string { + d.t.Helper() + if d.root == "" { + d.root, d.mod = d.module() + } + mod := d.mod + "/" + for _, name := range d.files(dir) { + for _, im := range d.parse(name).Imports { + path, err := strconv.Unquote(im.Path.Value) + if err != nil || !strings.HasPrefix(path, mod) { + continue + } + as := filepath.Base(path) + if im.Name != nil { + as = im.Name.Name + } + if as == pkg { + return filepath.Join(d.root, filepath.FromSlash(strings.TrimPrefix(path, mod))) + } + } + } + d.t.Fatalf("no package is imported as %q from %s; the dispatch cannot be read", pkg, dir) + return "" +} + +// module is the tree the sources are in and what its go.mod calls the +// module, found by walking up to the go.mod that names it. +func (d *dispatch) module() (string, string) { + d.t.Helper() + root, err := filepath.Abs(".") + if err != nil { + d.t.Fatalf("the tree the sources are in: %v", err) + } + for { + if b, err := os.ReadFile(filepath.Join(root, "go.mod")); err == nil { + for _, l := range strings.Split(string(b), "\n") { + if m, ok := strings.CutPrefix(l, "module "); ok { + return root, strings.TrimSpace(m) + } + } + } + if up := filepath.Dir(root); up != root { + root = up + continue + } + break + } + d.t.Fatal("no go.mod above the sources says what the module is called; the dispatch cannot be read") + return "", "" +} + +// files are the sources of a package, its tests left out. +func (d *dispatch) files(dir string) []string { + d.t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + d.t.Fatalf("the sources in %s: %v", dir, err) + } + var out []string + for _, e := range entries { + if n := e.Name(); !e.IsDir() && strings.HasSuffix(n, ".go") && !strings.HasSuffix(n, "_test.go") { + out = append(out, filepath.Join(dir, n)) + } + } + if len(out) == 0 { + d.t.Fatalf("no sources in %s", dir) + } + slices.Sort(out) + return out +} + +func (d *dispatch) parse(name string) *ast.File { + d.t.Helper() + f, err := parser.ParseFile(token.NewFileSet(), name, nil, 0) + if err != nil { + d.t.Fatalf("%s: %v", name, err) + } + return f +} diff --git a/internal/provider/test.go b/internal/provider/test.go index 19b9e0d1..ce579d32 100644 --- a/internal/provider/test.go +++ b/internal/provider/test.go @@ -48,7 +48,7 @@ func (p Provider) Test(ctx context.Context) []Result { out = append(out, Result{Protocol: proto, Model: model, Error: "no key is on for this endpoint"}) continue } - url, body := tiny(q, proto, model) + url, body := tiny(q, proto, UpstreamName(p, model)) out = append(out, probe(ctx, q, proto, url, q.Prepare([]byte(body)), model, testWait)) } return out @@ -172,7 +172,12 @@ func (p Provider) testOne(ctx context.Context, model string) Result { } if draws { // as the gateway does, one the images API doesn't serve is tried - // in chat, and only its answer said when that fails too + // in chat, and only its answer said when that fails too. Both go + // out under the name magpie knows the model by: the gateway builds + // those bodies itself and an upstream name is never written into + // one, so asking for the model's own here is what tests a drawing + // the way a real one is made — a name the vendor serves drawings + // by is not one magpie sends them as url, body := tinyDrawing(q, model) r := probe(ctx, q, proto, url, []byte(body), model, drawWait) if !r.OK && (r.Status == 404 || r.Status == 405) { @@ -183,7 +188,7 @@ func (p Provider) testOne(ctx context.Context, model string) Result { } return r } - url, body := tiny(q, proto, model) + url, body := tiny(q, proto, UpstreamName(p, model)) return probe(ctx, q, proto, url, q.Prepare([]byte(body)), model, testWait) } diff --git a/internal/settings/settings.go b/internal/settings/settings.go index 5711a9d2..04fd4b18 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -197,6 +197,15 @@ type Settings struct { // provider's, and the agents' own files are told of either // (see provider.SetModelOutput). ModelOutputs map[string]int `json:"modelOutputs,omitempty"` + // ModelWires is the name to send a vendor for a model magpie knows by + // another, by "/", and "*" for every model of that + // provider. A "*" in the name is the model itself, so one name covers a + // relay that namespaces its models — vendor-c/* asks for model-3 as + // vendor-c/model-3 — while a name with no "*" in it sends every model of + // that key under that one name. Everything else — the catalog agents see, + // the routing groups, the usage records and what a call is priced at — + // keeps the name magpie knows the model by. + ModelWires map[string]string `json:"modelWires,omitempty"` // The main window's size when it was last resized, width and height, // so it opens at it again after a restart. Window []int `json:"window,omitempty"` diff --git a/internal/usage/ledger.go b/internal/usage/ledger.go index cbecb49d..de159948 100644 --- a/internal/usage/ledger.go +++ b/internal/usage/ledger.go @@ -14,6 +14,7 @@ import ( "github.com/yetone/magpie/internal/catalog" "github.com/yetone/magpie/internal/provider" + "github.com/yetone/magpie/internal/settings" ) // Row is one call as the ledger lists it. @@ -67,12 +68,47 @@ func (p Period) Since(now time.Time) time.Time { // Ledger is the calls of a period that the filter keeps, newest first, // with their sum, and the agents that made any call in the period (their // ids, for a filter to offer). +// +// A row's swapped is judged by the upstream names in force now, not by the +// one its call went out under: the names are read from the settings here, +// once, so a name given or changed since a call was recorded re-judges that +// call — one recorded under the name magpie knows the model by, and read +// after a name was set for it, reads as a swap although the vendor answered +// with the very model that was asked for. A record keeps the canonical id +// and the name the vendor's own reply gave (served_model) and not the name +// the request was sent under, so this is the only name there is to judge by +// — read as the name that went out, which the effort a record keeps of +// makes exact: an Antigravity account is asked for the variant of the model +// that level picks, so a reply naming that one is the model that was asked +// for and not another, whatever the family it is a level of. +// It is how the loop below already treats the rest of a row: a provider +// renamed since is put back to the id it has now, and the names in force now +// are what the catalog lists the models under as well. +// +// Storing the name that was sent would judge each call by what it was +// really sent under, and it is the better answer, but it is not this +// function's to make: it is a field on every record the gateway writes, and +// so a change to the file's format — a column a row written by an older +// magpie has no value for, and a decision about whether an empty one means +// "no upstream name" or "unknown" — rather than a row of a report. The rows +// here are what a reader sets beside a bill, and a name in force today is +// the one whose reply a bill's model column is read against. +// +// The names are read here, once for the lot, rather than asked for a row +// at a time, so a caller already holding them — a report over the +// provider/model it was given — reads them the same way and reaches the +// same judgement: what a row says is the same however the rows were asked +// for. func Ledger(p Period, f Filter) (rows []Row, sum Totals, agents []string) { return ledger(p.Since(time.Now()), f, Load(time.Time{})) } func ledger(since time.Time, f Filter, recs []Record) (rows []Row, sum Totals, agents []string) { renamed := provider.Renamed() + // the upstream names in force now, read once for the lot: a row is + // judged by the names standing today, which is what Ledger says, and + // the rows here are what a reader compares a bill against + wires := settings.Load().ModelWires priceOf := pricer() rows = []Row{} agents = []string{} @@ -92,7 +128,7 @@ func ledger(since time.Time, f Filter, recs []Record) (rows []Row, sum Totals, a } pr := priceOf(r) sum.add(r, pr) - row := Row{Record: r, Swapped: r.Served != "" && Swapped(r.Model, r.Served)} + row := Row{Record: r, Swapped: r.Served != "" && Swapped(provider.SentNameIn(wires, r.Provider, r.Model, r.Effort), r.Served)} if pr != nil && r.Input+r.Output > 0 { row.Cost, row.Priced = pr.Cost(r.Input, r.Output, r.CacheRead, r.CacheWrite), true } diff --git a/internal/usage/ledger_test.go b/internal/usage/ledger_test.go index 0f3d3aba..08077499 100644 --- a/internal/usage/ledger_test.go +++ b/internal/usage/ledger_test.go @@ -193,3 +193,60 @@ func TestLedgerUsesTheStatedPrice(t *testing.T) { t.Fatalf("row cost %v, total %v; want %v", rows[0].Cost, sum.Cost, want) } } + +// A row is judged against the id the call went out under, which on an +// Antigravity account is the variant of the model the effort picks and not +// the family it is one level of: a reply naming that variant is the model +// that was asked for, and a name given for the whole provider asks each +// level by its own id, which is the very name the reply names back. Read +// against the family instead, every level of it reads as another model +// having swapped in — in a report a user sets beside a bill. +func TestLedgerJudgesAnAntigravityCallByTheVariantItWentOutUnder(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) + t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache")) + // Antigravity's ids of one model at three levels, as its fetch left them + var raw []catalog.Model + for _, id := range []string{"flash-9-low", "flash-9-medium", "flash-9-high"} { + raw = append(raw, catalog.Model{ID: id}) + } + if err := catalog.SaveLive("antigravity", "", raw); err != nil { + t.Fatal(err) + } + catalog.Reset() + t.Cleanup(catalog.Reset) + if err := settings.Save(settings.Settings{ModelWires: map[string]string{"antigravity/*": "vendor-c/*"}}); err != nil { + t.Fatal(err) + } + os.MkdirAll(filepath.Dir(Path()), 0o755) + now := time.Now() + for i, c := range []struct{ effort, level string }{ + {"high", "high"}, {"low", "low"}, + // with no effort asked for the family thinks at its default, which is + // the highest of it + {"", "high"}, + } { + Append(Record{Time: now.Add(time.Duration(i) * time.Minute), Provider: "antigravity", + Host: "antigravity.example", Model: "flash-9", Effort: c.effort, + Served: "vendor-c/flash-9-" + c.level, Input: 10, Output: 1, Status: 200}) + } + // a level the call did not go out at is another model answering + Append(Record{Time: now.Add(4 * time.Minute), Provider: "antigravity", + Host: "antigravity.example", Model: "flash-9", Effort: "high", + Served: "vendor-c/flash-9-low", Input: 10, Output: 1, Status: 200}) + + rows, sum, _ := Ledger(All, Filter{}) + if len(rows) != 4 || sum.Calls != 4 { + t.Fatalf("rows %d, %+v", len(rows), rows) + } + // newest first: the swap, then the three levels + if !rows[0].Swapped { + t.Errorf("a relay that answered with another level's id is read as no swap: %+v", rows[0]) + } + for _, row := range rows[1:] { + if row.Swapped { + t.Errorf("the call went out at %q and was answered with %q, which is that very model: %+v", row.Effort, row.Served, row) + } + } +} diff --git a/model_cli.go b/model_cli.go index dfbf4a37..30f4cfdd 100644 --- a/model_cli.go +++ b/model_cli.go @@ -35,6 +35,15 @@ const modelUsage = `usage: magpie model output the most a reply of it may hold, and what you said magpie model output say the most, as 128000 or 128k; '/*' is every model magpie model output --reset take your limit off this model + magpie model wire the name the vendor is asked for, and the one you gave + magpie model wire ask for the model by this name, for a relay that serves it + magpie model wire '/*' ask for every model of that provider by this name; a * in + it is the model, so vendor-c/* asks for model-3 as + vendor-c/model-3. Quote it: a shell reads a bare * as a glob + magpie model wire --reset ask for it by the name magpie knows it by again, under an + id of its own for a provider that has since been deleted, + and says when there was no name of its own to take away + magpie model wires the names your vendors are asked for models by magpie model names the models you named or narrowed magpie model suffix [on|off] whether the agents' lists name each model with its provider (or "routing group") after it: on, as by default, "Sol · OpenAI"; @@ -71,12 +80,16 @@ func modelCmd(args []string) error { return modelPrice(args[1:]) case "prices": return modelPrices() + case "wires": + return modelWires() case "suffix", "suffixes": return modelSuffix(args[1:]) case "context", "ctx": return modelContext(args[1:]) case "output", "max-output": return modelOutput(args[1:]) + case "wire", "upstream": + return modelWire(args[1:]) case "help", "-h", "--help": fmt.Println(modelUsage) return nil @@ -449,6 +462,42 @@ func modelNames() error { return nil } +// modelWires lists the names the user gave the vendors their models are asked +// for, and which provider and model each is for. It is `magpie model prices` +// for the wire names: those are what a provider is asked for, and no other +// command says what is kept for a provider as a whole, so a name given for +// every model of one was written with no way to read it back. +// +// A name kept for a provider that has since been deleted is listed like any +// other: the name outlives the provider it was given for, and is in force for +// whichever provider takes that id next, so it is read off the file rather +// than off a provider that may not be there. +// +// A name that is only whitespace is no name and is not listed, as it is +// nowhere else a name is read: UpstreamNames leaves it out and a request +// under it goes out by the name magpie knows the model by. Listing it would +// print a line with nothing on it for a name no vendor is asked for, which +// is the one thing a listing read back off the file must not say. +func modelWires() error { + wires := settings.Load().ModelWires + rows := make([][2]string, 0, len(wires)) + w := 0 + for _, k := range slices.Sorted(maps.Keys(wires)) { + if n := strings.TrimSpace(wires[k]); n != "" { + rows = append(rows, [2]string{k, n}) + w = max(w, len(k)) + } + } + if len(rows) == 0 { + fmt.Println(muted.Render("no vendor is asked for a model by another name yet · magpie model wire ")) + return nil + } + for _, r := range rows { + fmt.Println(" " + pad(r[0], w) + " " + r[1]) + } + return nil +} + func modelContext(args []string) error { if len(args) == 0 { return fmt.Errorf("%s", modelUsage) @@ -681,3 +730,282 @@ func tokenCount(n int) string { return fmt.Sprintf("%d", n) } } + +func modelWire(args []string) error { + if len(args) == 0 { + return fmt.Errorf("%s", modelUsage) + } + // --reset comes before the provider is looked for, and works off the key + // the name is stored at: a name outlives the provider it was given for, + // and one whose provider has since been deleted is in force for + // whichever provider takes that id next, its requests going out renamed + // to a vendor that never heard of it. So it is the one name --reset + // most has to reach, and the only one a reset that resolved the + // provider first could not. + // + // What decides this is the key, not the lookup: a name under an id no + // provider has now is in force for whoever takes that id, and so is one + // under an id another provider is shown by — the provider such a ref + // resolves to holding a name of its own, which is the one a removal + // taken through it would take away instead. A ref that names its + // provider by that provider's own id keeps to the path below, which + // reports the removal as one. + if len(args) > 1 && isReset(args[1:]) { + pid, model, err := splitModelRef(args[0]) + if err != nil { + return err + } + p, ferr := provider.Find(pid) + if ferr != nil || (p.ID != pid && provider.HasUpstreamName(pid+"/"+model)) { + dropped, err := provider.DropUpstreamName(pid + "/" + model) + if err != nil { + return err + } + if dropped { + fmt.Println(green.Render("✓"), pid+"/"+model, + muted.Render("no longer has a name of its own; a provider taking that id later is asked for it by the name magpie knows it by")) + return nil + } + // no name is kept under that key, so the id is a mistyped one + // and not a deleted provider's; the lookup below says which + // provider is not there, as it does for every other command + } + } + p, model, err := modelRef(args[0]) + if err != nil { + return err + } + id := p.ID + "/" + model + every, shown := "'"+p.ID+"/*'", p.ID+"/"+model // quoted: a shell reads a bare * as a glob + if model == "*" { + shown = every + } + names := p.UpstreamNames() + _, own := names[model] + _, one := names["*"] + rest := args[1:] + asked := provider.UpstreamName(*p, model) + if len(rest) == 0 { + fmt.Println(bold.Render(asked), muted.Render("· what "+p.Name+" is asked for, for "+shown)) + switch { + case asked == model: + fmt.Println(faint.Render(" · the same name magpie knows it by · magpie model wire " + shown + " to change it")) + case model == "*": + fmt.Println(faint.Render(" · " + exampleOf(p, asked))) + fmt.Println(faint.Render(" · --reset takes it away · magpie model wire " + p.ID + "/ gives a single model one of its own")) + case own: + fmt.Println(faint.Render(" · magpie and the agents still know it as " + model + " · --reset asks for that again")) + case one: + fmt.Println(faint.Render(" · from the name given for every model of " + p.Name + " (" + every + " " + names["*"] + ")" + + " · --reset on it leaves that one in force")) + } + return nil + } + name := strings.Join(rest, " ") + if isReset(rest) { + name = "" + } + // A blank name is a removal, --reset or a name of only whitespace + // alike, and it is DropUpstreamName that makes it: it says whether + // there was a name there to take away, which with the provider's own + // list is what tells a model already asked for by the name magpie + // knows it by from one the provider does not serve at all, for which + // no name could have been kept. A ✓ over a key holding no name would + // read as a name being gone that never was there, and under a + // provider-wide name as the model's own name being the one in force, + // which it is not. So the removal is asked for rather than made blind, + // the way `magpie model price --reset` asks DropModelPrice. + if strings.TrimSpace(name) == "" { + return resetWireName(*p, model, id, shown) + } + if err := provider.SetUpstreamName(id, name); err != nil { + return err + } + now := provider.UpstreamName(*p, model) + if model == "*" { + // the same precedence the reset above is honest about: a model's + // own key wins over the provider's, so the models that keep one + // are not asked for under this, and a ✓ claiming every model of + // the provider is would be false wherever there is one. The + // example stands on a model this reaches, so the ones it does + // not are named here rather than left to be found one at a time. + scope, kept := "every model of "+p.Name, ownKeptSaid(p) + if kept != "" { + scope += " with no name of its own" + } + fmt.Println(green.Render("✓"), shown, muted.Render("· "+scope+" is asked for as"), bold.Render(now)) + fmt.Println(faint.Render(" · " + exampleOf(p, now))) + if kept != "" { + fmt.Println(faint.Render(" · " + kept)) + } + return nil + } + fmt.Println(green.Render("✓"), shown, muted.Render("is asked for as"), bold.Render(now), + faint.Render("· magpie and the agents still know it as "+model)) + return nil +} + +// resetWireName takes a name off under the key it is stored at and says what +// the model is asked for now, which is the name a wildcard entry leaves in +// force where one is in force at all. +// +// A key holding no name is an error rather than a tick, as it is for a price: +// a removal over a name that is not there has taken nothing away, and the +// tick would say the model is asked for by the name magpie knows it by — a +// claim nothing stands behind, and one that is false wherever a name given +// for the whole provider is in force. So there are three answers, as there +// are for a price: the model is not one the provider serves, so none could +// have been kept for it and none is in force; nothing was kept under that key +// of a model it does serve; or what is in force is the provider's own name +// and that is the one --reset reaches. +func resetWireName(p provider.Provider, model, id, shown string) error { + dropped, err := provider.DropUpstreamName(id) + if err != nil { + return err + } + now := provider.UpstreamName(p, model) + if !dropped { + if model != "*" && !provider.ServesModel(p, model) { + // none was kept and none could have been: a model the + // provider has nothing like is never asked for by any name, + // so naming the one in force would claim a request magpie + // does not make + return fmt.Errorf("%s is not a model of %s magpie knows of, so there is nothing to take away and none in force: a name is only kept for a model the provider serves, and this one is asked for by no name", id, p.Name) + } + every := "'" + p.ID + "/*'" + if provider.HasUpstreamName(p.ID + "/*") { + return fmt.Errorf("%s has no name of its own; what it is asked for is %s, the name given for every model of %s, which magpie model wire %s --reset takes away", + id, now, p.Name, every) + } + if model == "*" { + // the same precedence the success below is honest about: a + // model's own key wins over the provider's, so one that keeps + // a name is asked for under it whatever this pattern said, and + // "every model …" is false the moment there is one — so the + // models that keep a name are named here as they are there + if kept := ownKeptSaid(&p); kept != "" { + return fmt.Errorf("%s has no name of its own to reset, so there was nothing to take away: %s", id, kept) + } + return fmt.Errorf("%s has no name of its own to reset, so there was nothing to take away: every model of %s is already asked for by the name magpie knows it by", id, p.Name) + } + return fmt.Errorf("%s has no name of its own to reset, so there was nothing to take away: it is already asked for as %s", id, now) + } + if now != model { + fmt.Println(green.Render("✓"), shown, muted.Render("has no name of its own again · it is still asked for as"), + bold.Render(now), faint.Render("· that is the name given for '"+p.ID+"/*'")) + return nil + } + if model == "*" { + // the provider's own name is gone, but a model's own beats it + // while it is there, so the models that keep one are still asked + // for by it — and saying every model of the provider is asked + // for by the name magpie knows it by is false the moment one + // does, so they are named instead + if kept := ownKeptSaid(&p); kept != "" { + fmt.Println(green.Render("✓"), shown, muted.Render("has no name of its own again ·"), kept) + return nil + } + fmt.Println(green.Render("✓"), shown, muted.Render("· every model of "+p.Name+" is asked for by the name magpie knows it by again")) + return nil + } + fmt.Println(green.Render("✓"), shown, muted.Render("is asked for as"), bold.Render(model), muted.Render("again")) + return nil +} + +// ownKept are the models of a provider that keep a name of their own, in the +// order they are shown. A name given for every model of a provider does not +// reach them: a model's own key wins over the provider's, so each is asked +// for by its own whatever the provider's entry says, and a line about every +// model of that provider is true only where there is none to leave out of it. +// Naming them is what says which of the two is in force, which the example +// such a name is shown with cannot: that one stands on a model with no name +// of its own, or on the pattern alone, and names no model the provider's entry +// does not reach. +// +// A name kept for a model the provider does not serve is not one of these: +// magpie asks for no such a model, so nothing goes out under that name, and +// saying it did would claim a request that is not made. +func ownKept(p *provider.Provider) []string { + names := p.UpstreamNames() + kept := make([]string, 0, len(names)) + for _, m := range slices.Sorted(maps.Keys(names)) { + if m != "*" && provider.ServesModel(*p, m) { + kept = append(kept, m) + } + } + return kept +} + +// ownKeptSaid is ownKept said as a sentence, and is "" where there is none. +// +// A model is named with the name it goes out under, not with the one as it +// is stored: a star in a model's own name is that model, as it is in one +// given for every model of a provider (UpstreamNameIn), so it is the model +// magpie knows the model by that stands there. The whole claim of the +// sentence is that this is the name in force, so a star left standing in it +// would name an id no vendor is ever asked for. +func ownKeptSaid(p *provider.Provider) string { + names, kept := p.UpstreamNames(), ownKept(p) + askedFor := func(m string) string { return strings.ReplaceAll(names[m], "*", m) } + switch len(kept) { + case 0: + return "" + case 1: + return kept[0] + " is asked for as " + askedFor(kept[0]) + ", a name of its own" + } + // a relay with a name of its own for every model it serves would + // otherwise answer in a line as long as its list, so a few are named + // and the rest counted + const room = 3 + shown, more := kept, "" + if len(shown) > room { + shown, more = shown[:room], ", … and "+strconv.Itoa(len(kept)-room)+" more" + } + asked := make([]string, 0, len(shown)) + for _, m := range shown { + asked = append(asked, askedFor(m)) + } + return strings.Join(shown, ", ") + " are asked for as " + strings.Join(asked, ", ") + ", names of their own" + more +} + +// exampleOf is what a name given for a provider's every model asks for: a +// model of that provider for it to stand on, and what that model goes out as. +// The star in such a name is that model, so it says what it stands for. +// +// The model is one of Exposed, the list the user is shown, and not of +// Available, which is every model the vendor is known to serve: a relay +// fronting one vendor under ids of its own shares that vendor's catalogue, +// so its first model is one nobody is offered, and an example standing on it +// names a model the user cannot pick and the name was not given for. Exposed +// is the user's own list where the file has one, so that is what it is read +// from there. +// +// It is the first of them with no name of its own, and that is what the +// provider's own name is really in force for: a model's own key wins over the +// provider's, so a model given a name of its own is asked for by that one +// whatever is given for every model, and an example standing on it would say +// a name no request goes out under, which reads as the one thing the +// provider's own entry does not reach. +// +// A provider with nothing to stand one on — no catalogue, nothing fetched, +// nothing in the file, or a name of its own for every model it shows — is +// left with the pattern alone, which is all a relay naming none of its own +// can be told. +func exampleOf(p *provider.Provider, name string) string { + if !strings.Contains(name, "*") { + return p.Name + " is asked for every one of its models as " + name + } + const star = " · * is the model magpie knows each of them by" + names := p.UpstreamNames() + model := "" + for _, m := range p.Exposed() { + if _, own := names[m.ID]; !own { + model = m.ID + break + } + } + if model == "" { + return p.Name + " is asked for every one of its models as " + name + star + } + return p.ID + "/" + model + " is asked for as " + strings.ReplaceAll(name, "*", model) + star +} diff --git a/model_cli_test.go b/model_cli_test.go index 0da7913c..7fef675b 100644 --- a/model_cli_test.go +++ b/model_cli_test.go @@ -14,6 +14,58 @@ import ( "github.com/yetone/magpie/internal/settings" ) +// saidArgs is what a command printed, which is where the wire name is +// reported and what the messages about it have to be read from. It is named +// for its arguments rather than said so it does not collide with the said of +// a per-model limits or a price command, which runs a closure instead of a +// command line: they are three separate pull requests and the three of them +// in one tree must still be one package. +func saidArgs(t *testing.T, args ...string) string { + t.Helper() + out, err := refusedArgs(t, args...) + if err != nil { + t.Fatal(err) + } + return out +} + +// refusedArgs is what a command printed and the error it refused with, which +// is the pair a command that answers with an error instead of a line has to +// be read as: whether a tick was printed anyway is half of what a user sees +// of a refused command, and the other half is the exit code the error gives +// it, which is 1 for anything this returns (main). +func refusedArgs(t *testing.T, args ...string) (string, error) { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + old := os.Stdout + os.Stdout = w + var out strings.Builder + done := make(chan struct{}) + go func() { + b, _ := io.ReadAll(r) + out.Write(b) + close(done) + }() + err = modelCmd(args) + w.Close() + os.Stdout = old + <-done + r.Close() + return out.String(), err +} + +func mustFind(t *testing.T, id string) provider.Provider { + t.Helper() + p, err := provider.Find(id) + if err != nil { + t.Fatal(err) + } + return *p +} + // magpie model name and --reset keep and drop the user's name for a // provider's model, a name of several words included. func TestModelNameCmd(t *testing.T) { @@ -1123,3 +1175,911 @@ func TestModelLimitCmdSaysWhatALimitIsInTheVerbOfItsOwnCommand(t *testing.T) { } } } + +// magpie model wire keeps and drops the name a provider's models are asked +// for, and says what is in force after every change. A provider-wide name +// stands on a model that provider really serves rather than on an id made up +// for it, a name with no "*" in it is not explained with one, and a name of +// only whitespace is a name taken away, which is answered as the removal it +// is rather than as a setting that took. +func TestModelWireCmd(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + wires := func() map[string]string { return settings.Load().ModelWires } + + // a custom provider's every model, with the model in the name: the + // example is one of that provider's own, which is the only kind this + // exists for — a relay — and not an id no provider here serves + if out := saidArgs(t, "wire", "b/*", "vendor-c/*"); !strings.Contains(out, "b/vendor/m is asked for as vendor-c/vendor/m") { + t.Fatalf("naming every model said %q; want an example that is one of B's own", out) + } else if strings.Contains(out, "model-2") { + t.Fatalf("the example is a model no provider here serves: %q", out) + } + if n := wires()["b/*"]; n != "vendor-c/*" { + t.Fatalf("the name given for every model of B is %q; want vendor-c/*", n) + } + if got := provider.UpstreamName(mustFind(t, "b"), "gpt-5.5"); got != "vendor-c/gpt-5.5" { + t.Errorf("every model asked for as %q; want vendor-c/gpt-5.5", got) + } + + // the query explains the "*" only where there is one: a name with none + // asks for every model alike, and a sentence about the star would + // contradict the name printed above it + if out := saidArgs(t, "wire", "b/*", "vendor-c/one"); !strings.Contains(out, "B is asked for every one of its models as vendor-c/one") { + t.Errorf("naming every model alike said %q", out) + } + if out := saidArgs(t, "wire", "b/*"); !strings.Contains(out, "B is asked for every one of its models as vendor-c/one") || + strings.Contains(out, "* is the model magpie knows each of them by") { + t.Errorf("asking about a name with no star in it said %q; want no sentence about the star", out) + } + if out := saidArgs(t, "wire", "b/*", "vendor-c/*"); !strings.Contains(out, "* is the model magpie knows each of them by") { + t.Errorf("naming every model with a star in it said %q; want the star explained", out) + } + if out := saidArgs(t, "wire", "b/*"); !strings.Contains(out, "b/vendor/m is asked for as vendor-c/vendor/m") || + !strings.Contains(out, "* is the model magpie knows each of them by") { + t.Errorf("asking about a name with a star in it said %q", out) + } + + // one model of its own, which the provider's name cannot outrank + if out := saidArgs(t, "wire", "a/m", "vendor-c/m"); !strings.Contains(out, "is asked for as vendor-c/m") || + !strings.Contains(out, "magpie and the agents still know it as m") { + t.Errorf("naming one model said %q", out) + } + if n := wires()["a/m"]; n != "vendor-c/m" { + t.Fatalf("the name given for a/m is %q; want vendor-c/m", n) + } + if out := saidArgs(t, "wire", "a/m"); !strings.Contains(out, "vendor-c/m · what A is asked for, for a/m") || + !strings.Contains(out, "--reset asks for that again") { + t.Errorf("asking about one model said %q", out) + } + // a name of only whitespace takes the one away, and where nothing is in + // force for the model it is asked for by the name magpie knows it by + // again — not read as a name just given + if out := saidArgs(t, "wire", "a/m", " "); !strings.Contains(out, "is asked for as m again") || + strings.Contains(out, "magpie and the agents still know it as m") { + t.Errorf("a name of only whitespace said %q; want the removal it is", out) + } + if _, ok := wires()["a/m"]; ok { + t.Error("a name of only whitespace left one behind for a/m") + } + // a blank name on a model that never had one takes nothing away, and is + // refused as what it is: there is no name there to be taken + if out, err := refusedArgs(t, "wire", "a/claude-opus-5-5", " "); err == nil { + t.Errorf("a blank name on a model that never had one said %q", out) + } + if got := len(wires()); got != 1 { + t.Errorf("the names given are %v; a blank name is none", wires()) + } + + // a name given for every model of the provider, which a model with no + // name of its own falls back to and a blank name leaves in force + if err := provider.SetUpstreamName("a/*", "vendor-c/one"); err != nil { + t.Fatal(err) + } + if out := saidArgs(t, "wire", "a/only-a"); !strings.Contains(out, "from the name given for every model of A") { + t.Errorf("a model with no name of its own said %q; want the provider's", out) + } + if got := provider.UpstreamName(mustFind(t, "a"), "m"); got != "vendor-c/one" { + t.Errorf("with a/m taken away, m is asked for as %q; want the provider's vendor-c/one", got) + } + + // a name of only whitespace under the provider's is the removal of a + // name a/m never had of its own, and is refused — and what it says is + // the name that *is* in force, the provider's, which a tick over a key + // holding nothing would have left the user with the wrong idea of + out, err := refusedArgs(t, "wire", "a/m", " ") + if err == nil { + t.Fatalf("a name of only whitespace under the provider's said %q", out) + } + if !strings.Contains(err.Error(), "a/m has no name of its own") || + !strings.Contains(err.Error(), "vendor-c/one") { + t.Errorf("the refusal is %q; want a/m's own name missing and the provider's in force", err) + } + if _, ok := wires()["a/m"]; ok { + t.Error("a name of only whitespace left one behind for a/m") + } + + // --reset takes away only the model's own, leaving the provider's + if out := saidArgs(t, "wire", "a/m", "vendor-c/m"); !strings.Contains(out, "is asked for as vendor-c/m") { + t.Fatalf("naming a/m again said %q", out) + } + if out := saidArgs(t, "wire", "a/m", "--reset"); !strings.Contains(out, "has no name of its own again") { + t.Errorf("--reset said %q", out) + } + if _, ok := wires()["a/m"]; ok { + t.Error("--reset left a name for a/m") + } + if n := wires()["a/*"]; n != "vendor-c/one" { + t.Errorf("--reset on a model took the provider's %q away too", n) + } + if out := saidArgs(t, "wire", "a/*", "--reset"); !strings.Contains(out, "every model of A is asked for by the name magpie knows it by again") { + t.Errorf("--reset on every model said %q", out) + } + if got := len(wires()); got != 1 { + t.Errorf("the names given are %v; want only the ones this left", wires()) + } + + if err := modelCmd([]string{"wire", "m", "x"}); err == nil { + t.Error("named a model without its provider") + } +} + +// The example a name given for a provider's every model stands on is a +// model of that provider the user can actually see. Available is every +// model the vendor is known to serve, and a relay fronting one vendor under +// ids of its own shares that vendor's catalogue, so the first of those is a +// model nobody is offered: an example on it names a model that cannot be +// picked and shows the name standing for a model it was not given for. +func TestModelWireExampleIsAModelTheUserIsShown(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + // C lists two models of its own while its fetched list has a third + // first: the one only the fetched list knows is the one a user never + // sees, and the one an example read off that list would name. + if err := provider.Save(provider.Provider{ + ID: "c", Name: "C", Key: "kc", Models: []string{"shown-1", "shown-2"}, Chat: "http://127.0.0.1:1/v1", + }); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(catalog.LivePath("c")), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(catalog.LivePath("c"), []byte(`{"models":[ + {"id":"only-in-the-fetched-list"},{"id":"shown-1"},{"id":"shown-2"}]}`), 0o644); err != nil { + t.Fatal(err) + } + catalog.Reset() + t.Cleanup(catalog.Reset) + if ms := mustFind(t, "c").Available(); len(ms) != 3 || ms[0].ID != "only-in-the-fetched-list" { + t.Fatalf("what C is known to serve is %+v; want the fetched list's three, the unknown one first", ms) + } + if ms := mustFind(t, "c").Exposed(); len(ms) != 2 || ms[0].ID != "shown-1" { + t.Fatalf("what C shows is %+v; want the two of its own file", ms) + } + // both the line that sets the name and the one that asks about it stand + // a model on, so both have to be one the user is shown + for _, args := range [][]string{{"wire", "c/*", "vendor-c/*"}, {"wire", "c/*"}} { + out := saidArgs(t, args...) + if !strings.Contains(out, "c/shown-1 is asked for as vendor-c/shown-1") { + t.Errorf("%v said %q; want the example to stand on shown-1", args, out) + } + if strings.Contains(out, "only-in-the-fetched-list") { + t.Errorf("%v said %q; the example stands on a model C does not show", args, out) + } + } +} + +// The model a provider-wide name is shown with is one that name is really in +// force for. A model's own key wins over the provider's, so a model already +// given a name is asked for by that one however the provider's entry reads, +// and an example standing on it named a name no request goes out under — and +// named it right after the CLI had set one, which is what the user is +// reading about. So the example stands on a model with no name of its own, +// and a provider whose every model has one is left with the pattern alone. +func TestModelWireExampleStandsOnAModelWithNoNameOfItsOwn(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + // a relay of three models, of which the first is given a name of its + // own before the provider's own name is set + if err := provider.Save(provider.Provider{ + ID: "relay-a", Name: "Relay A", Key: "k", Chat: "http://127.0.0.1:1/v1", + Models: []string{"sol", "sol-preview", "sol-low"}, + }); err != nil { + t.Fatal(err) + } + if err := modelCmd([]string{"wire", "relay-a/sol", "vendor-c/sol-preview"}); err != nil { + t.Fatal(err) + } + // the name in force for sol is the one given for sol itself, and the + // example stands for the provider's + if out := saidArgs(t, "wire", "relay-a/sol"); !strings.Contains(out, "vendor-c/sol-preview") { + t.Fatalf("magpie model wire relay-a/sol said %q; want the name given for sol itself to be the one in force", out) + } + for _, args := range [][]string{{"wire", "relay-a/*", "vendor-c/*"}, {"wire", "relay-a/*"}} { + out := saidArgs(t, args...) + if !strings.Contains(out, "relay-a/sol-preview is asked for as vendor-c/sol-preview") { + t.Errorf("%v said %q; want the example to stand on sol-preview, the first model given no name of its own", args, out) + } + if strings.Contains(out, "relay-a/sol is asked for as ") { + t.Errorf("%v said %q; the example stands on sol, which is asked for by the name given for sol itself", args, out) + } + } + // a name of its own for every model the relay shows leaves nothing to + // stand one on, and the pattern is all that can be said of it + for _, m := range []string{"sol-preview", "sol-low"} { + if err := modelCmd([]string{"wire", "relay-a/" + m, "vendor-c/" + m}); err != nil { + t.Fatal(err) + } + } + out := saidArgs(t, "wire", "relay-a/*") + if want := "Relay A is asked for every one of its models as vendor-c/*"; !strings.Contains(out, want) { + t.Errorf("magpie model wire relay-a/* said %q; want the pattern alone, %q", out, want) + } + if strings.Contains(out, "is asked for as ") { + t.Errorf("magpie model wire relay-a/* said %q; every model it shows has a name of its own, so no model stands for the provider's", out) + } +} + +// --reset on a name given for every model of a provider takes that one away +// and nothing else: a model's own key wins over the provider's, so a model +// given a name of its own is asked for by that one however the provider's +// entry reads. A tick saying every model of the provider is asked for by the +// name magpie knows it by is therefore false the moment one model keeps a +// name, and leaves the reader with the opposite of what the command did — so +// the models that keep one are named with the name in force for each of them, +// and the claim is made only where there are none. +func TestModelWireResetWildcardNamesTheModelsThatKeepAName(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + + if err := modelCmd([]string{"wire", "a/m", "vendor-c/m"}); err != nil { + t.Fatal(err) + } + if err := modelCmd([]string{"wire", "a/*", "vendor-c/one"}); err != nil { + t.Fatal(err) + } + out := saidArgs(t, "wire", "a/*", "--reset") + if strings.Contains(out, "every model of A is asked for by the name magpie knows it by again") { + t.Errorf("magpie model wire a/* --reset said %q; a/m is asked for as vendor-c/m, not by the name magpie knows it by", out) + } + if !strings.Contains(out, "m is asked for as vendor-c/m, a name of its own") { + t.Errorf("magpie model wire a/* --reset said %q; want the model that keeps a name named with the one in force for it", out) + } + // and what it says is what the provider is asked for + if got := provider.UpstreamName(mustFind(t, "a"), "m"); got != "vendor-c/m" { + t.Errorf("after --reset on every model, m is asked for as %q; want the name given for m itself", got) + } + if got := provider.UpstreamName(mustFind(t, "a"), "only-a"); got != "only-a" { + t.Errorf("after --reset on every model, only-a is asked for as %q; want the name magpie knows it by", got) + } + if provider.HasUpstreamName("a/*") { + t.Error("--reset on every model left the provider's own name behind") + } + + // two models keeping one, where the answer is the plural and has to + // name both of them and not one of them + if err := modelCmd([]string{"wire", "a/only-a", "vendor-c/only-a"}); err != nil { + t.Fatal(err) + } + if err := modelCmd([]string{"wire", "a/*", "vendor-c/two"}); err != nil { + t.Fatal(err) + } + out = saidArgs(t, "wire", "a/*", "--reset") + if !strings.Contains(out, "m, only-a are asked for as vendor-c/m, vendor-c/only-a, names of their own") { + t.Errorf("magpie model wire a/* --reset said %q; want both the models that keep a name, each with its own", out) + } + + // a name kept for a model the provider does not serve is not one of + // them: magpie asks for no such a model, so nothing goes out under + // that name, and saying it did would claim a request that is not made + if err := provider.Save(provider.Provider{ + ID: "a", Name: "A", Key: "ka", Models: []string{"claude-opus-5-5"}, Chat: "http://127.0.0.1:1/v1", + }); err != nil { + t.Fatal(err) + } + if err := modelCmd([]string{"wire", "a/*", "vendor-c/three"}); err != nil { + t.Fatal(err) + } + out = saidArgs(t, "wire", "a/*", "--reset") + if !strings.Contains(out, "every model of A is asked for by the name magpie knows it by again") { + t.Errorf("magpie model wire a/* --reset said %q; the names left are for models A no longer serves, and it is asked for none of them", out) + } + if strings.Contains(out, "vendor-c/m") || strings.Contains(out, "vendor-c/only-a") { + t.Errorf("magpie model wire a/* --reset said %q; it names models A no longer serves, which magpie never asks for", out) + } +} + +// The refusal for a pattern with nothing left under it says every model of +// the provider is asked for by the name magpie knows it by, and that is false +// the moment one model keeps a name of its own: the model's own key wins over +// the provider's, so it is asked for under that one however the provider's +// entry reads. The second --reset over a provider-wide name is exactly where +// a reader is left with the opposite of what magpie did, so the models that +// keep a name are named with the name in force for each of them, as the tick +// over the same removal already does. +func TestModelWireResetWildcardRefusalNamesTheModelsThatKeepAName(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + wires := func() map[string]string { return settings.Load().ModelWires } + + if out := saidArgs(t, "wire", "a/*", "vendor-c/*"); !strings.Contains(out, "every model of A is asked for as") { + t.Fatalf("naming every model of A said %q", out) + } + if out := saidArgs(t, "wire", "a/m", "own-m"); !strings.Contains(out, "is asked for as own-m") { + t.Fatalf("naming a/m said %q", out) + } + if out := saidArgs(t, "wire", "a/*", "--reset"); !strings.Contains(out, "m is asked for as own-m, a name of its own") { + t.Fatalf("the first --reset on every model said %q", out) + } + + out, err := refusedArgs(t, "wire", "a/*", "--reset") + if err == nil { + t.Fatalf("the second --reset on every model said %q; the provider's own name was already taken away", out) + } + if strings.Contains(out, "✓") { + t.Errorf("a --reset with nothing under the key printed a tick: %q", out) + } + if strings.Contains(err.Error(), "every model of A is already asked for by the name magpie knows it by") { + t.Errorf("the second --reset says %q; m is asked for as own-m, a name of its own", err) + } + if !strings.Contains(err.Error(), "m is asked for as own-m, a name of its own") { + t.Errorf("the second --reset says %q; want the model that keeps a name named with the one in force for it", err) + } + // and the name the refusal is about is still standing, which is the + // whole of what makes the claim it does not make false + if n := wires()["a/m"]; n != "own-m" { + t.Errorf("m's name is %q after the second --reset; want own-m", n) + } + if provider.HasUpstreamName("a/*") { + t.Error("the second --reset put a name back for every model of A") + } +} + +// A name given for a model may stand for that model with a star in it, the +// way one given for every model of a provider does. So the sentence naming +// the models a provider's own entry does not reach has to say the name the +// request goes out under, and the star in it is the model magpie knows it +// by — a star left standing there would name an id no vendor is ever asked +// for, in the one line whose whole claim is that this is the name in force. +func TestModelWireNamesTheModelsWithAStarInTheirOwnName(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + + if err := modelCmd([]string{"wire", "a/m", "vendor-c/*"}); err != nil { + t.Fatal(err) + } + if got := provider.UpstreamName(mustFind(t, "a"), "m"); got != "vendor-c/m" { + t.Fatalf("m is asked for as %q; the test is not standing on a name with a star in it", got) + } + out := saidArgs(t, "wire", "a/*", "vendor-d/*") + if !strings.Contains(out, "m is asked for as vendor-c/m, a name of its own") { + t.Errorf("magpie model wire a/* vendor-d/* said %q; want m named with the name the request goes out under", out) + } + if strings.Contains(out, "vendor-c/*") { + t.Errorf("magpie model wire a/* vendor-d/* said %q; the star in a model's own name stands for the model, and no request goes out under it", out) + } + // and the same is said of a removal, which is the other way the + // sentence is reached + out = saidArgs(t, "wire", "a/*", "--reset") + if !strings.Contains(out, "m is asked for as vendor-c/m, a name of its own") { + t.Errorf("magpie model wire a/* --reset said %q; want m named with the name the request goes out under", out) + } + if strings.Contains(out, "vendor-c/*") { + t.Errorf("magpie model wire a/* --reset said %q; the star in a model's own name stands for the model, and no request goes out under it", out) + } + + // two models keeping one, each with a star of its own, so the plural + // expands each for the model it was given for rather than one for all + if err := modelCmd([]string{"wire", "a/only-a", "vendor-e/*"}); err != nil { + t.Fatal(err) + } + out = saidArgs(t, "wire", "a/*", "vendor-d/*") + if !strings.Contains(out, "m, only-a are asked for as vendor-c/m, vendor-e/only-a, names of their own") { + t.Errorf("magpie model wire a/* vendor-d/* said %q; want both names, each with the star stood for by its own model", out) + } +} + +// A name given for every model of a provider does not reach a model that has +// one of its own, and the example such a name is shown with deliberately +// stands on a model it does reach — which is what a ✓ claiming every model of +// that provider is asked for under it leaves out, in the one case where the +// precedence between the two is the thing the user needs told. So the line +// narrows to the models with no name of their own, and the ones left alone +// are named with the name in force for each. +func TestModelWireWildcardSaysTheModelsItDoesNotReach(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + + if err := modelCmd([]string{"wire", "a/m", "vendor-c/m"}); err != nil { + t.Fatal(err) + } + out := saidArgs(t, "wire", "a/*", "vendor-c/*") + if strings.Contains(out, "· every model of A is asked for as") { + t.Errorf("magpie model wire a/* vendor-c/* said %q; a/m keeps a name of its own and is not asked for under that one", out) + } + if !strings.Contains(out, "with no name of its own is asked for as vendor-c/*") { + t.Errorf("magpie model wire a/* vendor-c/* said %q; want the claim narrowed to the models it reaches", out) + } + if !strings.Contains(out, "m is asked for as vendor-c/m, a name of its own") { + t.Errorf("magpie model wire a/* vendor-c/* said %q; want the model it does not reach named with the name in force for it", out) + } + // the example still stands on a model the name does reach, so the two + // lines are about different models and neither contradicts the other + if !strings.Contains(out, "a/only-a is asked for as vendor-c/only-a") { + t.Errorf("magpie model wire a/* vendor-c/* said %q; want the example on a model the name does reach", out) + } + // and where no model of the provider keeps a name, the claim is the + // whole of it + if out := saidArgs(t, "wire", "b/*", "vendor-c/*"); !strings.Contains(out, "every model of B is asked for as vendor-c/*") { + t.Errorf("magpie model wire b/* vendor-c/* said %q; no model of B keeps a name, so every one of them is asked for under it", out) + } +} + +// The wire names are the one thing a listing of what the user has said does +// not show: `magpie model names` is a list of what the models are called and +// of the levels they offer, both of which are magpie's own, and a relay +// serving a model under an id of its own changes neither — so a name given +// for every model of one was written with no command that read it back. +// `magpie model wires` is that listing, and it is read off the file rather +// than off a provider, since a name outlives the provider it was given for +// and is in force for whichever provider takes that id next. +func TestModelWiresCmd(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + wires := func() map[string]string { return settings.Load().ModelWires } + listed := func(t *testing.T) string { + t.Helper() + out, err := said(t, func() error { return modelCmd([]string{"wires"}) }) + if err != nil { + t.Fatal(err) + } + return out + } + + if out := listed(t); !strings.Contains(out, "no vendor is asked for a model by another name yet") { + t.Errorf("magpie model wires with nothing given said %q; want it to say there is none", out) + } + for _, ref := range [][2]string{{"a/m", "vendor-c/m"}, {"b/*", "vendor-c/one"}} { + if err := modelCmd([]string{"wire", ref[0], ref[1]}); err != nil { + t.Fatal(err) + } + } + out := listed(t) + // a name is listed with the key it is stored at, on a line of its own: + // a model's own name and the provider's are two entries, and merging + // them would read as the one in force for a model that has its own + for _, ref := range [][2]string{{"a/m", "vendor-c/m"}, {"b/*", "vendor-c/one"}} { + lines := 0 + for _, l := range strings.Split(out, "\n") { + if !strings.Contains(l, ref[0]) { + continue + } + lines++ + if !strings.Contains(l, ref[1]) { + t.Errorf("magpie model wires put %q on a line of its own, %q: %q; want the name with the key it is stored at", ref[0], ref[1], out) + } + } + if lines != 1 { + t.Errorf("magpie model wires listed %q on %d lines, %q; want it once", ref[0], lines, out) + } + } + // a name outlives the provider it was given for, so it is still listed + // once that provider is gone — it is in force for whichever provider + // takes that id next + if err := provider.Delete("b"); err != nil { + t.Fatal(err) + } + if out := listed(t); !strings.Contains(out, "b/*") || !strings.Contains(out, "vendor-c/one") { + t.Errorf("magpie model wires after b was deleted said %q; want the name it outlived listed still", out) + } + if len(wires()) != 2 { + t.Errorf("the names kept are %v; a listing takes none away", wires()) + } +} + +// A name that is only whitespace is no name, as it is where one is given and +// where one is looked up: UpstreamNames leaves it out and a request under it +// goes out by the name magpie knows the model by. So the listing reads it the +// way the file means it, one an older magpie stored or one hand-edited in, +// rather than printing a line with nothing on it — a listing whose whole +// purpose is reading the names back cannot be the one place a name shows up +// that no vendor is ever asked for by. +func TestModelWiresCmdSkipsANameOfOnlyWhitespace(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + // magpie will not write one itself — a name of only whitespace is + // taken as a removal — so it is the file that is read, as one left by + // an older magpie or edited by hand is + s := settings.Load() + s.ModelWires = map[string]string{"a/m": " vendor-c/m ", "a/only-a": " "} + if err := settings.Save(s); err != nil { + t.Fatal(err) + } + out := saidArgs(t, "wires") + listed := false + for _, l := range strings.Split(out, "\n") { + if strings.Contains(l, "only-a") { + t.Errorf("magpie model wires listed %q; a name of only whitespace is no name, and no request goes out under one", l) + } + if strings.Contains(l, "a/m") { + listed = true + // the name ends the line, so what the file stored around + // it is not in force and is not what a request goes out as + if !strings.HasSuffix(l, "vendor-c/m") { + t.Errorf("magpie model wires said %q; want the name listed as it will be sent, trimmed", out) + } + } + } + if !listed { + t.Errorf("magpie model wires said %q; want the name that is in force listed", out) + } + + // and where every name in the file is one of those, the listing says + // there is none rather than printing nothing at all + s.ModelWires = map[string]string{"a/m": " "} + if err := settings.Save(s); err != nil { + t.Fatal(err) + } + if out := saidArgs(t, "wires"); !strings.Contains(out, "no vendor is asked for a model by another name yet") { + t.Errorf("magpie model wires with nothing but whitespace names said %q; want it to say no vendor is asked for one by another name", out) + } +} + +// --reset takes a name away, and where there is no name to take away the +// model is already asked for by the name magpie knows it by, or by the name +// its provider's own entry puts in force. A tick over either would report a +// removal that did not happen — and under a provider-wide name it would +// leave the reader with the one thing that is false, that the model's own +// name is the one in force. So a removal with nothing under the key is +// refused, as `magpie model price --reset` refuses a price that was never +// set, and the refusal says which of the two the model is asked for by. An +// error here is a command that exits 1. +func TestModelWireResetRefusesANameThereIsNot(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + wires := func() map[string]string { return settings.Load().ModelWires } + + // the second --reset of a name that is gone: the first took it, so this + // one has nothing left to do + if out := saidArgs(t, "wire", "a/m", "vendor-c/m"); !strings.Contains(out, "is asked for as vendor-c/m") { + t.Fatalf("naming a/m said %q", out) + } + if out := saidArgs(t, "wire", "a/m", "--reset"); !strings.Contains(out, "is asked for as m again") { + t.Fatalf("the first --reset said %q", out) + } + out, err := refusedArgs(t, "wire", "a/m", "--reset") + if err == nil { + t.Fatalf("--reset over a name already taken away said %q; there was none to take", out) + } + if strings.Contains(out, "✓") { + t.Errorf("--reset over no name at all printed a tick: %q", out) + } + if !strings.Contains(err.Error(), "a/m has no name of its own to reset") { + t.Errorf("the second --reset says %q; want it to say a/m had no name of its own", err) + } + + // A model the provider does not serve is one magpie never asks for by + // any name, so there is none to take away — and saying which name it is + // asked for under would claim a request magpie does not make, which is + // the one thing a mistyped id must not be answered with. It is refused + // as not one of the provider's models, and the same id is refused when + // a name is given for it, which is what the README says a name is + // given only for. + out, err = refusedArgs(t, "wire", "a/nosuch", "--reset") + if err == nil { + t.Fatalf("--reset on a model A does not serve said %q; A is never asked for a nosuch", out) + } + if strings.Contains(out, "✓") { + t.Errorf("--reset on a model A does not serve printed a tick: %q", out) + } + if !strings.Contains(err.Error(), "a/nosuch is not a model of A magpie knows of") { + t.Errorf("--reset on a model A does not serve says %q; want it to say magpie knows no such model", err) + } + if strings.Contains(err.Error(), "already asked for as") { + t.Errorf("--reset on a model A does not serve says %q; that model is asked for by no name at all", err) + } + if _, ok := wires()["a/nosuch"]; ok { + t.Error("a refused --reset left a name behind") + } + if out, err := refusedArgs(t, "wire", "a/nosuch", "vendor-c/nosuch"); err == nil { + t.Errorf("a name was given for a model A does not serve: %q", out) + } + if _, ok := wires()["a/nosuch"]; ok { + t.Error("a name for a model A does not serve was stored anyway") + } + + // the same over the provider's own name, which is a name of its own + if out := saidArgs(t, "wire", "a/*", "vendor-c/one"); !strings.Contains(out, "every model of A is asked for as vendor-c/one") { + t.Fatalf("naming every model of A said %q", out) + } + if out := saidArgs(t, "wire", "a/*", "--reset"); !strings.Contains(out, "every model of A is asked for by the name magpie knows it by again") { + t.Fatalf("the first --reset on every model said %q", out) + } + out, err = refusedArgs(t, "wire", "a/*", "--reset") + if err == nil { + t.Fatalf("--reset over the provider's own name, twice, said %q", out) + } + if strings.Contains(out, "✓") { + t.Errorf("--reset over no name at all printed a tick: %q", out) + } + if !strings.Contains(err.Error(), "a/* has no name of its own to reset") || + !strings.Contains(err.Error(), "every model of A is already asked for by the name magpie knows it by") { + t.Errorf("the second --reset on every model says %q", err) + } + + // a model with only the provider's name over it: --reset reaches the + // model's own and nothing else, so the one that takes this away is the + // 'a/*' one, and it is named rather than left to a tick over a key that + // was never set + if err := provider.SetUpstreamName("a/*", "vendor-c/one"); err != nil { + t.Fatal(err) + } + out, err = refusedArgs(t, "wire", "a/m", "--reset") + if err == nil { + t.Fatalf("--reset on a model with no name of its own said %q", out) + } + if !strings.Contains(err.Error(), "a/m has no name of its own") || + !strings.Contains(err.Error(), "magpie model wire 'a/*' --reset") { + t.Errorf("the refusal is %q; want the 'a/*' --reset that takes the name in force away", err) + } + if strings.Contains(out, "✓") { + t.Errorf("--reset with only the provider's name in force printed a tick: %q", out) + } + if n := wires()["a/*"]; n != "vendor-c/one" { + t.Errorf("the provider's own name is %q; a refused --reset took it away", n) + } + // and what is in force for that model is what the refusal says, so a + // user who meant to take that name away is sent to the one command + // that does take it + if got := provider.UpstreamName(mustFind(t, "a"), "m"); got != "vendor-c/one" { + t.Errorf("with only the provider's name in force, m is asked for as %q", got) + } + // a model A does not serve is outside all of that: the provider's own + // name reaches every model of A, and a model that is not one of them is + // asked for by it no more than by a name of its own, so the answer is + // the one about magpie not knowing the model rather than the name in + // force for the others + out, err = refusedArgs(t, "wire", "a/nosuch", "--reset") + if err == nil { + t.Fatalf("--reset on a model A does not serve, under the provider's own name, said %q", out) + } + if !strings.Contains(err.Error(), "a/nosuch is not a model of A magpie knows of") || + strings.Contains(err.Error(), "vendor-c/one") { + t.Errorf("the refusal is %q; want magpie knowing no such model, not the name in force", err) + } + if out := saidArgs(t, "wire", "a/*", "--reset"); !strings.Contains(out, "every model of A is asked for by the name magpie knows it by again") { + t.Errorf("--reset on every model said %q", out) + } + if got := len(wires()); got != 0 { + t.Errorf("the names left are %v; every one of them was taken away", wires()) + } +} + +// A wire name outlives the provider it was given for: that provider can be +// deleted, and the name is then in force for whichever provider takes that +// id next, its models going out renamed to a vendor that never heard of +// them, with nothing saying so. --reset therefore works off the key the +// name is stored at rather than off a provider that is not there any more — +// the one name most in need of taking off is the one a reset that resolved +// the provider first could not take off at all. +func TestModelWireResetAfterTheProviderIsGone(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + wires := func() map[string]string { return settings.Load().ModelWires } + + if out := saidArgs(t, "wire", "b/vendor/m", "vendor-c/m"); !strings.Contains(out, "is asked for as vendor-c/m") { + t.Fatalf("naming one of b's models said %q", out) + } + if err := provider.Delete("b"); err != nil { + t.Fatal(err) + } + // deleting the provider takes the models out of the catalog, not the + // name out of the file: the id is free again, and whoever takes it next + // would be asked for its models under the old vendor's name + if n := wires()["b/vendor/m"]; n != "vendor-c/m" { + t.Fatalf("with b deleted the name under b/vendor/m is %q; the point is that one is still there", n) + } + + if out := saidArgs(t, "wire", "b/vendor/m", "--reset"); !strings.Contains(out, "b/vendor/m") || + !strings.Contains(out, "no longer has a name of its own") { + t.Errorf("--reset with the provider gone said %q", out) + } + if left := wires(); len(left) != 0 { + t.Errorf("--reset left the names behind: %v", left) + } + + // a provider that takes the id afterwards is asked for its models by + // the names magpie knows them by, which is the whole of what clearing + // the name buys + if err := provider.Save(provider.Provider{ID: "b", Name: "B", Key: "kb", + Models: []string{"vendor/m"}, Chat: "http://127.0.0.1:1/v1"}); err != nil { + t.Fatal(err) + } + if got := provider.UpstreamName(mustFind(t, "b"), "vendor/m"); got != "vendor/m" { + t.Errorf("the new b is asked for its model as %q; want vendor/m", got) + } + + // an id that is mistyped rather than a deleted provider's is still the + // error it was: there is no name under it to take away, and the user is + // told which provider is missing rather than that nothing was there + if err := modelCmd([]string{"wire", "bb/vendor/m", "--reset"}); err == nil { + t.Error("--reset on a provider that never existed was taken as a removal") + } +} + +// The id a wire name is stored under is free again once its provider is +// deleted, and another provider can be shown by it — the name a provider is +// shown by is a spelling any other provider answers to as well. So with b's +// provider gone and a provider c shown as B, `wire b/vendor/m --reset` still +// has to take the name off b/vendor/m, which is in force for whichever +// provider takes that id next, going out renamed to a vendor that never +// heard of it. Resolving the ref first lands on c, where what a reset would +// remove is c's own name — one that is really in force — and the name left +// behind is the one doing the damage, with nothing having said so. +func TestModelWireResetUnderAProviderShownAsTheGoneId(t *testing.T) { + groupsHome(t) + catalog.Changed = nil // no agent's files are rewritten here + wires := func() map[string]string { return settings.Load().ModelWires } + + if out := saidArgs(t, "wire", "b/vendor/m", "vendor-c/m"); !strings.Contains(out, "is asked for as vendor-c/m") { + t.Fatalf("naming one of b's models said %q", out) + } + if err := provider.Delete("b"); err != nil { + t.Fatal(err) + } + // a provider whose own id is c, shown by the id b that is now nobody's, + // with a name of its own that a reset has to leave in force + if err := provider.Save(provider.Provider{ID: "c", Name: "B", Key: "kc", + Models: []string{"vendor/m"}, Chat: "http://127.0.0.1:1/v1"}); err != nil { + t.Fatal(err) + } + if err := provider.SetUpstreamName("c/vendor/m", "vendor-d/m"); err != nil { + t.Fatal(err) + } + + if out := saidArgs(t, "wire", "b/vendor/m", "--reset"); !strings.Contains(out, "b/vendor/m") || + !strings.Contains(out, "no longer has a name of its own") { + t.Errorf("--reset with the id b shown by another provider said %q", out) + } + if _, ok := wires()["b/vendor/m"]; ok { + t.Errorf("b/vendor/m still has a name, %v; it is in force for whichever provider takes that id next", wires()) + } + if got := wires()["c/vendor/m"]; got != "vendor-d/m" { + t.Errorf("c's own name is %q; --reset took away the name c's model is asked for", got) + } +} + +// The help is two columns, and every description starts in the second of +// them — a continuation line included. A description a column early lines +// its text up under the line above rather than under the command it belongs +// to, which is how the wire lines read until they were put in line. +func TestModelUsageLinesUpItsDescriptions(t *testing.T) { + const cmd = " magpie model " + // where a description starts: past the command and the run of spaces + // after it, or — for a wrapped description — at its own first letter + startsAt := func(l string) (int, bool) { + if !strings.HasPrefix(l, cmd) { + return 0, false + } + rest := l[len(cmd):] + i := strings.Index(rest, " ") + if i < 0 { + return 0, false + } + return len(cmd) + i + len(rest[i:]) - len(strings.TrimLeft(rest[i:], " ")), true + } + // a line that is neither a command nor prose is a description wrapped + // onto a line of its own, and starts where those start + wrappedAt := func(l string) (int, bool) { + lead := len(l) - len(strings.TrimLeft(l, " ")) + return lead, strings.TrimSpace(l) != "" && lead > len(cmd) + } + + want, laid := -1, 0 + for _, l := range strings.Split(modelUsage, "\n") { + at, ok := startsAt(l) + if !ok { + at, ok = wrappedAt(l) + } + if !ok { + continue // a paragraph of prose, which is in no column + } + if want < 0 { + want = at + } + if at != want { + t.Errorf("the description on %q starts at column %d; the rest start at %d", l, at, want) + } + laid++ + } + if laid < 11 { + t.Fatalf("only %d lines of the help were laid out; the check is not reading it", laid) + } +} + +// The columns line up whatever a description belongs to, so a command with a +// description another command's line finishes is laid out perfectly and +// reads as though the two of them said the same thing: the wire lines all +// carried on into the stack under them, and which of the three each +// sentence went with was anybody's guess. What says who a wrapped line +// belongs to is that it follows its own command, so this reads the block in +// the groups a reader takes it in — a command and the lines wrapped under +// it — and fails on a group whose description stops on a word no sentence +// ends on, which is what a description whose continuation went to another +// command looks like. +// +// The gap after a command is not always two spaces: a command as long as the +// column the descriptions are laid out in leaves one, and to a check that +// counts spaces that reads as a command with no description at all. So the +// column is read off the block rather than written down, and such a line's +// description is taken from the first space past it — which is where the +// layout puts it, and which a reader reads as readily as any other line. +func TestModelUsageGivesEveryCommandItsOwnDescription(t *testing.T) { + const cmd = " magpie model " + // words a sentence cannot end on, which is what a description is left + // reading as when the line it carried on into was somebody else's. A + // preposition like "by" is not one of them: a description can end on it + // and does — the name a model goes by. + dangling := map[string]bool{ + "and": true, "or": true, "but": true, "in": true, "of": true, + "to": true, "for": true, "as": true, "with": true, "that": true, + "which": true, "a": true, "an": true, "the": true, + } + lines := strings.Split(modelUsage, "\n") + // the column the descriptions are laid out in: the leftmost one a + // command with room for a gap reaches. It is read off the block rather + // than written down, so the layout can move and still be read. + col := -1 + gapAt := func(l string) int { + i := strings.Index(l[len(cmd):], " ") + if i < 0 { + return -1 + } + k := len(cmd) + i + for k < len(l) && l[k] == ' ' { + k++ + } + return k + } + for _, l := range lines { + if !strings.HasPrefix(l, cmd) { + continue + } + if k := gapAt(l); k >= 0 && (col < 0 || k < col) { + col = k + } + } + // a command with no description beside it carries its words onto the + // next line, and that is what tells such a line from one whose command + // is as long as the column and its description is pushed past it + wrapped := func(i int) bool { + return col >= 0 && i+1 < len(lines) && strings.TrimSpace(lines[i+1]) != "" && + len(lines[i+1])-len(strings.TrimLeft(lines[i+1], " ")) == col + } + type group struct{ command, said string } + var groups []group + laid := 0 + for i, l := range lines { + switch { + case strings.HasPrefix(l, cmd): + // the command, and the description that starts past the run + // of spaces after it — the column the block is laid out in, + // or, for a command that fills that column, the one space it + // leaves before the description + g := group{command: strings.TrimSpace(l)} + rest := l[len(cmd):] + switch k := gapAt(l); { + case k >= 0: + g.command, g.said = strings.TrimSpace(rest[:k-len(cmd)]), strings.TrimSpace(l[k:]) + case col >= 0 && col < len(l) && !wrapped(i): + if k := col + strings.Index(l[col:], " "); k >= 0 { + g.command, g.said = strings.TrimSpace(rest[:k-len(cmd)]), strings.TrimSpace(l[k+1:]) + } + } + groups = append(groups, g) + laid++ + + case strings.HasPrefix(l, strings.Repeat(" ", len(cmd)+1)): + // a description wrapped onto a line of its own, which belongs + // to the command above it and to no other + if len(groups) == 0 { + t.Fatalf("a description is wrapped onto a line of its own with no command above it: %q", l) + } + groups[len(groups)-1].said += " " + strings.TrimSpace(l) + laid++ + } + } + if laid < 11 { + t.Fatalf("only %d lines of the help were read; the check is not reading it", laid) + } + for _, g := range groups { + if g.said == "" { + t.Errorf("%s is a command with no description of its own", g.command) + continue + } + last := g.said[strings.LastIndex(g.said, " ")+1:] + if dangling[last] { + t.Errorf("%s is described as %q, which stops on %q: its continuation is under another command", g.command, g.said, last) + } + } +}