catalog: a model the provider's catalog doesn't list is named as the other providers name it (John on Discord #feedback: GLM Coding Plan and ZCode name the same models in different case, in the provider list and the group picker) — the Coding Plan's /models (open.bigmodel.cn and api.z.ai, read with a real key) gives ids alone, glm-5-turbo, glm-5.3-flashx …; catalog.Decorate names only those Zhipu's models.dev entry (zhipuai) lists, so glm-5-turbo kept its id as its name beside ZCode's GLM-5-Turbo. models.dev's load now keeps each model's name as most providers serving it give it (a name that is only the id casts no vote), catalog.NameOf reads it by bare id in any case, and catalog.Named fills it in for a model still named by its id: in Available's live list and for a pick the list doesn't have. Ids, and what is sent upstream, are unchanged; a name the user gave stays theirs (Catalog's Name, with the new one as Default); an Azure provider is left out, its deployments named as their owner named them. TestModelNamedAsElsewhere (the Coding Plan's real ids, a user name, an Azure deployment) fails without the provider change and passes; TestNameOf is new. TestSearcherChosen read whatever catalog an earlier test had left loaded, and now resets it: with none, its lists' models go by their ids, as it expects. go test ./internal/catalog/ ./internal/provider/ ./internal/gateway/ ./internal/gui/ . pass; -race -count=5 for the two new tests; go vet, linux(nogui)/windows builds pass. No GUI change (the GUI and the group picker show name || id).
This commit is contained in:
1 parent
b0558485fb
commit
e0bf3e61ed
6 files changed
+210
-4
No files matched your search
+4
-1
@@ -59,7 +59,10 @@ line; agents connected to magpie lose it when it quits.
|
||||
- **Real model lists, nothing compiled in.** With a key in hand magpie asks
|
||||
the vendor which models it serves and offers exactly those; the
|
||||
[models.dev](https://models.dev) catalog fills in names, reasoning efforts
|
||||
and the list for vendors that have none, and refreshes itself in the
|
||||
and the list for vendors that have none (a model the vendor's own entry
|
||||
doesn't list is named as the other providers serving it name it, so
|
||||
glm-5-turbo reads GLM-5-Turbo under Zhipu as under ZCode; an Azure
|
||||
deployment keeps its own name), and refreshes itself in the
|
||||
background once it goes stale. Choose which models each provider exposes,
|
||||
or expose them all — a model released this morning is in the picker on
|
||||
the next refresh.
|
||||
|
||||
@@ -327,6 +327,9 @@ var (
|
||||
// thinks are the models, by bare id, most of the providers serving
|
||||
// them say reason, levels or not
|
||||
thinks map[string]bool
|
||||
// names are the models' names, by bare id, as most of the providers
|
||||
// serving them give it
|
||||
names map[string]string
|
||||
|
||||
syncMu sync.Mutex
|
||||
)
|
||||
@@ -364,6 +367,7 @@ func load() map[string]mdProvider {
|
||||
if json.Unmarshal(b, &m) == nil && len(m) > 0 {
|
||||
mdev = m
|
||||
votes, reasons := map[string]int{}, map[string]int{}
|
||||
named := map[string]map[string]int{}
|
||||
sizes, outs := map[string]map[int]int{}, map[string]map[int]int{}
|
||||
levels := map[string]map[string]int{}
|
||||
// one vote a provider for each list it gives a model: a
|
||||
@@ -374,6 +378,12 @@ func load() map[string]mdProvider {
|
||||
for pid, p := range m {
|
||||
for id, x := range p.Models {
|
||||
OneHourFor(id, x.Cost)
|
||||
if x.Name != "" && x.Name != id && x.Name != bareID(id) {
|
||||
if named[bareID(id)] == nil {
|
||||
named[bareID(id)] = map[string]int{}
|
||||
}
|
||||
named[bareID(id)][x.Name]++
|
||||
}
|
||||
if e := x.efforts(); len(e) > 0 {
|
||||
l := strings.Join(e, ",")
|
||||
if levels[bareID(id)] == nil {
|
||||
@@ -420,6 +430,10 @@ func load() map[string]mdProvider {
|
||||
for id, by := range levels {
|
||||
efforts[id] = strings.Split(mostListed(by), ",")
|
||||
}
|
||||
names = map[string]string{}
|
||||
for id, by := range named {
|
||||
names[id] = mostNamed(by)
|
||||
}
|
||||
thinks = map[string]bool{}
|
||||
for id, v := range reasons {
|
||||
if v > 0 {
|
||||
@@ -445,7 +459,7 @@ func Reset() {
|
||||
loadMu.Lock()
|
||||
defer loadMu.Unlock()
|
||||
loaded = false
|
||||
mdev, images, windows, outputs, efforts, thinks = nil, nil, nil, nil, nil, nil
|
||||
mdev, images, windows, outputs, efforts, thinks, names = nil, nil, nil, nil, nil, nil, nil
|
||||
}
|
||||
|
||||
// Sync downloads the models.dev catalog into CachePath. It serializes with
|
||||
@@ -622,6 +636,32 @@ func Thinks(id string) bool {
|
||||
return thinks[bareID(id)]
|
||||
}
|
||||
|
||||
// NameOf is the name models.dev gives a model of this id, as most of the
|
||||
// providers it lists serving it do, matched as SeesImages matches it:
|
||||
// without a vendor's prefix, in any case. "" when no provider names it
|
||||
// other than by its id.
|
||||
func NameOf(id string) string {
|
||||
load()
|
||||
return names[bareID(id)]
|
||||
}
|
||||
|
||||
// Named gives a model the list names by its id alone the name models.dev
|
||||
// knows it by (NameOf), so one model reads the same under every provider:
|
||||
// a GLM Coding Plan's glm-5-turbo, which Zhipu's catalog doesn't list, is
|
||||
// GLM-5-Turbo as ZCode's is. Its id, and where requests go, stay as they
|
||||
// are.
|
||||
func Named(ms []Model) []Model {
|
||||
out := slices.Clone(ms)
|
||||
for i, m := range out {
|
||||
if m.Name == "" || m.Name == m.ID {
|
||||
if n := NameOf(m.ID); n != "" {
|
||||
out[i].Name = n
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Knows reports whether models.dev lists a model of this id at all, under
|
||||
// any provider, as ContextOf and EffortsOf match it.
|
||||
func Knows(id string) bool {
|
||||
@@ -769,6 +809,18 @@ func mostListed(by map[string]int) string {
|
||||
return best
|
||||
}
|
||||
|
||||
// mostNamed is the name most providers give; a tie goes to the shorter,
|
||||
// then the first in order, so the answer doesn't change from run to run.
|
||||
func mostNamed(by map[string]int) string {
|
||||
best, n := "", 0
|
||||
for s, c := range by {
|
||||
if c > n || c == n && (len(s) < len(best) || len(s) == len(best) && s < best) {
|
||||
best, n = s, c
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// mostGiven is the size most providers give; a tie goes to the smaller,
|
||||
// which a prompt fits either way.
|
||||
func mostGiven(by map[int]int) int {
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A model is named as most of the providers serving it name it, matched
|
||||
// without a vendor's prefix and in any case; a name that is only the id
|
||||
// casts no vote.
|
||||
func TestNameOf(t *testing.T) {
|
||||
writeCatalog(t, `{
|
||||
"zai": {"models": {"glm-5-turbo": {"id":"glm-5-turbo","name":"GLM-5-Turbo"}}},
|
||||
"zai-coding-plan": {"models": {"glm-5-turbo": {"id":"glm-5-turbo","name":"GLM-5-Turbo"}}},
|
||||
"a": {"models": {"z-ai/glm-5-turbo": {"id":"z-ai/glm-5-turbo","name":"GLM 5 Turbo"}}},
|
||||
"b": {"models": {"glm-5-turbo": {"id":"glm-5-turbo","name":"glm-5-turbo"}},
|
||||
"plain": {"id":"plain","name":"plain"}}
|
||||
}`)
|
||||
for id, want := range map[string]string{
|
||||
"glm-5-turbo": "GLM-5-Turbo",
|
||||
"GLM-5-Turbo": "GLM-5-Turbo",
|
||||
"z-ai/glm-5-turbo": "GLM-5-Turbo",
|
||||
"plain": "",
|
||||
"unknown": "",
|
||||
} {
|
||||
if got := NameOf(id); got != want {
|
||||
t.Errorf("NameOf(%q) = %q, want %q", id, got, want)
|
||||
}
|
||||
}
|
||||
in := []Model{
|
||||
{ID: "glm-5-turbo", Name: "glm-5-turbo"},
|
||||
{ID: "GLM-5-Turbo"},
|
||||
{ID: "glm-5-turbo", Name: "Turbo"}, // the list's own name stays
|
||||
{ID: "plain", Name: "plain"},
|
||||
}
|
||||
var got []string
|
||||
for _, m := range Named(in) {
|
||||
got = append(got, m.ID+"|"+m.Name)
|
||||
}
|
||||
want := "glm-5-turbo|GLM-5-Turbo\nGLM-5-Turbo|GLM-5-Turbo\nglm-5-turbo|Turbo\nplain|plain"
|
||||
if strings.Join(got, "\n") != want {
|
||||
t.Fatalf("got\n%s\nwant\n%s", strings.Join(got, "\n"), want)
|
||||
}
|
||||
if in[0].Name != "glm-5-turbo" {
|
||||
t.Fatalf("the list given was changed: %q", in[0].Name)
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/yetone/magpie/internal/catalog"
|
||||
"github.com/yetone/magpie/internal/provider"
|
||||
"github.com/yetone/magpie/internal/settings"
|
||||
)
|
||||
@@ -20,6 +21,9 @@ func TestSearcherChosen(t *testing.T) {
|
||||
t.Setenv("XDG_CONFIG_HOME", t.TempDir())
|
||||
t.Setenv("XDG_CACHE_HOME", t.TempDir())
|
||||
setHome(t, t.TempDir()) // no signed-in agent searches
|
||||
// no models.dev catalog either: the lists' models go by their ids
|
||||
catalog.Reset()
|
||||
t.Cleanup(catalog.Reset)
|
||||
lists := func(ids ...string) string {
|
||||
var b strings.Builder
|
||||
for i, id := range ids {
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/yetone/magpie/internal/catalog"
|
||||
)
|
||||
|
||||
// John (Discord #feedback): a GLM Coding Plan's models read glm-5-turbo
|
||||
// where ZCode's read GLM-5-Turbo. The plan's list gives ids alone, and
|
||||
// Zhipu's catalog (zhipuai) doesn't list glm-5-turbo, so it kept its id as
|
||||
// its name. A model Zhipu's catalog doesn't list is now named as the other
|
||||
// providers serving it name it; the id sent upstream stays, a name the
|
||||
// user gave stays theirs, and an Azure deployment keeps the name its owner
|
||||
// gave it.
|
||||
func TestModelNamedAsElsewhere(t *testing.T) {
|
||||
home := t.TempDir()
|
||||
t.Setenv("HOME", home)
|
||||
t.Setenv("USERPROFILE", home)
|
||||
t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config"))
|
||||
t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache"))
|
||||
if err := os.MkdirAll(filepath.Dir(catalog.CachePath()), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(catalog.CachePath(), []byte(`{
|
||||
"zhipuai": {"models": {"glm-5.3": {"id":"glm-5.3","name":"GLM-5.3"}}},
|
||||
"zai": {"models": {"glm-5-turbo": {"id":"glm-5-turbo","name":"GLM-5-Turbo"}, "gpt-x": {"id":"gpt-x","name":"GPT-X"}}},
|
||||
"zai-coding-plan": {"models": {"glm-5.3-flashx": {"id":"glm-5.3-flashx","name":"GLM-5.3-FlashX"}}}
|
||||
}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
catalog.Reset()
|
||||
t.Cleanup(catalog.Reset)
|
||||
// as https://open.bigmodel.cn/api/coding/paas/v4/models lists them
|
||||
listed := []catalog.Model{{ID: "glm-5.3", Name: "glm-5.3"}, {ID: "glm-5-turbo", Name: "glm-5-turbo"}, {ID: "glm-5.3-flashx", Name: "glm-5.3-flashx"}, {ID: "glm-9", Name: "glm-9"}}
|
||||
if err := catalog.SaveLive("zhipu", "", listed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := Save(Provider{ID: "zhipu", Name: "Zhipu GLM", Preset: "zhipu", Catalog: "zhipuai", Key: "test-key", Chat: "https://open.bigmodel.cn/api/coding/paas/v4",
|
||||
Models: []string{"glm-5.3", "glm-5-turbo", "glm-5.3-flashx", "glm-9", "gpt-x"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, err := Find("zhipu")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got []string
|
||||
for _, m := range p.Exposed() {
|
||||
got = append(got, m.ID+"|"+m.Name)
|
||||
}
|
||||
want := []string{
|
||||
"glm-5.3|GLM-5.3",
|
||||
"glm-5-turbo|GLM-5-Turbo",
|
||||
"glm-5.3-flashx|GLM-5.3-FlashX",
|
||||
"glm-9|glm-9", // no provider names it
|
||||
"gpt-x|GPT-X", // picked, not in the list
|
||||
}
|
||||
if strings.Join(got, "\n") != strings.Join(want, "\n") {
|
||||
t.Fatalf("got\n%s\nwant\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
|
||||
if err := SetModelName("zhipu/glm-5-turbo", "turbo mine"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
byID := map[string]Entry{}
|
||||
for _, e := range Catalog() {
|
||||
byID[e.ID] = e
|
||||
}
|
||||
if e := byID["zhipu/glm-5-turbo"]; e.Name != "turbo mine" || e.Default != "GLM-5-Turbo" || e.Model != "glm-5-turbo" {
|
||||
t.Errorf("named by the user: %+v", e)
|
||||
}
|
||||
|
||||
if err := catalog.SaveLive("my-azure", "", []catalog.Model{{ID: "gpt-x", Name: "gpt-x"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := Save(Provider{ID: "my-azure", Name: "Azure", Preset: AzurePreset, Key: "test-key", Chat: "https://me.openai.azure.com/openai/v1", Models: []string{"gpt-x"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, err := Find("my-azure")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ms := a.Exposed(); len(ms) != 1 || ms[0].Name != "gpt-x" {
|
||||
t.Errorf("an Azure deployment was renamed: %+v", ms)
|
||||
}
|
||||
}
|
||||
@@ -76,7 +76,14 @@ func (p Provider) available() []catalog.Model {
|
||||
// levels aren't taken off by a known model of its id
|
||||
return collapseAntigravityModels(catalog.Decorate(live, known))
|
||||
}
|
||||
return catalog.Decorate(live, known)
|
||||
if p.IsAzure() {
|
||||
// a deployment is named as the user named it
|
||||
return catalog.Decorate(live, known)
|
||||
}
|
||||
// a model the provider's catalog doesn't list reads as it does
|
||||
// under the other providers serving it (GLM-5-Turbo, not
|
||||
// glm-5-turbo, beside ZCode's)
|
||||
return catalog.Named(catalog.Decorate(live, known))
|
||||
}
|
||||
if p.IsAzure() {
|
||||
// an Azure resource serves its deployments alone, named as the
|
||||
@@ -716,7 +723,11 @@ func (p Provider) Exposed() []catalog.Model {
|
||||
// with the levels the gateway fits an effort to (Known),
|
||||
// not the none effortsOf takes a vendor's word for: the
|
||||
// vendor's list doesn't have it, so it gave no word (#597)
|
||||
out = append(out, catalog.Model{ID: id, Name: id, Provider: p.firstCatalog(), Efforts: p.knownElsewhere(id)})
|
||||
m := catalog.Model{ID: id, Name: id, Provider: p.firstCatalog(), Efforts: p.knownElsewhere(id)}
|
||||
if !p.IsAzure() {
|
||||
m = catalog.Named([]catalog.Model{m})[0]
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
|
||||
Reference in new issue
Block a user