diff --git a/internal/catalog/catalog.go b/internal/catalog/catalog.go index d590a97f..a4bfbcb7 100644 --- a/internal/catalog/catalog.go +++ b/internal/catalog/catalog.go @@ -13,6 +13,7 @@ import ( "path/filepath" "slices" "sort" + "strconv" "strings" "sync" "time" @@ -338,6 +339,9 @@ var ( // names are the models' names, by bare id, as most of the providers // serving them give it names map[string]string + // listed are the bare ids of every model models.dev lists, whatever it + // says of them: an id listed is never taken for another (renamed) + listed map[string]bool syncMu sync.Mutex ) @@ -383,8 +387,10 @@ func load() map[string]mdProvider { // (llmgateway's deepinfra/…, xiaomi/…) votes once, not once // a host voted := map[string]bool{} + listed = map[string]bool{} for pid, p := range m { for id, x := range p.Models { + listed[bareID(id)] = true OneHourFor(id, x.Cost) if x.Name != "" && x.Name != id && x.Name != bareID(id) { if named[bareID(id)] == nil { @@ -467,7 +473,7 @@ func Reset() { loadMu.Lock() defer loadMu.Unlock() loaded = false - mdev, images, windows, outputs, efforts, thinks, names = nil, nil, nil, nil, nil, nil, nil + mdev, images, windows, outputs, efforts, thinks, names, listed = nil, nil, nil, nil, nil, nil, nil, nil } // Sync downloads the models.dev catalog into CachePath. It serializes with @@ -575,6 +581,11 @@ func PricedBy(providers []string, id string) (Price, bool) { return *all[pid].Models[keys[0]].Cost, true } } + // and a relay's rename of a model none of them lists as given + // (claude-deepseek-v4.1-flash[1M] is deepseek-v4.1-flash, #1383) + if r, _, ok := renamed(b); ok { + return PricedBy(providers, r) + } return Price{}, false } @@ -641,7 +652,8 @@ func ProviderName(id string) string { // others. func Thinks(id string) bool { load() - return thinks[bareID(id)] + v, _ := lookup(id, thinks) + return v } // NameOf is the name models.dev gives a model of this id, as most of the @@ -650,7 +662,21 @@ func Thinks(id string) bool { // other than by its id. func NameOf(id string) string { load() - return names[bareID(id)] + b := bareID(id) + if n, ok := found(b, names); ok { + return n + } + r, w, ok := renamed(b) + if !ok { + return "" + } + n, _ := found(r, names) + if n != "" && w > 0 { + // two of a relay's models apart by their window alone read apart: + // "DeepSeek V4.1 Flash [1M]", the mark as the id writes it + n += " " + id[strings.LastIndexByte(id, '['):] + } + return n } // Named gives a model the list names by its id alone the name models.dev @@ -682,7 +708,8 @@ func Knows(id string) bool { // glm-5.3). func SeesImages(id string) bool { load() - return images[bareID(id)] + v, _ := lookup(id, images) + return v } // ContextOf is the context window models.dev gives a model of this id, as @@ -691,23 +718,22 @@ func SeesImages(id string) bool { // proxy serving "openai/gpt-5.4", or "gpt-5.4(high)" with the reasoning // level CLIProxyAPI takes in the id, which a static model list would // otherwise leave at the agent's default. +// +// A relay's rename of a model it knows (renamed) that names its window +// ("[1M]", "[200K]", Claude Code's "[1m]") has the window it names: what +// the relay serves it at, as Claude Code takes claude-opus-5[1m] for 1M. func ContextOf(id string) int { load() b := bareID(id) - if w, ok := windows[b]; ok { + if w, ok := found(b, windows); ok { return w } - if r, ok := unprofiled(b); ok { - return ContextOf(r) - } - if i := strings.IndexByte(b, '('); i > 0 && strings.HasSuffix(b, ")") { - b = b[:i] - if w, ok := windows[b]; ok { + if r, w, ok := renamed(b); ok { + if w > 0 { return w } - } - if i := strings.IndexByte(b, ':'); i > 0 { // ":free", ":batch" - return windows[b[:i]] + w, _ = found(r, windows) + return w } return 0 } @@ -716,17 +742,8 @@ func ContextOf(id string) int { // most of the providers serving it give it, or 0 when not known. func OutputOf(id string) int { load() - b := bareID(id) - if o, ok := outputs[b]; ok { - return o - } - if r, ok := unprofiled(b); ok { - return OutputOf(r) - } - if i := strings.IndexAny(b, "(:"); i > 0 { - return outputs[b[:i]] - } - return 0 + o, _ := lookup(id, outputs) + return o } // EffortsOf is the reasoning levels models.dev gives a model of this id, as @@ -737,23 +754,113 @@ func OutputOf(id string) int { // reason, and an agent offer no levels for it. func EffortsOf(id string) []string { load() + e, _ := lookup(id, efforts) + return slices.Clone(e) +} + +// lookup is what table holds for a model of this id: as found finds it, +// else as the relay's rename of a model found finds (renamed). Every +// capability is looked up the same way, so a model the catalog knows has +// all of them or none. +func lookup[T any](id string, table map[string]T) (T, bool) { b := bareID(id) - if e, ok := efforts[b]; ok { - return slices.Clone(e) + if v, ok := found(b, table); ok { + return v, true + } + if r, _, ok := renamed(b); ok { + return found(r, table) + } + var zero T + return zero, false +} + +// found is what table holds for a bare id: by the id itself, a Bedrock +// profile's without its geography, "gpt-5.4(high)" (CLIProxyAPI's level in +// the id) as gpt-5.4, or "glm-5:free" (":batch", ":thinking") as glm-5. +func found[T any](b string, table map[string]T) (T, bool) { + if v, ok := table[b]; ok { + return v, true } if r, ok := unprofiled(b); ok { - return EffortsOf(r) + return found(r, table) } if i := strings.IndexByte(b, '('); i > 0 && strings.HasSuffix(b, ")") { + if v, ok := table[b[:i]]; ok { + return v, true + } b = b[:i] - if e, ok := efforts[b]; ok { - return slices.Clone(e) + } + if i := strings.IndexByte(b, ':'); i > 0 { + v, ok := table[b[:i]] + return v, ok + } + var zero T + return zero, false +} + +// relayPrefixes are what relays put before a model's id to rename it, so +// Claude Code, which takes a model for a Claude one by its name, serves it: +// claude-deepseek-v4.1-flash (#1383). +var relayPrefixes = []string{"claude-"} + +// renamed is the model a relay renamed to b, when models.dev lists none +// by b (found), and the window the rename names, if any: a window mark +// taken off ("deepseek-v4.1-flash[1m]", "[200k]", and Claude Code's +// claude-opus-5[1m], which is claude-opus-5), then a relay's prefix +// ("claude-deepseek-v4.1-flash"). An id models.dev lists is always its own +// model, never renamed; a prefix comes off only when what is left is +// listed, and names a version ("claude-auto" is not the router "auto"). +func renamed(b string) (string, int, bool) { + if _, ok := found(b, listed); ok { + return "", 0, false + } + r, w := unmarked(b) + if r != b { + if _, ok := found(r, listed); ok { + return r, w, true } } - if i := strings.IndexByte(b, ':'); i > 0 { // ":free", ":thinking" - return slices.Clone(efforts[b[:i]]) + for _, pre := range relayPrefixes { + rest, ok := strings.CutPrefix(r, pre) + if !ok || !strings.ContainsAny(rest, "0123456789") { + continue + } + if _, ok := found(rest, listed); ok { + return rest, w, true + } } - return nil + return "", 0, false +} + +// unmarked is a bare id without a window mark at its end ("[1m]", +// "[200k]", "[1.5m]") and the window it names, or the id and 0 without one. +func unmarked(b string) (string, int) { + if !strings.HasSuffix(b, "]") { + return b, 0 + } + i := strings.LastIndexByte(b, '[') + if i <= 0 { + return b, 0 + } + mark := b[i+1 : len(b)-1] + unit := 0 + switch { + case strings.HasSuffix(mark, "k"): + unit = 1_000 + case strings.HasSuffix(mark, "m"): + unit = 1_000_000 + default: + return b, 0 + } + num := mark[:len(mark)-1] + if strings.Trim(num, "0123456789.") != "" { + return b, 0 + } + n, err := strconv.ParseFloat(num, 64) + if err != nil || n <= 0 { + return b, 0 + } + return b[:i], int(n * float64(unit)) } // ListedBy is the reasoning levels the first of providers (models.dev @@ -768,6 +875,9 @@ func EffortsOf(id string) []string { func ListedBy(providers []string, id string) ([]string, bool) { all := load() b := bareID(id) + if r, _, ok := renamed(b); ok { + b = r + } if r, ok := unprofiled(b); ok { b = r } diff --git a/internal/catalog/renamed_test.go b/internal/catalog/renamed_test.go new file mode 100644 index 00000000..0995e3d4 --- /dev/null +++ b/internal/catalog/renamed_test.go @@ -0,0 +1,189 @@ +package catalog + +import ( + "slices" + "strings" + "testing" +) + +// The reporter's own test (#1383, uWydnA): a relay renames +// deepseek-v4.1-flash claude-deepseek-v4.1-flash[1M], and the renamed +// model had no levels, no window and no images. +func TestRelayRenamedIDLosesCapabilities(t *testing.T) { + writeCatalog(t, `{ + "deepseek": {"models": {"deepseek-v4.1-flash": {"id":"deepseek-v4.1-flash", + "reasoning_options":[{"type":"effort","values":["low","high","max"]}], + "modalities":{"input":["text","image"],"output":["text"]}, + "limit":{"context":1000000,"output":65536}}}}}`) + + for _, id := range []string{"deepseek-v4.1-flash", "claude-deepseek-v4.1-flash[1M]"} { + t.Run(id, func(t *testing.T) { + if got := strings.Join(EffortsOf(id), ","); got != "low,high,max" { + t.Errorf("EffortsOf(%q) = %q, want low,high,max", id, got) + } + if got := ContextOf(id); got != 1000000 { + t.Errorf("ContextOf(%q) = %d, want 1000000", id, got) + } + if !SeesImages(id) { + t.Errorf("SeesImages(%q) = false, want true", id) + } + }) + } +} + +const renamedCatalog = `{ + "deepseek": {"name":"DeepSeek","models": { + "deepseek-v4.1-flash": {"id":"deepseek-v4.1-flash","name":"DeepSeek V4.1 Flash", + "reasoning_options":[{"type":"effort","values":["low","high","max"]}], + "modalities":{"input":["text","image"],"output":["text"]}, + "cost":{"input":0.3,"output":1.2}, + "limit":{"context":1000000,"output":65536}}}}, + "anthropic": {"name":"Anthropic","models": { + "claude-opus-5": {"id":"claude-opus-5","name":"Claude Opus 5","reasoning":true, + "reasoning_options":[{"type":"effort","values":["low","medium","high"]}], + "modalities":{"input":["text","image"],"output":["text"]}, + "cost":{"input":5,"output":25}, + "limit":{"context":200000,"output":64000}}}}, + "elsewhere": {"name":"Elsewhere","models": { + "opus-5": {"id":"opus-5","name":"Not Claude", + "reasoning_options":[{"type":"effort","values":["minimal"]}], + "limit":{"context":32000,"output":4000}}, + "glm-5": {"id":"glm-5","name":"GLM-5","reasoning":true, + "reasoning_options":[{"type":"effort","values":["low","high"]}], + "modalities":{"input":["text","image"],"output":["text"]}, + "limit":{"context":200000,"output":8000}}, + "claude-glm-5": {"id":"claude-glm-5","name":"Claude GLM 5", + "modalities":{"input":["text"],"output":["text"]}, + "limit":{"context":64000,"output":4000}}}}, + "model-oracle-ai": {"name":"Oracle","models": { + "model-oracle-ai/auto": {"id":"model-oracle-ai/auto","name":"Auto Router", + "reasoning_options":[{"type":"effort","values":["none","minimal","low","medium","high","xhigh","max"]}], + "modalities":{"input":["text","image"],"output":["text"]}, + "limit":{"context":2000000,"output":100000}}}}, + "openai": {"name":"OpenAI","models": { + "gpt-5.5": {"id":"gpt-5.5","name":"GPT-5.5", + "reasoning_options":[{"type":"effort","values":["low","medium","high","xhigh"]}], + "modalities":{"input":["text","image"],"output":["text"]}, + "cost":{"input":1.25,"output":10}, + "limit":{"context":400000,"output":128000}}}} +}` + +// A relay's rename is matched by every getter, each of its forms, and a +// window mark names the window the relay serves it at. +func TestRelayRenameReachesEveryGetter(t *testing.T) { + writeCatalog(t, renamedCatalog) + + for id, window := range map[string]int{ + "claude-deepseek-v4.1-flash[1M]": 1_000_000, + "claude-deepseek-v4.1-flash[200K]": 200_000, + "claude-deepseek-v4.1-flash": 1_000_000, + "deepseek-v4.1-flash[1M]": 1_000_000, + "relay/claude-DeepSeek-V4.1-Flash": 1_000_000, + } { + t.Run(id, func(t *testing.T) { + if got := strings.Join(EffortsOf(id), ","); got != "low,high,max" { + t.Errorf("EffortsOf = %q", got) + } + if got := ContextOf(id); got != window { + t.Errorf("ContextOf = %d, want %d", got, window) + } + if got := OutputOf(id); got != 65536 { + t.Errorf("OutputOf = %d", got) + } + if !SeesImages(id) || !Thinks(id) || !Knows(id) { + t.Errorf("SeesImages %v, Thinks %v, Knows %v", SeesImages(id), Thinks(id), Knows(id)) + } + if p, ok := PricedBy([]string{"deepseek"}, id); !ok || p.Input != 0.3 { + t.Errorf("PricedBy = %+v, %v", p, ok) + } + if e, ok := ListedBy([]string{"deepseek"}, id); !ok || strings.Join(e, ",") != "low,high,max" { + t.Errorf("ListedBy = %v, %v", e, ok) + } + }) + } + // the name keeps the mark, so two of a relay's models apart by their + // window alone read apart + if got := NameOf("claude-deepseek-v4.1-flash[1M]"); got != "DeepSeek V4.1 Flash [1M]" { + t.Errorf("NameOf marked = %q", got) + } + if got := NameOf("claude-deepseek-v4.1-flash"); got != "DeepSeek V4.1 Flash" { + t.Errorf("NameOf = %q", got) + } +} + +// Claude Code's own [1m] on a Claude model is that model at 1M: the mark +// comes off, the "claude-" stays (claude-opus-5[1m] is not "opus-5"). +func TestClaude1MMarkIsTheClaudeModel(t *testing.T) { + writeCatalog(t, renamedCatalog) + + if got := strings.Join(EffortsOf("claude-opus-5[1m]"), ","); got != "low,medium,high" { + t.Errorf("EffortsOf = %q", got) + } + if got := ContextOf("claude-opus-5[1m]"); got != 1_000_000 { + t.Errorf("ContextOf [1m] = %d", got) + } + if got := ContextOf("claude-opus-5"); got != 200_000 { + t.Errorf("ContextOf = %d", got) + } + if got := OutputOf("claude-opus-5[1m]"); got != 64000 { + t.Errorf("OutputOf = %d", got) + } + if p, ok := PricedBy([]string{"anthropic"}, "claude-opus-5[1m]"); !ok || p.Input != 5 { + t.Errorf("PricedBy = %+v, %v", p, ok) + } + if got := NameOf("claude-opus-5[1m]"); got != "Claude Opus 5 [1m]" { + t.Errorf("NameOf = %q", got) + } +} + +// A listed id is always its own model, and a near miss is no model. +func TestRenameNeverTakesAModelForAnother(t *testing.T) { + writeCatalog(t, renamedCatalog) + + // claude-glm-5 is listed: no levels, no images, its own window, though + // glm-5 has them + if got := EffortsOf("claude-glm-5"); got != nil { + t.Errorf("EffortsOf(claude-glm-5) = %v", got) + } + if SeesImages("claude-glm-5") || Thinks("claude-glm-5") { + t.Error("claude-glm-5 took glm-5's images or reasoning") + } + if got := ContextOf("claude-glm-5"); got != 64000 { + t.Errorf("ContextOf(claude-glm-5) = %d", got) + } + if got := NameOf("claude-glm-5"); got != "Claude GLM 5" { + t.Errorf("NameOf(claude-glm-5) = %q", got) + } + // the relay's own router is not models.dev's "auto" (the reporter's + // counter-example), and ids that only look like a known one match none + for _, id := range []string{ + "claude-auto[1M]", "claude-auto", + "claude-deepseek-v4.1-flashy[1M]", "claude-deepseek-v4.1", + "deepseek-v4.1-flash[fast]", "deepseek-v4.1-flash[1M]x", "deepseek-v4.1-flash[infm]", + "anthropic-deepseek-v4.1-flash", "claude-claude-opus-5x", + "[1m]", "claude-[1m]", + } { + if e, c, i, n := EffortsOf(id), ContextOf(id), SeesImages(id), NameOf(id); e != nil || c != 0 || i || n != "" || Knows(id) { + t.Errorf("%q matched: efforts %v, context %d, images %v, name %q", id, e, c, i, n) + } + if p, ok := PricedBy([]string{"deepseek", "anthropic", "model-oracle-ai"}, id); ok { + t.Errorf("%q priced %+v", id, p) + } + } + // other vendors' known models are as they were + if got := strings.Join(EffortsOf("gpt-5.5"), ","); got != "low,medium,high,xhigh" { + t.Errorf("EffortsOf(gpt-5.5) = %q", got) + } + if ContextOf("openai/gpt-5.5") != 400000 || OutputOf("gpt-5.5") != 128000 || !SeesImages("gpt-5.5") { + t.Error("gpt-5.5 changed") + } + if got := EffortsOf("auto"); !slices.Contains(got, "xhigh") { + t.Errorf("EffortsOf(auto) = %v", got) + } + if got := strings.Join(EffortsOf("opus-5"), ","); got != "minimal" { + t.Errorf("EffortsOf(opus-5) = %q", got) + } + if got := ContextOf("glm-5"); got != 200000 { + t.Errorf("ContextOf(glm-5) = %d", got) + } +}