diff --git a/internal/gui/assets/app.js b/internal/gui/assets/app.js index fa843978..2d360774 100644 --- a/internal/gui/assets/app.js +++ b/internal/gui/assets/app.js @@ -5041,7 +5041,9 @@ function modelInfo(m) { const box = el("span", "minfo"); const lines = []; const levels = m.efforts || []; - if (levels.length) { + // a decision model answers a routing group's questions, it doesn't reason + if (decideEntry(m)) lines.push(t("Decision model: routing groups ask it, agents never see it")); + else if (levels.length) { box.append(el("span", "badge mi-effort", effortSpan(levels))); lines.push(t("Reasoning: {levels}", { levels: levels.map((l) => t(l)).join(", ") })); } else lines.push(t("Reasoning levels: none known")); diff --git a/internal/gui/assets/i18n.js b/internal/gui/assets/i18n.js index 5ea4fd4b..19ffe7a6 100644 --- a/internal/gui/assets/i18n.js +++ b/internal/gui/assets/i18n.js @@ -644,6 +644,7 @@ const I18N = { "Words match a model's id, name, provider or a group's models": "按模型 ID、名称、供应商或模型组里的模型查找", "Reasoning: {levels}": "推理强度：{levels}", "Reasoning levels: none known": "推理强度：未知", + "Decision model: routing groups ask it, agents never see it": "决策模型：路由组会问它，agent 看不到它", "Accepts images (every model in it does)": "支持图片输入（组内每个模型都支持）", "Text only": "仅文本", "Context: {n} tokens": "上下文：{n} tokens", @@ -3624,6 +3625,7 @@ const I18N = { "Words match a model's id, name, provider or a group's models": "モデルの ID、名前、プロバイダ、またはグループのモデルに一致する語で検索します", "Reasoning: {levels}": "推論：{levels}", "Reasoning levels: none known": "推論レベル：不明", + "Decision model: routing groups ask it, agents never see it": "決定モデル：ルーティンググループが問い合わせ、エージェントには見えません", "Accepts images (every model in it does)": "画像入力に対応（含まれるすべてのモデルが対応）", "Text only": "テキストのみ", "Context: {n} tokens": "コンテキスト：{n} トークン", @@ -6594,6 +6596,7 @@ const I18N = { "Words match a model's id, name, provider or a group's models": "Suchbegriffe passen auf ID, Name oder Anbieter eines Modells oder auf die Modelle einer Gruppe", "Reasoning: {levels}": "Reasoning: {levels}", "Reasoning levels: none known": "Reasoning-Stufen: keine bekannt", + "Decision model: routing groups ask it, agents never see it": "Entscheidungsmodell: Routing-Gruppen fragen es, Agents sehen es nie", "Accepts images (every model in it does)": "Akzeptiert Bilder (jedes enthaltene Modell)", "Text only": "Nur Text", "Context: {n} tokens": "Kontext: {n} Token", diff --git a/internal/gui/tests/gateway-decide-info.test.cjs b/internal/gui/tests/gateway-decide-info.test.cjs new file mode 100644 index 00000000..b11070ae --- /dev/null +++ b/internal/gui/tests/gateway-decide-info.test.cjs @@ -0,0 +1,84 @@ +// Run with Node's test runner and Playwright on the module path; see README.md. +// A decision model on the Gateway list (ARNO on Discord: decision模型也是需要 +// 拉取正确的上下文长度以及是否支持视觉功能的): its row says its context and +// whether it takes images, as any model's does, and its tooltip says it is a +// decision model routing groups ask, rather than "Reasoning levels: none known". +const assert = require("node:assert/strict"); +const fs = require("node:fs/promises"); +const path = require("node:path"); +const { test } = require("node:test"); +const { chromium, webkit } = require("playwright"); + +const assets = path.resolve(__dirname, "../assets"); +const providers = (lang) => ({ + providers: [ + { id: "workers", name: "Workers AI", icon: "generic", agents: [], decide: "https://api.cloudflare.com/client/v4/accounts/a/ai/run", models: [ + { id: "@cf/typesafe/jev", name: "Jev", on: true, images: false, context: 32000 }, + { id: "@cf/cloudflare/clef", name: "Clef", on: true, images: true, context: 65536 }, + ] }, + { id: "acme", name: "Acme", icon: "generic", agents: [], chat: "https://acme.test/v1", models: [ + { id: "text-only", name: "Text Only", on: true, images: false, context: 128000 }, + ] }, + ], + gateway: { running: true, window: true, mine: true, url: "http://127.0.0.1:3999", models: 3, calls: [], groups: [] }, +}); + +function server(lang) { + const state = { agents: [], profiles: [], settings: { lang, theme: "light" } }; + return async (route) => { + const url = new URL(route.request().url()); + const json = (data) => route.fulfill({ json: data }); + if (url.pathname === "/boot.js") return route.fulfill({ contentType: "text/javascript", body: `window.bootPrefs = {lang:"${lang}",theme:"light",web:true};` }); + if (url.pathname === "/wails/runtime.js") return route.fulfill({ contentType: "text/javascript", body: "export const Window = {};" }); + if (url.pathname === "/api/state") return json(state); + if (url.pathname === "/api/providers") return json(providers(lang)); + if (url.pathname === "/api/groups") return json({ groups: [] }); + if (url.pathname === "/api/gateway/trace") { + if (url.searchParams.get("wait")) await new Promise((r) => setTimeout(r, 20e3)); + return json({ mine: true, now: new Date().toISOString(), seq: 1, totals: { requests: 0, rerouted: 0, errors: 0 }, routes: [] }); + } + if (url.pathname.startsWith("/api/")) return json({}); + const file = path.join(assets, url.pathname === "/" ? "index.html" : url.pathname); + const contentType = { ".html": "text/html", ".js": "text/javascript", ".css": "text/css", ".svg": "image/svg+xml", ".png": "image/png" }[path.extname(file)]; + await route.fulfill({ body: await fs.readFile(file), contentType }); + }; +} + +const rowOf = (page, id) => page.locator("#gwModels .row.model").filter({ has: page.locator(".name", { hasText: new RegExp("^" + id.replace(/[./@]/g, "\\$&") + "$") }) }); + +for (const engine of (process.env.BROWSER ? [process.env.BROWSER] : ["chromium", "webkit"])) { + for (const lang of ["en", "zh"]) { + test(`${engine} ${lang}: a decision model's row says its context and images`, async (t) => { + assert(["chromium", "webkit"].includes(engine), "BROWSER must be chromium or webkit"); + const browser = await (engine === "webkit" ? webkit.launch() : chromium.launch({ channel: "chromium" })); + t.after(() => browser.close()); + const page = await (await browser.newContext({ viewport: { width: 900, height: 700 }, reducedMotion: "reduce" })).newPage(); + page.setDefaultTimeout(5000); + const errors = []; + page.on("pageerror", (e) => errors.push(e.message)); + await page.route("**/*", server(lang)); + await page.goto("http://magpie.test/?view=gateway"); + await rowOf(page, "workers/@cf/cloudflare/clef").waitFor(); + + const info = async (id) => { + const box = rowOf(page, id).locator(".minfo"); + return { chips: (await box.locator(".badge").allTextContents()).map((s) => s.trim()), img: await box.locator(".mi-img").count(), tip: await box.getAttribute("title") }; + }; + const said = lang === "zh" ? "决策模型：路由组会问它，agent 看不到它" : "Decision model: routing groups ask it, agents never see it"; + let i = await info("workers/@cf/cloudflare/clef"); + assert.equal(i.img, 1, "Clef takes images"); + assert.deepEqual(i.chips, ["", "66K"]); + assert.equal(i.tip.split("\n")[0], said); + assert.match(i.tip, /65,536/); + i = await info("workers/@cf/typesafe/jev"); + assert.equal(i.img, 0, "Jev is text only"); + assert.deepEqual(i.chips, ["32K"]); + assert.equal(i.tip.split("\n")[0], said); + assert.match(i.tip, /32,000/); + // a chat model's tooltip is as it was + i = await info("acme/text-only"); + assert.notEqual(i.tip.split("\n")[0], said); + assert.deepEqual(errors, []); + }); + } +} diff --git a/internal/provider/decide.go b/internal/provider/decide.go index b0269dac..a1455a18 100644 --- a/internal/provider/decide.go +++ b/internal/provider/decide.go @@ -10,6 +10,7 @@ package provider // decision models are excluded from agents' lists and group members. import ( + "cmp" "context" "encoding/json" "errors" @@ -164,8 +165,58 @@ func (p Provider) Jev() string { } // decideModels are the models a decision provider offers: the vendor's -// list when fetched, else Jev's aliases (a gateway's one Jev). +// list when fetched, else Jev's aliases (a gateway's one Jev), each with +// its window and whether it takes images where its vendor's docs say. func (p Provider) decideModels() []catalog.Model { + return withDecideFacts(p.decideListed()) +} + +// A decision model's window and whether it reads images, as its vendor's +// docs give them where its list doesn't (ARNO on Discord: decision models +// showed neither on the Gateway list): Jev holds 32k tokens of state and +// reads text alone (TypeSafe's docs; OpenRouter's and Vercel's lists say +// 32000 too), Cloudflare's Clef and Clef Flash 65,536 and up to 4 images, +// Bailian's decision model 65,536 and text alone. +type decideFact struct { + context int + images bool +} + +func decideFactOf(id string) (decideFact, bool) { + last := strings.ToLower(id[strings.LastIndex(id, "/")+1:]) + last = strings.TrimPrefix(last, "~") + switch { + case jevID(id): + return decideFact{context: 32_000}, true + case last == "clef" || last == "clef-flash": + return decideFact{context: 65_536, images: true}, true + case last == BailianDecision: + return decideFact{context: 65_536}, true + } + return decideFact{}, false +} + +// withDecideFacts fills in the window and images of the decision models +// it knows that a list left unsaid; what a list said is kept. +func withDecideFacts(ms []catalog.Model) []catalog.Model { + ms = slices.Clone(ms) + for i, m := range ms { + f, ok := decideFactOf(m.ID) + if !ok { + continue + } + if m.Context == 0 { + ms[i].Context = f.context + } + if m.ImageInput == nil { + in := f.images + ms[i].ImageInput, ms[i].Images = &in, in + } + } + return ms +} + +func (p Provider) decideListed() []catalog.Model { if live, _, ok := catalog.Live(p.ID); ok && len(live) > 0 { if p.DecideOnly() { return live @@ -637,11 +688,9 @@ func listedDecide(b []byte, all bool) ([]catalog.Model, error) { var out struct { Models []struct { Name string `json:"name"` - ID string `json:"id"` + listedFacts } `json:"models"` - Data []struct { - ID string `json:"id"` - } `json:"data"` + Data []listedFacts `json:"data"` } if err := json.Unmarshal(b, &out); err != nil { return nil, fmt.Errorf("not a model list") @@ -649,29 +698,54 @@ func listedDecide(b []byte, all bool) ([]catalog.Model, error) { var ms []catalog.Model for _, m := range out.Models { if m.Name != "" { - ms = append(ms, catalog.Model{ID: m.Name, Name: m.Name}) + ms = append(ms, m.model(m.Name)) } } if len(ms) > 0 { return ms, nil } seen := map[string]bool{} - take := func(id string) { - if id == "" || seen[id] || !all && !jevID(id) { + take := func(m listedFacts) { + if m.ID == "" || seen[m.ID] || !all && !jevID(m.ID) { return } - seen[id] = true - ms = append(ms, catalog.Model{ID: id, Name: id}) + seen[m.ID] = true + ms = append(ms, m.model(m.ID)) } for _, m := range out.Data { - take(m.ID) + take(m) } for _, m := range out.Models { - take(m.ID) + take(m.listedFacts) } return ms, nil } +// listedFacts is a decision model as an OpenAI-shaped list gives it, with +// its window and input as OpenRouter's (context_length, +// architecture.input_modalities) or Vercel's (context_window, +// modalities.input) say them, where they do. +type listedFacts struct { + ID string `json:"id"` + ContextLength int `json:"context_length"` + ContextWindow int `json:"context_window"` + Architecture struct { + InputModalities []string `json:"input_modalities"` + } `json:"architecture"` + Modalities struct { + Input []string `json:"input"` + } `json:"modalities"` +} + +func (f listedFacts) model(name string) catalog.Model { + m := catalog.Model{ID: name, Name: name, Context: cmp.Or(f.ContextLength, f.ContextWindow)} + if in := cmp.Or(len(f.Architecture.InputModalities), len(f.Modalities.Input)); in > 0 { + images := slices.Contains(f.Architecture.InputModalities, "image") || slices.Contains(f.Modalities.Input, "image") + m.ImageInput, m.Images = &images, images + } + return m +} + // jevID reports whether id names Jev: a path segment that is "jev" or // "jev-…" (jev-latest, typesafe/jev, typesafe-ai/jev), not jevons or jevx. func jevID(id string) bool { diff --git a/internal/provider/decide_facts_test.go b/internal/provider/decide_facts_test.go new file mode 100644 index 00000000..facfd4f6 --- /dev/null +++ b/internal/provider/decide_facts_test.go @@ -0,0 +1,81 @@ +package provider + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/yetone/magpie/internal/catalog" +) + +// A decision model's window and whether it takes images (ARNO on Discord): +// read from a list that says them (OpenRouter's context_length and +// architecture.input_modalities), else what its vendor's docs say — Jev +// 32k and text, Clef 65,536 and images, Bailian's 65,536 and text — so +// the Providers page and the Gateway list show them. +func TestDecideModelsKnowWindowAndImages(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("USERPROFILE", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + t.Setenv("XDG_CACHE_HOME", t.TempDir()) + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"data":[{"id":"perplexity/pplx-decider-v1-27b","context_length":262144,"architecture":{"input_modalities":["text","image"]}},{"id":"liquid/d1","context_length":65536,"architecture":{"input_modalities":["text"]}},{"id":"typesafe/jev-1.13"}]}`)) + })) + defer up.Close() + p := Provider{ID: "openrouter-s1", Name: "OpenRouter", Key: "k", Decide: up.URL + "/api/v1", ModelsURL: up.URL + "/api/v1/models?output_modalities=decisions"} + if err := Save(p); err != nil { + t.Fatal(err) + } + if _, err := p.Fetch(context.Background()); err != nil { + t.Fatal(err) + } + want := map[string][2]any{ + "perplexity/pplx-decider-v1-27b": {262144, true}, + "liquid/d1": {65536, false}, + "typesafe/jev-1.13": {32000, false}, // the list said nothing: Jev's docs + } + check := func(who string, ms []catalog.Model, want map[string][2]any) { + t.Helper() + seen := 0 + for _, m := range ms { + w, ok := want[m.ID] + if !ok { + continue + } + seen++ + if ListedWindow(m) != w[0] || m.ImageInput == nil || *m.ImageInput != w[1] || m.Images != w[1] { + t.Errorf("%s %s: window %d, images %v (%v), want %v", who, m.ID, ListedWindow(m), m.Images, m.ImageInput, w) + } + } + if seen != len(want) { + t.Errorf("%s: %d of %d models listed: %v", who, seen, len(want), ms) + } + } + check("openrouter", p.Available(), want) + + // presets, before any list: Workers AI's Jev and Clefs, Bailian's + cf, err := FromPreset("cloudflare-jev") + if err != nil { + t.Fatal(err) + } + cf.Decide = "https://api.cloudflare.com/client/v4/accounts/acc7/ai/run" + check("cloudflare", cf.Available(), map[string][2]any{CloudflareJev: {32000, false}, CloudflareClefModel: {65536, true}, CloudflareClefFlash: {65536, true}}) + bl := Provider{ID: "bailian-s1", Decide: "https://ws-1.cn-beijing.maas.aliyuncs.com/api/v1"} + check("bailian", bl.Available(), map[string][2]any{BailianDecision: {65536, false}}) + ts := Provider{ID: "typesafe", Decide: "https://api.typesafe.ai/v1"} + check("typesafe", ts.Available(), map[string][2]any{JevLatest: {32000, false}, "jev-preview": {32000, false}}) + + // a gateway's Jev among its chat models has them too; its chat models + // are left as they are + if err := catalog.SaveLive("zen", "https://opencode.ai/zen/v1", []catalog.Model{{ID: "jev-1.13"}, {ID: "chat-x", Context: 1000}}); err != nil { + t.Fatal(err) + } + zen := Provider{ID: "zen", Chat: "https://opencode.ai/zen/v1", Decide: "https://opencode.ai/zen/v1"} + check("zen", zen.Available(), map[string][2]any{"jev-1.13": {32000, false}}) + for _, m := range zen.Available() { + if m.ID == "chat-x" && (m.Context != 1000 || m.ImageInput != nil) { + t.Errorf("a chat model changed: %+v", m) + } + } +} diff --git a/internal/provider/models.go b/internal/provider/models.go index b7c29418..47e3438a 100644 --- a/internal/provider/models.go +++ b/internal/provider/models.go @@ -29,6 +29,20 @@ func (p Provider) Available() []catalog.Model { if p.DecideOnly() { return p.decideModels() } + ms := p.available() + if p.Decides() { + // a gateway's Jev among its chat models, with its window and input + ms = slices.Clone(ms) + for i, m := range ms { + if p.DecidesModel(m.ID) { + ms[i] = withDecideFacts([]catalog.Model{m})[0] + } + } + } + return ms +} + +func (p Provider) available() []catalog.Model { signedIn := p.Account != nil && p.Account.models != nil var known []catalog.Model if signedIn {