usage, gui: the Requests tab ranks one model at each provider it went to (inaction on Discord) — the trend's ranking already gave each model its speed, first token and requests, but "Model" put a model's requests at every provider together, so glm-5.3 at Zhipu and at ZCode couldn't be told apart. The split has "Model · provider" now: a "modelAt" dimension (provider/model) whose rows are named by both with the provider's icon, each with its own tokens/s, first token and requests, ranked by any metric (Speed: the fastest first). A click lists that provider's requests of that model, again all of them; the others stay ranked. The trend's bar wraps at the narrowest window, "Split by" with its choices. The key of each provider and model is built once per page (TestRequestPageAppendDoesNotRepriceHistory's allocations). TestRequestPageRanksModelAtEachProvider, TestLedgerNamesModelAtProvider and usage-model-at-provider (WebKit, en and zh) fail without the change. go vet, go build (darwin, GOOS=linux nogui, GOOS=windows) and go test -tags nogui ./... under a temp HOME pass (TestProvidersAnswerWhileListsComeIn and TestUserPath failed under load and pass alone).
This commit is contained in:
1 parent
4c001cef85
commit
f506d08d28
11 files changed
+274
-17
No files matched your search
@@ -773,9 +773,11 @@ body.window .field.solo { grid-column: auto; }
|
||||
.led-strip .meter { height: 5px; margin-top: 8px; border-radius: 3px; background: var(--pill); overflow: hidden; }
|
||||
.led-strip .meter i { display: block; height: 100%; border-radius: 3px; background: var(--green); }
|
||||
.led-trend { padding: 10px 14px 10px; background: var(--card); border: 1px solid var(--line); border-radius: 12px; }
|
||||
.led-bar { display: flex; align-items: center; gap: 8px; margin-bottom: 8px; }
|
||||
.led-bar { display: flex; align-items: center; flex-wrap: wrap; gap: 6px 8px; margin-bottom: 8px; }
|
||||
.led-bar .grow { flex: 1; }
|
||||
.led-by { font-size: 11.5px; color: var(--muted); }
|
||||
/* "Split by" and its choices wrap together, under the metrics when narrow */
|
||||
.led-split { display: flex; align-items: center; gap: 8px; }
|
||||
.led-body { position: relative; display: grid; grid-template-columns: minmax(0, 1fr) 272px; gap: 18px; }
|
||||
.led-chart { min-width: 0; }
|
||||
.led-chart .plot { position: relative; height: 236px; }
|
||||
|
||||
@@ -13364,7 +13364,9 @@ function ledMoney(v) {
|
||||
const LED_METRICS = [["tokens", "Tokens"], ["cost", "Cost"], ["calls", "Requests"]];
|
||||
// the window's chart can also say how fast the replies came (#860)
|
||||
const LED_TREND = [...LED_METRICS, ["speed", "Speed"]];
|
||||
const LED_SPLITS = [["model", "Model"], ["provider", "Provider"], ["agent", "Agent"]];
|
||||
// "modelAt" is a model at the provider it went to, so one model's speed at
|
||||
// each provider is ranked side by side (inaction on Discord)
|
||||
const LED_SPLITS = [["model", "Model"], ["modelAt", "Model · provider"], ["provider", "Provider"], ["agent", "Agent"]];
|
||||
const LED_SHOWN = 7; // told apart in a chart; the rest are "Other"
|
||||
const allTokens = (x) => x.input + x.output + x.cache_read + x.cache_write;
|
||||
// how fast the timed replies of a point, a share or a part wrote, in tokens
|
||||
@@ -13723,10 +13725,17 @@ function drawLedTrend() {
|
||||
});
|
||||
rank.chart = chart;
|
||||
if (!rank.rail) ledRail($("#ledRail"), rank);
|
||||
const picked = ledSplit === "provider" ? ledProvider : ledSplit === "agent" ? ledAgent : ledModel;
|
||||
const picked = ledSplit === "provider" ? ledProvider : ledSplit === "agent" ? ledAgent
|
||||
: ledSplit === "modelAt" ? (ledProvider && ledModel ? ledProvider + "/" + ledModel : "") : ledModel;
|
||||
drawLedRank(rank, l, ledSplit, ledMetric, picked, (x) => {
|
||||
// a click lists only that one's requests; on the one listed, all again
|
||||
if (ledSplit === "provider") ledProvider = ledProvider === x.id ? "" : x.id;
|
||||
if (ledSplit === "modelAt") {
|
||||
// provider/model: both filters, the provider's id has no "/"
|
||||
const q = $("#ledQ"), cut = x.id.indexOf("/"), off = picked === x.id;
|
||||
ledProvider = off ? "" : x.id.slice(0, cut);
|
||||
ledModel = off ? "" : x.id.slice(cut + 1);
|
||||
ledQuery = ledModel; q.value = ledQuery;
|
||||
} else if (ledSplit === "provider") ledProvider = ledProvider === x.id ? "" : x.id;
|
||||
else if (ledSplit === "agent") ledAgent = ledAgent === x.id ? "" : x.id;
|
||||
else { const q = $("#ledQ"); ledModel = ledModel === x.id ? "" : x.id; ledQuery = ledModel; q.value = ledQuery; }
|
||||
ledOffset = 0;
|
||||
|
||||
@@ -1554,6 +1554,7 @@ const I18N = {
|
||||
"Key": "密钥",
|
||||
"{env}=magpie. The gateway trusts everything on loopback, so any value works.": "{env}=magpie。网关信任本机回环请求,可填任意值。",
|
||||
"Model": "模型",
|
||||
"Model · provider": "模型 · 供应商",
|
||||
"Model id": "模型 ID",
|
||||
"Model ids": "模型 ID",
|
||||
"provider/model, as listed below. Click a model there to put it in the snippets.": "格式为 provider/model,见下方列表。点击某个模型即可放入示例。",
|
||||
@@ -4765,6 +4766,7 @@ const I18N = {
|
||||
"Key": "キー",
|
||||
"{env}=magpie. The gateway trusts everything on loopback, so any value works.": "{env}=magpie。ゲートウェイはループバック上のすべてを信頼するため、どんな値でも動作します。",
|
||||
"Model": "モデル",
|
||||
"Model · provider": "モデル · プロバイダ",
|
||||
"Model id": "モデル ID",
|
||||
"Model ids": "モデル ID",
|
||||
"provider/model, as listed below. Click a model there to put it in the snippets.": "下の一覧にある provider/model の形式です。モデルをクリックするとスニペットに入ります。",
|
||||
@@ -7952,6 +7954,7 @@ const I18N = {
|
||||
"Key": "Schlüssel",
|
||||
"{env}=magpie. The gateway trusts everything on loopback, so any value works.": "{env}=magpie. Der Zugangspunkt vertraut allem über Loopback, daher funktioniert jeder Wert.",
|
||||
"Model": "Modell",
|
||||
"Model · provider": "Modell · Anbieter",
|
||||
"Model id": "Modell-ID",
|
||||
"Model ids": "Modell-IDs",
|
||||
"provider/model, as listed below. Click a model there to put it in the snippets.": "provider/model, wie unten aufgeführt. Klicken Sie dort auf ein Modell, um es in die Snippets zu übernehmen.",
|
||||
|
||||
@@ -167,8 +167,7 @@
|
||||
<div class="led-bar">
|
||||
<div class="segs" id="ledMetric"></div>
|
||||
<span class="grow"></span>
|
||||
<span class="led-by" data-t>Split by</span>
|
||||
<div class="segs" id="ledSplit"></div>
|
||||
<span class="led-split"><span class="led-by" data-t>Split by</span><span class="segs" id="ledSplit"></span></span>
|
||||
</div>
|
||||
<div class="led-body">
|
||||
<div class="led-chart" id="ledChart"></div>
|
||||
|
||||
@@ -779,6 +779,13 @@ request listed there is nothing; in English and Chinese, light and dark.
|
||||
The daily chart also checks selection, gray bars on other days, filtered rows
|
||||
and CSV, an empty day, clearing a selection, keyboard input and resizing.
|
||||
|
||||
`usage-model-at-provider.test.cjs` splits the Requests tab's chart by "Model ·
|
||||
provider" (inaction on Discord): one model at two providers is two rows of the
|
||||
ranking, each named by both with the provider's icon and its own first token,
|
||||
the faster first by Speed though fewer requests went to it. A click lists that
|
||||
provider's requests of that model (provider and model both asked for), a click
|
||||
again all of them, and the page doesn't move. English and Chinese.
|
||||
|
||||
`usage-chart-axis.test.cjs` checks the usage chart's side labels, in the
|
||||
window's Requests tab (1180 and 560 wide) and the tray panel's Usage tab (440
|
||||
and 340 wide): for tokens, cost in dollars and in yuan, and requests, small and
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
// Run with Node's test runner and Playwright on the module path; see README.md.
|
||||
// The Usage page's Requests tab ranks one model at each provider it went to
|
||||
// (inaction on Discord: "相同模型在不同提供商的速度情况"). The chart's split
|
||||
// has "Model · provider": glm-5.3 at Zhipu and at ZCode are two rows of the
|
||||
// ranking, each with its own speed and first token, the fastest first by
|
||||
// Speed; a click lists that provider's requests of that model, a click again
|
||||
// all of them, and the page doesn't move. English and Chinese; no backend,
|
||||
// the API is faked.
|
||||
const assert = require("node:assert/strict");
|
||||
const fs = require("node:fs/promises");
|
||||
const path = require("node:path");
|
||||
const { test } = require("node:test");
|
||||
const { chromium, webkit } = require("playwright");
|
||||
const reader = require("./reader.cjs");
|
||||
|
||||
const assets = path.resolve(__dirname, "../assets");
|
||||
const now = Date.now();
|
||||
const H = 3600e3;
|
||||
const hour = (i) => new Date(Math.floor(now / H) * H - i * H).toISOString();
|
||||
|
||||
const row = (i, provider, providerName) => ({
|
||||
route_id: 900 + i, t: new Date(now - (i + 1) * 60e3).toISOString(), agent: "claude", agentName: "Claude Code", icon: "claudecode-color",
|
||||
provider, providerName, req: "glm-5.3", model: "glm-5.3", served: "glm-5.3", in: 1000, out: 500, cache_read: 0, cache_write: 0,
|
||||
ms: 2000, ttft_ms: 300, status: 200, rid: "req_" + i, ep: "/v1/messages", cost: 0.01, priced: true,
|
||||
});
|
||||
// Zhipu: 2 replies, 300 ms to the first token, 100 tok/s; ZCode: 4 replies,
|
||||
// 1200 ms, 25 tok/s. More requests went to ZCode, so by requests it leads.
|
||||
const ZHIPU = { id: "zhipu/glm-5.3", name: "glm-5.3 · Zhipu", icon: "zhipu-color", calls: 2, input: 1, output: 1000, cache_read: 0, cache_write: 0, cost: 0.1, timed: 2, ttft_ms: 600, decode_ms: 10000, decode_out: 1000 };
|
||||
const ZCODE = { id: "zcode/glm-5.3", name: "glm-5.3 · ZCode", icon: "zcode", calls: 4, input: 1, output: 2000, cache_read: 0, cache_write: 0, cost: 0.2, timed: 4, ttft_ms: 4800, decode_ms: 80000, decode_out: 2000 };
|
||||
const MODEL = { id: "glm-5.3", calls: 6, input: 2, output: 3000, cache_read: 0, cache_write: 0, cost: 0.3, timed: 6, ttft_ms: 5400, decode_ms: 90000, decode_out: 3000 };
|
||||
const part = (x) => ({ calls: x.calls, tokens: x.output, cost: x.cost, timed: x.timed, ttft_ms: x.ttft_ms, decode_ms: x.decode_ms, decode_out: x.decode_out });
|
||||
const timing = { timed: 6, ttft_ms: 5400, decode_ms: 90000, decode_out: 3000 };
|
||||
|
||||
function ledger(q) {
|
||||
const provider = q.get("provider"), model = q.get("model");
|
||||
const rows = [row(0, "zhipu", "Zhipu"), row(1, "zcode", "ZCode")].filter((r) => (!provider || r.provider === provider) && (!model || r.model === model));
|
||||
return {
|
||||
period: q.get("period"), rows, offset: 0, total: rows.length, calls: 6, errors: 0,
|
||||
input: 2, output: 3000, cache_read: 0, cache_write: 0, cost: 0.3, unpriced: 0, ...timing,
|
||||
by: {
|
||||
model: [MODEL], modelAt: [ZCODE, ZHIPU],
|
||||
provider: [{ ...ZCODE, id: "zcode", name: "ZCode" }, { ...ZHIPU, id: "zhipu", name: "Zhipu" }],
|
||||
},
|
||||
bucket: "hour",
|
||||
series: [{
|
||||
label: "", time: hour(0), calls: 6, input: 2, output: 3000, ...timing,
|
||||
by: { model: { "glm-5.3": part(MODEL) }, modelAt: { "zhipu/glm-5.3": part(ZHIPU), "zcode/glm-5.3": part(ZCODE) } },
|
||||
}],
|
||||
agents: [{ id: "claude", name: "Claude Code", icon: "claudecode-color" }],
|
||||
providers: [{ id: "zhipu", name: "Zhipu", icon: "zhipu-color" }, { id: "zcode", name: "ZCode", icon: "zcode" }],
|
||||
};
|
||||
}
|
||||
|
||||
function server(lang, asked) {
|
||||
return async (route) => {
|
||||
const url = new URL(route.request().url());
|
||||
const json = (data) => route.fulfill({ json: data });
|
||||
if (url.pathname === "/boot.js") return route.fulfill({ contentType: "text/javascript", body: `window.bootPrefs = {lang:"${lang}",theme:"light",web:false};` });
|
||||
if (url.pathname === "/wails/runtime.js") return route.fulfill({ contentType: "text/javascript", body: "export const Window = {};" });
|
||||
if (url.pathname === "/api/state") return json({ agents: [], profiles: [], settings: { lang, theme: "light" } });
|
||||
if (url.pathname === "/api/usage/requests") { asked.push(url.searchParams); return json(ledger(url.searchParams)); }
|
||||
if (url.pathname === "/api/usage/quotas") return json([]);
|
||||
if (url.pathname === "/api/usage") return json({ calls: 0, errors: 0, input: 0, output: 0, reasoning: 0, unpriced: 0, cost: 0, bucket: "day", series: [], agents: [], models: [], path: "" });
|
||||
if (url.pathname === "/api/providers") return json({ providers: [], presets: [], excluded: [], models: [], gateway: { running: false } });
|
||||
if (url.pathname === "/api/sessions") return json({ sessions: [], dirs: [] });
|
||||
if (url.pathname === "/api/sessions/stats") return json({ from: "", to: "", days: [], agents: {} });
|
||||
if (url.pathname === "/api/groups") return json({ groups: [], models: [] });
|
||||
if (url.pathname.startsWith("/api/")) return json({});
|
||||
const file = path.join(assets, url.pathname === "/" ? "index.html" : url.pathname);
|
||||
const contentType = { ".html": "text/html", ".js": "text/javascript", ".css": "text/css", ".svg": "image/svg+xml", ".png": "image/png" }[path.extname(file)];
|
||||
await route.fulfill({ body: await fs.readFile(file), contentType });
|
||||
};
|
||||
}
|
||||
|
||||
const L = {
|
||||
en: { split: "Model · provider", speed: "Speed", fast: "TTFT 300 ms", slow: "TTFT 1.2 s" },
|
||||
zh: { split: "模型 · 供应商", speed: "速度", fast: "首字 300 毫秒", slow: "首字 1.2 秒" },
|
||||
};
|
||||
|
||||
for (const engine of (process.env.BROWSER ? [process.env.BROWSER] : ["chromium", "webkit"])) {
|
||||
test(engine + ": one model ranked at each provider", async (t) => {
|
||||
const browser = await (engine === "webkit" ? webkit.launch() : chromium.launch({ channel: "chromium" }));
|
||||
t.after(() => browser.close());
|
||||
for (const lang of ["en", "zh"]) {
|
||||
const w = L[lang];
|
||||
await t.test(lang, async () => {
|
||||
const errors = [], asked = [];
|
||||
const ctx = await browser.newContext({ viewport: { width: 1100, height: 800 }, reducedMotion: "reduce" });
|
||||
const p = await ctx.newPage();
|
||||
p.setDefaultTimeout(5000);
|
||||
p.on("pageerror", (e) => errors.push(e.message));
|
||||
await p.route("**/*", server(lang, asked));
|
||||
await p.goto("http://magpie.test/");
|
||||
await p.locator('[data-view="usage"]').first().click();
|
||||
await p.locator("#usageTab .opt").nth(1).click();
|
||||
await p.locator("#ledWrap .led tbody tr").first().waitFor();
|
||||
await p.waitForTimeout(300);
|
||||
|
||||
await reader.click(p, p.locator("#ledSplit .opt").filter({ hasText: w.split }));
|
||||
await reader.click(p, p.locator("#ledMetric .opt").filter({ hasText: w.speed }));
|
||||
await p.waitForTimeout(150);
|
||||
const rows = p.locator("#ledRank button.rk");
|
||||
assert.equal(await rows.count(), 2);
|
||||
// by speed the fastest first, though fewer requests went to it
|
||||
const names = await p.locator("#ledRank button.rk .rk-nm").allTextContents();
|
||||
assert.deepEqual(names, ["glm-5.3 · Zhipu", "glm-5.3 · ZCode"]);
|
||||
const said = await p.locator("#ledRank button.rk .rk-b").allTextContents();
|
||||
assert(said[0].includes(w.fast) && said[1].includes(w.slow), "each its own first token: " + said.join(" | "));
|
||||
// the provider's icon beside each
|
||||
assert.deepEqual(await p.locator("#ledRank button.rk .rk-a .ic").evaluateAll((x) => x.map((e) => e.dataset.icon)), ["zhipu-color", "zcode"]);
|
||||
|
||||
// a click lists ZCode's glm-5.3 only; the page stays where it was
|
||||
const y = await p.evaluate(() => [document.scrollingElement.scrollTop, document.querySelector("#ledRank").scrollTop]);
|
||||
const before = asked.length;
|
||||
await reader.click(p, rows.nth(1));
|
||||
await p.waitForFunction((n) => document.querySelectorAll("#ledRank button.rk.on").length === 1, before);
|
||||
const q = asked.at(-1);
|
||||
assert(asked.length > before);
|
||||
assert.equal(q.get("provider"), "zcode");
|
||||
assert.equal(q.get("model"), "glm-5.3");
|
||||
assert.equal(await p.locator("#ledRank button.rk.on .rk-nm").textContent(), "glm-5.3 · ZCode");
|
||||
assert.deepEqual(await p.evaluate(() => [document.scrollingElement.scrollTop, document.querySelector("#ledRank").scrollTop]), y);
|
||||
|
||||
// again: all of them
|
||||
await reader.click(p, p.locator("#ledRank button.rk.on"));
|
||||
await p.waitForFunction(() => !document.querySelector("#ledRank button.rk.on"));
|
||||
assert.equal(asked.at(-1).get("provider"), null);
|
||||
assert.equal(asked.at(-1).get("model"), null);
|
||||
assert.deepEqual(errors, []);
|
||||
await ctx.close();
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/yetone/magpie/internal/access"
|
||||
@@ -149,7 +150,8 @@ type ledgerJSON struct {
|
||||
// Series: the period by hour, day or week (Bucket), before the day filter
|
||||
Bucket string `json:"bucket"`
|
||||
Series []usage.SeriesPoint `json:"series"`
|
||||
// By: the rows told apart by provider, agent and model, the most tokens
|
||||
// By: the rows told apart by provider, agent, model and model at each
|
||||
// provider ("modelAt", named "model · provider"), the most tokens
|
||||
// first. The one by a dimension the filter has picked is of the rows
|
||||
// without that pick, so the others are still there to switch to.
|
||||
By map[string][]ledgerShare `json:"by"`
|
||||
@@ -293,6 +295,11 @@ func ledgerPage(p usage.Period, f usage.Filter, offset, limit int) ledgerJSON {
|
||||
case "agent":
|
||||
a := who(s.ID)
|
||||
ls.Name, ls.Icon = a.Name, a.Icon
|
||||
case "modelAt":
|
||||
// the model, and the provider it went to
|
||||
prov, model, _ := strings.Cut(s.ID, "/")
|
||||
a := which(prov)
|
||||
ls.Name, ls.Icon = model+" · "+a.Name, a.Icon
|
||||
}
|
||||
shares = append(shares, ls)
|
||||
}
|
||||
|
||||
@@ -288,6 +288,22 @@ func TestLedgerSingleSnapshotAndLocalSession(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A model at a provider is named by both, with the provider's icon, so
|
||||
// one model's speed at each provider reads apart (inaction on Discord).
|
||||
func TestLedgerNamesModelAtProvider(t *testing.T) {
|
||||
sandboxHome(t)
|
||||
before := usage.LogCalls
|
||||
usage.LogCalls = func(time.Time) []sessions.Call {
|
||||
return []sessions.Call{{Time: time.Now(), Agent: "claude", Session: "s", Model: "claude-sonnet-5", Tokens: sessions.Tokens{Input: 10, Output: 2}}}
|
||||
}
|
||||
t.Cleanup(func() { usage.LogCalls = before })
|
||||
l := ledgerPage(usage.Today, usage.Filter{}, 0, 1)
|
||||
at := l.By["modelAt"]
|
||||
if len(at) != 1 || at[0].ID != usage.ModelAtKey(usage.UnknownProvider, "claude-sonnet-5") || at[0].Name != "claude-sonnet-5 · Local session" {
|
||||
t.Fatalf("model at provider %+v", at)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVStamp(t *testing.T) {
|
||||
for _, day := range []string{"", "2026-02-30", "../../other", "2026-09-30\""} {
|
||||
want := "magpie-requests-7d-" + time.Now().Format(time.DateOnly)
|
||||
|
||||
@@ -516,8 +516,14 @@ func (t *Totals) addRow(r Row) {
|
||||
}
|
||||
|
||||
// Dimensions are what calls are told apart by: the provider they went to,
|
||||
// the agent that made them, the model.
|
||||
var Dimensions = []string{"provider", "agent", "model"}
|
||||
// the agent that made them, the model, and the model at the provider it
|
||||
// went to ("modelAt"), so one model's speed at each provider can be compared
|
||||
// (inaction on Discord).
|
||||
var Dimensions = []string{"provider", "agent", "model", "modelAt"}
|
||||
|
||||
// ModelAtKey is a call's "modelAt": its provider's id and its model, as
|
||||
// "provider/model" (a provider's id has no "/").
|
||||
func ModelAtKey(provider, model string) string { return provider + "/" + model }
|
||||
|
||||
// key is the row's part of a dimension: its provider's id, its agent, its model.
|
||||
func (r Row) key(by string) string {
|
||||
@@ -526,6 +532,8 @@ func (r Row) key(by string) string {
|
||||
return r.Provider
|
||||
case "agent":
|
||||
return r.Agent
|
||||
case "modelAt":
|
||||
return ModelAtKey(r.Provider, r.Model)
|
||||
case "computer":
|
||||
if r.Computer == "" {
|
||||
return ThisComputer
|
||||
@@ -535,6 +543,23 @@ func (r Row) key(by string) string {
|
||||
return r.Model
|
||||
}
|
||||
|
||||
// keyer is Row.key for many rows, made once for each provider and model:
|
||||
// a "modelAt" is a string built of two, which a page of thousands of rows
|
||||
// would otherwise build again for each.
|
||||
type keyer map[[2]string]string
|
||||
|
||||
func (m keyer) key(r Row, by string) string {
|
||||
if by != "modelAt" {
|
||||
return r.key(by)
|
||||
}
|
||||
k, ok := m[[2]string{r.Provider, r.Model}]
|
||||
if !ok {
|
||||
k = r.key(by)
|
||||
m[[2]string{r.Provider, r.Model}] = k
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// AllTokens is what went in and out and through the cache.
|
||||
func (t Totals) AllTokens() int { return t.Input + t.Output + t.CacheRead + t.CacheWrite }
|
||||
|
||||
@@ -549,11 +574,12 @@ type Share struct {
|
||||
func Breakdown(rows []Row, by string) []Share {
|
||||
at := map[string]*Share{}
|
||||
var out []*Share
|
||||
keys := keyer{}
|
||||
for _, r := range rows {
|
||||
if r.IsRejected() {
|
||||
continue
|
||||
}
|
||||
k := r.key(by)
|
||||
k := keys.key(r, by)
|
||||
s := at[k]
|
||||
if s == nil {
|
||||
s = &Share{ID: k}
|
||||
@@ -642,6 +668,7 @@ func LedgerSeries(p Period, rows []Row) (bucket string, pts []SeriesPoint) {
|
||||
}
|
||||
}
|
||||
kept := map[string]map[string]bool{}
|
||||
keys := keyer{}
|
||||
for _, d := range Dimensions {
|
||||
kept[d] = map[string]bool{}
|
||||
for i, s := range Breakdown(rows, d) {
|
||||
@@ -664,7 +691,7 @@ func LedgerSeries(p Period, rows []Row) (bucket string, pts []SeriesPoint) {
|
||||
}
|
||||
pts[i].addRow(r)
|
||||
for _, d := range Dimensions {
|
||||
if k := r.key(d); kept[d][k] {
|
||||
if k := keys.key(r, d); kept[d][k] {
|
||||
part := pts[i].By[d][k]
|
||||
part.add(r)
|
||||
pts[i].By[d][k] = part
|
||||
|
||||
@@ -746,6 +746,7 @@ func buildRequestBlocks(p Period, f Filter, offset, limit int, gateways, chunks,
|
||||
groups := map[string]map[string]*Share{}
|
||||
chartGroups := map[string]map[string]*Share{}
|
||||
seriesGroups := map[string]map[string]*Share{}
|
||||
keys := keyer{}
|
||||
for _, d := range Dimensions {
|
||||
groups[d] = map[string]*Share{}
|
||||
chartGroups[d] = map[string]*Share{}
|
||||
@@ -791,7 +792,7 @@ func buildRequestBlocks(p Period, f Filter, offset, limit int, gateways, chunks,
|
||||
}
|
||||
if keep || f.Day != "" && chartFilter.keeps(r.Record) {
|
||||
for _, d := range Dimensions {
|
||||
k := r.key(d)
|
||||
k := keys.key(r, d)
|
||||
s := seriesGroups[d][k]
|
||||
if s == nil {
|
||||
s = &Share{ID: k}
|
||||
@@ -822,13 +823,16 @@ func buildRequestBlocks(p Period, f Filter, offset, limit int, gateways, chunks,
|
||||
if d == "agent" {
|
||||
g.Agent = ""
|
||||
}
|
||||
if d == "model" {
|
||||
if d == "model" || d == "modelAt" {
|
||||
g.Model = ""
|
||||
}
|
||||
if d == "modelAt" {
|
||||
g.Provider = ""
|
||||
}
|
||||
if f.Day != "" {
|
||||
g.Day = ""
|
||||
if g.keeps(r.Record) {
|
||||
k := r.key(d)
|
||||
k := keys.key(r, d)
|
||||
if chartGroups[d][k] == nil {
|
||||
chartGroups[d][k] = &Share{ID: k}
|
||||
}
|
||||
@@ -839,7 +843,7 @@ func buildRequestBlocks(p Period, f Filter, offset, limit int, gateways, chunks,
|
||||
if !g.keeps(r.Record) {
|
||||
continue
|
||||
}
|
||||
k := r.key(d)
|
||||
k := keys.key(r, d)
|
||||
s := groups[d][k]
|
||||
if s == nil {
|
||||
s = &Share{ID: k}
|
||||
@@ -909,7 +913,7 @@ func buildRequestBlocks(p Period, f Filter, offset, limit int, gateways, chunks,
|
||||
pt := &out.Series[i]
|
||||
pt.addRow(r)
|
||||
for _, d := range Dimensions {
|
||||
k := r.key(d)
|
||||
k := keys.key(r, d)
|
||||
part := pt.By[d][k]
|
||||
part.add(r)
|
||||
pt.By[d][k] = part
|
||||
@@ -967,9 +971,12 @@ func pageFromLedger(p Period, f Filter, offset, limit int, all Ledgered) Request
|
||||
if d == "agent" {
|
||||
g.Agent = ""
|
||||
}
|
||||
if d == "model" {
|
||||
if d == "model" || d == "modelAt" {
|
||||
g.Model = ""
|
||||
}
|
||||
if d == "modelAt" {
|
||||
g.Provider = ""
|
||||
}
|
||||
out.By[d] = Breakdown(all.Filtered(g).Rows, d)
|
||||
if f.Day != "" {
|
||||
g.Day = ""
|
||||
|
||||
@@ -257,3 +257,49 @@ func TestQueryPageSourceAndIdentityInvalidation(t *testing.T) {
|
||||
os.Remove(Path())
|
||||
check(0, "", false)
|
||||
}
|
||||
|
||||
// One model at two providers is ranked once at each, with each one's own
|
||||
// first token and decode, and a provider picked still ranks the other
|
||||
// (inaction on Discord).
|
||||
func TestRequestPageRanksModelAtEachProvider(t *testing.T) {
|
||||
pageHome(t)
|
||||
now := time.Now()
|
||||
records := []Record{
|
||||
{Time: now, Provider: "zhipu", Agent: "claude", Model: "glm-5.3", Output: 1000, Millis: 2200, TTFT: 200},
|
||||
{Time: now, Provider: "zhipu", Agent: "claude", Model: "glm-5.3", Output: 1000, Millis: 2200, TTFT: 200},
|
||||
{Time: now, Provider: "zcode", Agent: "claude", Model: "glm-5.3", Output: 1000, Millis: 10800, TTFT: 800},
|
||||
{Time: now, Provider: "zcode", Agent: "claude", Model: "kimi-k3", Output: 10, Millis: 100, TTFT: 50},
|
||||
}
|
||||
gateway := &rowChunk{}
|
||||
all := Ledgered{}
|
||||
for i, r := range records {
|
||||
row := Row{Record: r}
|
||||
gateway.add(row, "", int64(i), false)
|
||||
all.Rows = append(all.Rows, row)
|
||||
}
|
||||
f := Filter{Provider: "zhipu", Model: "glm-5.3"}
|
||||
for name, page := range map[string]RequestPage{
|
||||
"compact": buildRequestPage(Today, f, 0, 100, gateway, nil),
|
||||
"ledger": pageFromLedger(Today, f, 0, 100, all),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got := map[string]Share{}
|
||||
for _, s := range page.By["modelAt"] {
|
||||
got[s.ID] = s
|
||||
}
|
||||
fast, slow := got[ModelAtKey("zhipu", "glm-5.3")], got[ModelAtKey("zcode", "glm-5.3")]
|
||||
if len(got) != 3 || fast.Calls != 2 || slow.Calls != 1 || got[ModelAtKey("zcode", "kimi-k3")].Calls != 1 {
|
||||
t.Fatalf("model at each provider = %+v, want glm-5.3 at zhipu (2) and at zcode (1), kimi-k3 at zcode (1)", page.By["modelAt"])
|
||||
}
|
||||
if fast.Timed != 2 || fast.TTFT != 400 || slow.Timed != 1 || slow.TTFT != 800 {
|
||||
t.Fatalf("first tokens mixed between providers: %+v %+v", fast, slow)
|
||||
}
|
||||
if fast.DecodeMs == 0 || slow.DecodeMs == 0 || fast.DecodeOut*1000/int(fast.DecodeMs) <= slow.DecodeOut*1000/int(slow.DecodeMs) {
|
||||
t.Fatalf("decode speeds mixed between providers: %+v %+v", fast, slow)
|
||||
}
|
||||
if page.Total != 2 {
|
||||
t.Fatalf("the provider and model picked no longer filter the requests: %d", page.Total)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user