gateway: a member whose proxy is down goes on to the next without resting (#381) — with the local proxy (127.0.0.1:1082) not listening, Cursor and SuperGrok in a group failed with "proxyconnect tcp: … connection refused", were counted as failing ("other") and set aside on a backoff that grew with every request, so they stayed out after the proxy came back. A 502 whose error is Go's proxyconnect or socks connect is now "proxy": nothing reached the vendor, the next member is asked as for a request a vendor couldn't read, and nobody rests. The Routing page tags it "proxy not reachable" and says the request never reached the vendor, in English and Chinese. TestProxyDownDoesNotRest (HTTP and SOCKS proxies) and routing-proxy.test.cjs (Chromium, WebKit, en/zh) fail without the change; routing-shape, click-scroll, old-webkit pass. go vet, go build -tags nogui (darwin, GOOS=linux), GOOS=windows go build, go test -tags nogui ./... pass. (#381)

This commit is contained in:
yetone committed 2026-10-01 12:44:48 +08:00
1 parent 89672eb966
commit 9547cee9ce
7 files changed
+214 -2

No files matched your search

+12
View File
@@ -1157,6 +1157,18 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro
skipped = append(skipped, c.label()+": "+call.Error)
continue
}
if !last && hw.failed() && failure(hw.code(), hw.errBody()) == failProxy {
// the proxy didn't take the connection: the request never got
// to the vendor, so this one doesn't rest, and a member that
// goes another way (or the proxy back up) may still answer (#381)
if other == nil {
other = &Try{Status: call.Status, Error: call.Error}
}
try.Fail = failProxy
s.trace.update(tr, func(t *Route) { t.Tries[len(t.Tries)-1] = try })
skipped = append(skipped, c.label()+": "+call.Error)
continue
}
if wait, ok := passing(hw.code(), hw.header, again); ok && !last && again < lastRetries && hw.failed() && spentAfter(cands[i+1:]) {
// the others left are out of their allowance (Discord, waroy: a
// Codex account run out, Grok busy a moment): this one is the
+71
View File
@@ -0,0 +1,71 @@
package gateway
import (
"io"
"net"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/yetone/magpie/internal/provider"
)
// TestProxyDownDoesNotRest (#381): a member whose proxy isn't listening
// (127.0.0.1:1082 down, "proxyconnect tcp: … connection refused") never
// reached its vendor. The next member answers, and the first isn't set
// aside: it is asked first again as soon as the proxy is back, not after a
// backoff that grew with every request made while it was down.
func TestProxyDownDoesNotRest(t *testing.T) {
for _, scheme := range []string{"http", "socks5"} {
t.Run(scheme, func(t *testing.T) {
fresh(t)
for _, k := range []string{"HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy", "NO_PROXY", "no_proxy"} {
t.Setenv(k, "")
}
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
down := l.Addr().String()
l.Close() // nothing listens there now
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
io.ReadAll(r.Body)
w.Header().Set("Content-Type", "application/json")
io.WriteString(w, `{"id":"c","object":"chat.completion","choices":[{"index":0,"message":{"role":"assistant","content":"from b"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`)
}))
t.Cleanup(up.Close)
for _, p := range []provider.Provider{
{ID: "a", Name: "A", Key: "ka", Models: []string{"m"}, Chat: "http://a.test/v1", Proxy: scheme + "://" + down},
{ID: "b", Name: "B", Key: "kb", Models: []string{"m"}, Chat: up.URL + "/v1"},
} {
if err := provider.Save(p); err != nil {
t.Fatal(err)
}
}
if err := provider.SaveGroup(provider.Group{Name: "Grok", Members: []string{"a/m", "b/m"}, Routing: provider.Ordered}); err != nil {
t.Fatal(err)
}
s := New()
for i := 0; i < 2; i++ {
code, body := postAs(t, s, "", `{"model":"group/grok","messages":[{"role":"user","content":"hi"}]}`)
if code != 200 || !strings.Contains(body, "from b") {
t.Fatalf("request %d: %d %s", i, code, body)
}
r := s.trace.routes[len(s.trace.routes)-1]
if len(r.Tries) != 2 || r.Tries[0].ID != r.Order[0].ID || r.Order[0].Provider != "a" {
t.Fatalf("request %d: a wasn't asked first: %+v", i, r.Tries)
}
if f := r.Tries[0]; f.Fail != "proxy" || f.Rest != nil || !strings.Contains(f.Error, down) {
t.Fatalf("request %d: a's try %+v", i, f)
}
}
restingUntil.Lock()
n := len(restingUntil.m)
restingUntil.Unlock()
if n != 0 {
t.Fatalf("%d set aside over a proxy that was down", n)
}
})
}
}
+15
View File
@@ -138,10 +138,21 @@ const (
// failShape: the vendor couldn't read the request's shape (#350) — the
// next one is asked, and nobody rests
failShape = "shape"
// failProxy: the proxy magpie sends through (its own setting, the
// *_PROXY variables, the system's) didn't take the connection (#381) —
// nothing reached the vendor, so the next one is asked, and nobody rests
failProxy = "proxy"
)
// proxyDown is the error Go gives when the proxy itself can't be reached,
// over HTTP (proxyconnect) or SOCKS (socks connect).
var proxyDown = regexp.MustCompile(`proxyconnect |socks connect `)
// failure says why a reply failed.
func failure(status int, body []byte) string {
if status == http.StatusBadGateway && proxyDown.Match(body) {
return failProxy
}
if _, ok := provider.Verification(body); ok && (status == 401 || status == 403) {
return failVerify
}
@@ -275,6 +286,10 @@ func (s *Server) restAfterMarked(c candidate, status int, header http.Header, bo
d := fallbackCooldown
why := failure(status, body)
r := Rest{Why: why, Status: status, By: "cooldown"}
if why == failProxy {
// the account is as good as it was; the proxy is the user's to start
return r
}
switch why {
case failCredit:
d, r.By = creditRest, "credit"
+3
View File
@@ -1370,6 +1370,9 @@ const I18N = {
"needs verification": "账号需验证",
"refused (safety filter)": "被安全过滤拒绝",
"request not understood": "请求格式不被接受",
"proxy not reachable": "代理连不上",
"{who}: the proxy magpie goes through didn't take the connection, so the request never reached the vendor and goes on to the next. Nothing is wrong with {who}, so it doesn't rest: once the proxy is up it is asked first again.": "{who}:magpie 所走的代理没有接受连接,请求根本没到厂商,所以转给了下一个。{who} 本身没出问题,所以不用休息:代理恢复后它仍会最先被问。",
"{who}: the proxy magpie goes through didn't take the connection, so the request never reached the vendor, and nobody is left to try: {agent} gets the error. Start the proxy, or change it in Settings.": "{who}:magpie 所走的代理没有接受连接,请求根本没到厂商,也没有别的可以再试,所以 {agent} 收到了这个错误。请启动代理,或在设置里更换。",
"{who} answered {status}: its API couldn't read something in the request that another's may, so it goes on to the next before any of the reply reaches {agent}. Nothing is wrong with {who}, so it doesn't rest.": "{who} 返回 {status}:它的 API 读不懂请求里的某些内容,而别家的可能可以,所以在任何回复到达 {agent} 之前就转给了下一个。{who} 本身没出问题,所以不用休息。",
"{who}'s safety filter refused the request before saying anything, so it goes on to the next before any of the reply reaches {agent}. Nothing is wrong with {who}, so it doesn't rest.": "{who} 的安全过滤在说出任何内容前拒绝了这个请求,所以在任何回复到达 {agent} 之前就转给了下一个。{who} 本身没出问题,所以不用休息。",
"{who}'s safety filter refused the request before saying anything, and nobody is left to try, so {agent} gets an error saying so, not an empty reply to ask again for.": "{who} 的安全过滤在说出任何内容前拒绝了这个请求,且没有其他可试的,所以 {agent} 收到一个说明被拒绝的错误,而不是一个会让它反复重试的空回复。",
+5 -1
View File
@@ -145,7 +145,7 @@
}
const tokens = (n) => n >= 1e6 ? (n / 1e6).toFixed(1) + "M" : n >= 1e3 ? (n / 1e3).toFixed(1) + "k" : String(Math.round(n));
const pct = (n) => Math.round(n) + "%";
const FAIL = { rate: "rate limited", credit: "out of credit", quota: "quota used up", other: "failed", canceled: "canceled", foreign: "another account's reasoning", floor: "reply too short", verify: "needs verification", refused: "refused (safety filter)", shape: "request not understood" };
const FAIL = { rate: "rate limited", credit: "out of credit", quota: "quota used up", other: "failed", canceled: "canceled", foreign: "another account's reasoning", floor: "reply too short", verify: "needs verification", refused: "refused (safety filter)", shape: "request not understood", proxy: "proxy not reachable" };
const failWord = (why) => t(FAIL[why] || "failed");
const API = { anthropic: "Anthropic", chat: "OpenAI", responses: "OpenAI Responses", gemini: "Gemini" };
const MODES = {
@@ -483,6 +483,10 @@
: t("{who}'s safety filter refused the request before saying anything, and nobody is left to try, so {agent} gets an error saying so, not an empty reply to ask again for.", { who: name, agent });
if (tr.fail === "shape")
return t("{who} answered {status}: its API couldn't read something in the request that another's may, so it goes on to the next before any of the reply reaches {agent}. Nothing is wrong with {who}, so it doesn't rest.", { who: name, status: tr.status, agent });
if (tr.fail === "proxy")
return r.tries[i + 1]
? t("{who}: the proxy magpie goes through didn't take the connection, so the request never reached the vendor and goes on to the next. Nothing is wrong with {who}, so it doesn't rest: once the proxy is up it is asked first again.", { who: name })
: t("{who}: the proxy magpie goes through didn't take the connection, so the request never reached the vendor, and nobody is left to try: {agent} gets the error. Start the proxy, or change it in Settings.", { who: name, agent });
if (tr.again)
return t("{who} answered {status} · {fail}, and nobody else is left to ask — a failure that may pass, so it is tried again in {d}, before any of the reply reaches {agent}.",
{ who: name, status: tr.status, fail: failWord(tr.fail), d: took(tr.again), agent });
+9 -1
View File
@@ -421,6 +421,14 @@ it went on to the next and that the first doesn't rest, tagged "request not
understood", in English and Chinese, never "an error another account
wouldn't fix"; picking the request doesn't move the page. The API is faked.
`routing-proxy.test.cjs` opens a group request whose first member's proxy
wasn't listening (#381: "proxyconnect tcp: … 127.0.0.1:1082: connection
refused") and whose next member answered: the Routing page says the request
never reached the vendor, went on to the next and that the first doesn't
rest, tagged "proxy not reachable", in English and Chinese, never "an error
another account wouldn't fix"; picking the request doesn't move the page.
Chromium and WebKit, API faked.
`s3-sync.test.cjs` sets up sync to an S3 bucket from Settings (#296). It
works as follows, in English and Chinese:
@@ -607,7 +615,7 @@ with `/api/plugins` faked.
With Node.js and Playwright available:
```sh
node --test internal/gui/tests/menu-scroll.test.cjs internal/gui/tests/click-scroll.test.cjs internal/gui/tests/panel-fold.test.cjs internal/gui/tests/rollup-short.test.cjs internal/gui/tests/panel-routing.test.cjs internal/gui/tests/gateway-fold.test.cjs internal/gui/tests/routing-kind.test.cjs internal/gui/tests/balance-fix.test.cjs internal/gui/tests/cli-update.test.cjs internal/gui/tests/login-import.test.cjs internal/gui/tests/agy-launch.test.cjs internal/gui/tests/currency.test.cjs internal/gui/tests/usage-ledger.test.cjs internal/gui/tests/update-check.test.cjs internal/gui/tests/omarchy.test.cjs internal/gui/tests/brand.test.cjs internal/gui/tests/library-loading.test.cjs node --test internal/gui/tests/signin-callback.test.cjs internal/gui/tests/routing-steady.test.cjs internal/gui/tests/old-webkit.test.cjs internal/gui/tests/session-terminal.test.cjs internal/gui/tests/text-size.test.cjs internal/gui/tests/zcode-site.test.cjs internal/gui/tests/rail-tip.test.cjs internal/gui/tests/sync-update.test.cjs internal/gui/tests/plugin-signin.test.cjs internal/gui/tests/plugin-market.test.cjs internal/gui/tests/s3-sync.test.cjs internal/gui/tests/routing-wb-refused.test.cjs internal/gui/tests/routing-shape.test.cjs internal/gui/tests/market-have.test.cjs internal/gui/tests/sessions-today.test.cjs internal/gui/tests/routing-side-calls.test.cjs internal/gui/tests/import-all-skills.test.cjs internal/gui/tests/routing-flood.test.cjs internal/gui/tests/routing-busy.test.cjs internal/gui/tests/own-skill.test.cjs internal/gui/tests/routing-manual.test.cjs internal/gui/tests/tray-usages.test.cjs internal/gui/tests/provider-remove.test.cjs internal/gui/tests/signin-link.test.cjs internal/gui/tests/tray-inuse.test.cjs internal/gui/tests/codex-auto-reset.test.cjs internal/gui/tests/routing-auto-reset.test.cjs internal/gui/tests/claude-usage-asked.test.cjs internal/gui/tests/claude-risk.test.cjs internal/gui/tests/library-toggle-status.test.cjs internal/gui/tests/library-all.test.cjs internal/gui/tests/routing-reroute-title.test.cjs internal/gui/tests/plain-names.test.cjs internal/gui/tests/panel-effort.test.cjs internal/gui/tests/agent-models-all-hidden.test.cjs internal/gui/tests/omp-roles.test.cjs internal/gui/tests/claude-effort.test.cjs internal/gui/tests/remote-magpie.test.cjs internal/gui/tests/qoder-cn-tile.test.cjs internal/gui/tests/provider-levels-scroll.test.cjs internal/gui/tests/signin-paste-codex.test.cjs internal/gui/tests/usage-alert.test.cjs internal/gui/tests/quota-families.test.cjs internal/gui/tests/provider-search.test.cjs internal/gui/tests/webdav-settings-refresh.test.cjs internal/gui/tests/lan-docker.test.cjs internal/gui/tests/plugin-key-hint.test.cjs internal/gui/tests/update-hide.test.cjs internal/gui/tests/tray-logos.test.cjs
node --test internal/gui/tests/menu-scroll.test.cjs internal/gui/tests/click-scroll.test.cjs internal/gui/tests/panel-fold.test.cjs internal/gui/tests/rollup-short.test.cjs internal/gui/tests/panel-routing.test.cjs internal/gui/tests/gateway-fold.test.cjs internal/gui/tests/routing-kind.test.cjs internal/gui/tests/balance-fix.test.cjs internal/gui/tests/cli-update.test.cjs internal/gui/tests/login-import.test.cjs internal/gui/tests/agy-launch.test.cjs internal/gui/tests/currency.test.cjs internal/gui/tests/usage-ledger.test.cjs internal/gui/tests/update-check.test.cjs internal/gui/tests/omarchy.test.cjs internal/gui/tests/brand.test.cjs internal/gui/tests/library-loading.test.cjs node --test internal/gui/tests/signin-callback.test.cjs internal/gui/tests/routing-steady.test.cjs internal/gui/tests/old-webkit.test.cjs internal/gui/tests/session-terminal.test.cjs internal/gui/tests/text-size.test.cjs internal/gui/tests/zcode-site.test.cjs internal/gui/tests/rail-tip.test.cjs internal/gui/tests/sync-update.test.cjs internal/gui/tests/plugin-signin.test.cjs internal/gui/tests/plugin-market.test.cjs internal/gui/tests/s3-sync.test.cjs internal/gui/tests/routing-wb-refused.test.cjs internal/gui/tests/routing-shape.test.cjs internal/gui/tests/market-have.test.cjs internal/gui/tests/sessions-today.test.cjs internal/gui/tests/routing-side-calls.test.cjs internal/gui/tests/import-all-skills.test.cjs internal/gui/tests/routing-flood.test.cjs internal/gui/tests/routing-busy.test.cjs internal/gui/tests/own-skill.test.cjs internal/gui/tests/routing-manual.test.cjs internal/gui/tests/tray-usages.test.cjs internal/gui/tests/provider-remove.test.cjs internal/gui/tests/signin-link.test.cjs internal/gui/tests/tray-inuse.test.cjs internal/gui/tests/codex-auto-reset.test.cjs internal/gui/tests/routing-auto-reset.test.cjs internal/gui/tests/claude-usage-asked.test.cjs internal/gui/tests/claude-risk.test.cjs internal/gui/tests/library-toggle-status.test.cjs internal/gui/tests/library-all.test.cjs internal/gui/tests/routing-reroute-title.test.cjs internal/gui/tests/plain-names.test.cjs internal/gui/tests/panel-effort.test.cjs internal/gui/tests/agent-models-all-hidden.test.cjs internal/gui/tests/omp-roles.test.cjs internal/gui/tests/claude-effort.test.cjs internal/gui/tests/remote-magpie.test.cjs internal/gui/tests/qoder-cn-tile.test.cjs internal/gui/tests/provider-levels-scroll.test.cjs internal/gui/tests/signin-paste-codex.test.cjs internal/gui/tests/usage-alert.test.cjs internal/gui/tests/quota-families.test.cjs internal/gui/tests/provider-search.test.cjs internal/gui/tests/webdav-settings-refresh.test.cjs internal/gui/tests/lan-docker.test.cjs internal/gui/tests/plugin-key-hint.test.cjs internal/gui/tests/update-hide.test.cjs internal/gui/tests/tray-logos.test.cjs internal/gui/tests/routing-proxy.test.cjs
```
If Playwright is installed outside the repository, set `NODE_PATH` to the
+99
View File
@@ -0,0 +1,99 @@
// Run with Node's test runner and Playwright on the module path; see README.md.
// A group member whose proxy wasn't listening (#381: 127.0.0.1:1082 down,
// "proxyconnect tcp: … connection refused") hands the request to the next:
// the Routing page's story says the request never reached it and that it
// doesn't rest, in the page's language, rather than "an error another
// account wouldn't fix".
const assert = require("node:assert/strict");
const fs = require("node:fs/promises");
const path = require("node:path");
const { test } = require("node:test");
const { chromium, webkit } = require("playwright");
const assets = path.resolve(__dirname, "../assets");
const now = new Date();
const day = [now.getFullYear(), now.getMonth() + 1, now.getDate()].map((n) => String(n).padStart(2, "0")).join("-");
const at = (i) => new Date(now.getTime() - (i + 1) * 60e3).toISOString();
const vendor = 'Grok (SuperGrok): Post "https://cli-chat-proxy.grok.com/v1/responses": proxyconnect tcp: dial tcp 127.0.0.1:1082: connect: connection refused';
const grok = { id: "grok", provider: "grok", name: "Grok (SuperGrok)", kind: "provider", model: "grok-4.7-build-fast" };
const cursor = { id: "cursor", provider: "cursor", name: "Cursor", kind: "provider", model: "grok-4.7-fast" };
const routes = [{
id: 100, seq: 100, time: at(0), agent: "codex", model: "group/grok", provider: "cursor", order: [grok, cursor],
tries: [
{ id: grok.id, model: grok.model, start: at(0), done: true, status: 502, ms: 3, fail: "proxy", error: vendor },
{ id: cursor.id, model: cursor.model, start: at(0), done: true, status: 200, ms: 800 },
],
done: true, status: 200, ms: 900, fallback: vendor,
}, {
id: 99, seq: 99, time: at(1), agent: "codex", model: "group/grok", provider: "grok", order: [grok, cursor],
tries: [{ id: grok.id, model: grok.model, start: at(1), done: true, status: 200, ms: 500 }], done: true, status: 200, ms: 500,
}];
function serve(lang) {
const state = { agents: [{ id: "codex", name: "Codex", path: "/test/config.toml", fields: [] }], profiles: [], settings: { lang, theme: "light" } };
return async (route) => {
const url = new URL(route.request().url());
const json = (data) => route.fulfill({ json: data });
if (url.pathname === "/boot.js") return route.fulfill({ contentType: "text/javascript", body: `window.bootPrefs = {lang:"${lang}",theme:"light",web:true};` });
if (url.pathname === "/wails/runtime.js") return route.fulfill({ contentType: "text/javascript", body: "export const Window = {};" });
if (url.pathname === "/api/state") return json(state);
if (url.pathname === "/api/plugins") return json({ plugins: [] });
if (url.pathname === "/api/gateway/trace") {
if (url.searchParams.get("wait")) await new Promise((r) => setTimeout(r, 20e3));
return json({ mine: true, now: now.toISOString(), seq: 1, totals: { requests: 0, rerouted: 0, errors: 0 }, routes: [] });
}
if (url.pathname === "/api/gateway/history") {
const d = url.searchParams.get("day");
return json({ cut: false, days: [{ day, requests: routes.length }], routes: d ? routes : [] });
}
if (url.pathname === "/api/groups") return json({ groups: [] });
if (url.pathname === "/api/providers") return json({ providers: [], gateway: { running: true, window: true } });
if (url.pathname.startsWith("/api/")) return json({});
const file = path.join(assets, url.pathname === "/" ? "index.html" : url.pathname);
const contentType = { ".html": "text/html", ".js": "text/javascript", ".css": "text/css", ".svg": "image/svg+xml", ".png": "image/png" }[path.extname(file)];
await route.fulfill({ body: await fs.readFile(file), contentType });
};
}
const want = {
en: { story: "the proxy magpie goes through didn't take the connection, so the request never reached the vendor and goes on to the next", rest: "so it doesn't rest", tag: "502 · proxy not reachable" },
zh: { story: "magpie 所走的代理没有接受连接,请求根本没到厂商,所以转给了下一个", rest: "所以不用休息", tag: "502 · 代理连不上" },
};
for (const engine of (process.env.BROWSER ? [process.env.BROWSER] : ["chromium", "webkit"])) {
for (const lang of ["en", "zh"]) {
test(`${engine} ${lang}: a member whose proxy is down goes on to the next`, async (t) => {
const browser = await (engine === "webkit" ? webkit.launch() : chromium.launch({ channel: "chromium" }));
const context = await browser.newContext({ viewport: { width: 1100, height: 760 }, reducedMotion: "reduce" });
const page = await context.newPage();
page.setDefaultTimeout(5000);
const errors = [];
page.on("pageerror", (e) => errors.push(e.message));
await page.route("**/*", serve(lang));
t.after(async () => {
if (process.env.ARTIFACT_DIR) {
await fs.mkdir(process.env.ARTIFACT_DIR, { recursive: true });
await page.screenshot({ path: path.join(process.env.ARTIFACT_DIR, `${engine}-${lang}-proxy.png`), fullPage: true });
}
await browser.close();
});
await page.goto("http://magpie.test/?view=routing");
await page.locator(".rt-day").nth(1).click();
await page.locator(".rt-req").nth(routes.length - 1).waitFor();
const row = page.locator(".rt-req").nth(0);
const was = await row.evaluate((e) => e.getBoundingClientRect().top);
await row.click();
await page.waitForTimeout(400);
assert(Math.abs((await row.evaluate((e) => e.getBoundingClientRect().top)) - was) <= 1, "picking the request moved the page");
const steps = await page.locator(".rt-steps li").allTextContents();
const story = steps.find((s) => s.includes(want[lang].story));
assert(story, JSON.stringify(steps));
assert(story.includes(want[lang].rest) && story.includes("Grok (SuperGrok)"), story);
assert(!steps.some((s) => s.includes("wouldn't fix") || s.includes("也解决不了")), JSON.stringify(steps));
const text = await page.locator("body").innerText();
assert(text.includes(want[lang].tag), "no tag " + want[lang].tag);
assert.deepEqual(errors, []);
});
}
}