copilot: Auto falls back to gpt-4.1 first, the reply names the pick that answered, and the route says each pick's API and session token (infinitr0us, #256) — the route data from v0.1.921 showed /auto does answer a Student account: it picked gpt-5.6-luna, which Copilot refused "The requested model is not supported." magpie then went down /models in its order, but Copilot's list no longer flags any model as the chat default or fallback, nor bills any, so the order was the list's own: gpt-5.4-mini, gpt-6-luna, kimi-k3, mai-code-1.1-flash, gpt-5-mini all refused and the request failed with 400 before reaching gpt-4.1, last on the list and the one the plan is served; a second request reached it after claude-haiku-4.5. gpt-4.1 (Copilot's base model) now ranks right after a flagged fallback or default. X-Magpie-Model named mai-code-1.1-flash / claude-haiku-4.5, a refused pick: each new pick now updates it, so it ends on the one sent last. Each pick in tries[].auto now carries api (the path it went to) and session (whether Auto's Copilot-Session-Token went with it), shown on the Routing page's pick line (zh, ja, de strings), since why /auto's own pick is refused is still open: VS Code's /auto request matches magpie's. TestCopilotAutoFallsBackToBase (a Student list as the reporter's, /auto picking a refused model, only gpt-4.1 served) fails on the old code: eight refusals, X-Magpie-Model copilot/claude-haiku-4.5; routing-copilot-auto.test.cjs (WebKit, en and zh) asserts the API/session text. routing-wb-refused, routing-effort, routing-kind, routing-zcode-blocked pass on WebKit; go vet, builds (darwin, GOOS=linux nogui, GOOS=windows) and go test -tags nogui -count=1 ./... under a temp HOME pass.
This commit is contained in:
1 parent
6fb05259d7
commit
251ef21866
7 files changed
+129
-26
No files matched your search
@@ -209,3 +209,74 @@ func TestCopilotAutoPicksInRoute(t *testing.T) {
|
||||
t.Fatalf("the picks said %+v", tries)
|
||||
}
|
||||
}
|
||||
|
||||
// copilotStudentList is a Student plan's /models as Copilot lists it now
|
||||
// (#256, 22:07): nothing flagged as the chat default or fallback, nothing
|
||||
// billed, gpt-4.1 last and out of the picker; the plan is served gpt-4.1
|
||||
// alone.
|
||||
const copilotStudentList = `{"data":[
|
||||
{"id":"gpt-5.4-mini","model_picker_enabled":true,"policy":{"state":"enabled"},"supported_endpoints":["/chat/completions","/responses"],"capabilities":{"type":"chat"}},
|
||||
{"id":"gpt-6-luna","model_picker_enabled":true,"policy":{"state":"enabled"},"supported_endpoints":["/responses"],"capabilities":{"type":"chat"}},
|
||||
{"id":"kimi-k3","model_picker_enabled":true,"policy":{"state":"enabled"},"supported_endpoints":["/chat/completions"],"capabilities":{"type":"chat"}},
|
||||
{"id":"mai-code-1.1-flash","model_picker_enabled":true,"policy":{"state":"enabled"},"supported_endpoints":["/chat/completions"],"capabilities":{"type":"chat"}},
|
||||
{"id":"gpt-5-mini","model_picker_enabled":true,"policy":{"state":"enabled"},"supported_endpoints":["/chat/completions","/responses"],"capabilities":{"type":"chat"}},
|
||||
{"id":"claude-haiku-4.5","model_picker_enabled":true,"policy":{"state":"enabled"},"supported_endpoints":["/chat/completions"],"capabilities":{"type":"chat"}},
|
||||
{"id":"gpt-4.1","model_picker_enabled":false,"model_picker_category":"versatile","policy":{"state":"enabled"},"capabilities":{"type":"chat"}}]}`
|
||||
|
||||
// #256 (infinitr0us, 22:07, v0.1.921): /auto picked gpt-5.6-luna, which
|
||||
// Copilot refused "The requested model is not supported."; magpie then went
|
||||
// down the list in its order, gpt-4.1 last, five more refusals, and the
|
||||
// request failed with 400 before reaching gpt-4.1. The reply named
|
||||
// mai-code-1.1-flash, a refused pick. gpt-4.1 is now the first fallback,
|
||||
// the reply names it, and the route says each pick's API and whether Auto's
|
||||
// session token went with it.
|
||||
func TestCopilotAutoFallsBackToBase(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var asked []string
|
||||
copilotFake(t, "gho_student256base", func(w http.ResponseWriter, r *http.Request) {
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
switch r.URL.Path {
|
||||
case "/models":
|
||||
io.WriteString(w, copilotStudentList)
|
||||
case "/auto":
|
||||
io.WriteString(w, `{"session_token":"v2-tok","selected_model":{"id":"gpt-5.6-luna","supported_endpoints":["/responses"]},"expires_at":`+
|
||||
mustString(time.Now().Add(24*time.Hour).Unix())+`}`)
|
||||
case "/models/session":
|
||||
http.NotFound(w, r)
|
||||
default:
|
||||
m := modelOf(b)
|
||||
asked = append(asked, r.URL.Path+" "+m)
|
||||
if m != "gpt-4.1" {
|
||||
w.WriteHeader(400)
|
||||
io.WriteString(w, copilotNotSupported)
|
||||
return
|
||||
}
|
||||
io.WriteString(w, `{"id":"c1","model":"gpt-4.1","choices":[{"index":0,"message":{"role":"assistant","content":"hello from gpt-4.1"},"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":3}}`)
|
||||
}
|
||||
})
|
||||
s := New()
|
||||
rec := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rec, httptest.NewRequest("POST", "/v1/chat/completions",
|
||||
strings.NewReader(`{"model":"copilot/auto","messages":[{"role":"user","content":"Reply with the single word: ok"}]}`)))
|
||||
if rec.Code != 200 || !strings.Contains(rec.Body.String(), "hello from gpt-4.1") {
|
||||
t.Fatalf("copilot/auto: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := rec.Header().Get("X-Magpie-Model"); got != "copilot/gpt-4.1" {
|
||||
t.Errorf("X-Magpie-Model %q, want copilot/gpt-4.1", got)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if strings.Join(asked, "|") != "/responses gpt-5.6-luna|/chat/completions gpt-4.1" {
|
||||
t.Errorf("Copilot was asked %q", asked)
|
||||
}
|
||||
var picks []provider.AutoPick
|
||||
for _, tr := range lastRoute(s).Tries {
|
||||
picks = append(picks, tr.Auto...)
|
||||
}
|
||||
if len(picks) != 2 || picks[0].Model != "gpt-5.6-luna" || picks[0].Via != "/auto" || picks[0].API != "/responses" || !picks[0].Session ||
|
||||
picks[1].Model != "gpt-4.1" || picks[1].API != "/chat/completions" || picks[1].Session {
|
||||
t.Fatalf("the route told the picks %+v", picks)
|
||||
}
|
||||
}
|
||||
@@ -1559,7 +1559,10 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro
|
||||
ctx, stop := context.WithCancel(r.Context())
|
||||
hw.stop = stop
|
||||
if autoPicks(c) {
|
||||
ctx, autoPicked = provider.WithAutoPicks(ctx)
|
||||
// the reply names the pick it went as last (#256: it
|
||||
// named one Copilot refused)
|
||||
p := c.p
|
||||
ctx, autoPicked = provider.WithAutoPicks(ctx, func(m string) { noteMember(hw, r, p, m) })
|
||||
}
|
||||
// when the request last went out to the vendor, its body
|
||||
// written: what came before is magpie's, what after the
|
||||
|
||||
@@ -2250,6 +2250,8 @@ const I18N = {
|
||||
"Copilot's Auto picked {model} ({via})": "Copilot 的 Auto 选了 {model}({via})",
|
||||
"the model the account may pick by hand": "账号可手动选的模型",
|
||||
"not from /auto: {why}": "未取自 /auto:{why}",
|
||||
"sent to {api}": "发往 {api}",
|
||||
"sent to {api} with Auto's session token": "发往 {api},带 Auto 的会话令牌",
|
||||
"Copilot refused it: {error}": "Copilot 拒绝了它:{error}",
|
||||
"WorkBuddy refuses chats from Codex and Claude Code (their system prompt); use it from Hermes, OpenCode or Pi, or add another provider to this group": "WorkBuddy 会拒绝来自 Codex 和 Claude Code 的对话(因为它们的系统提示词);请在 Hermes、OpenCode 或 Pi 里使用它,或在这个分组里再加一个供应商",
|
||||
"the provider's network firewall blocked requests from this IP; wait a while, or switch to another network or proxy": "供应商的网络防火墙拦截了来自这个 IP 的请求;请稍等一会儿,或换个网络或代理",
|
||||
@@ -5288,6 +5290,8 @@ const I18N = {
|
||||
"Copilot's Auto picked {model} ({via})": "Copilot の Auto が {model} を選択({via})",
|
||||
"the model the account may pick by hand": "アカウントが手動で選べるモデル",
|
||||
"not from /auto: {why}": "/auto からではない:{why}",
|
||||
"sent to {api}": "{api} に送信",
|
||||
"sent to {api} with Auto's session token": "{api} に送信(Auto のセッショントークン付き)",
|
||||
"Copilot refused it: {error}": "Copilot に拒否されました:{error}",
|
||||
"WorkBuddy refuses chats from Codex and Claude Code (their system prompt); use it from Hermes, OpenCode or Pi, or add another provider to this group": "WorkBuddy は Codex と Claude Code からのチャットを(そのシステムプロンプトが原因で)拒否します。Hermes、OpenCode、Pi から使うか、このグループに別のプロバイダを追加してください",
|
||||
"the provider's network firewall blocked requests from this IP; wait a while, or switch to another network or proxy": "プロバイダのネットワークファイアウォールがこの IP からのリクエストをブロックしました。しばらく待つか、別のネットワークやプロキシに切り替えてください",
|
||||
@@ -8319,6 +8323,8 @@ const I18N = {
|
||||
"Copilot's Auto picked {model} ({via})": "Copilots Auto wählte {model} ({via})",
|
||||
"the model the account may pick by hand": "das Modell, das das Konto selbst wählen darf",
|
||||
"not from /auto: {why}": "nicht über /auto: {why}",
|
||||
"sent to {api}": "an {api} gesendet",
|
||||
"sent to {api} with Auto's session token": "an {api} gesendet, mit Autos Sitzungstoken",
|
||||
"Copilot refused it: {error}": "Copilot lehnte es ab: {error}",
|
||||
"WorkBuddy refuses chats from Codex and Claude Code (their system prompt); use it from Hermes, OpenCode or Pi, or add another provider to this group": "WorkBuddy lehnt Chats von Codex und Claude Code ab (wegen ihres System-Prompts); nutzen Sie es aus Hermes, OpenCode oder Pi oder fügen Sie dieser Gruppe einen weiteren Anbieter hinzu",
|
||||
"the provider's network firewall blocked requests from this IP; wait a while, or switch to another network or proxy": "die Netzwerk-Firewall des Anbieters hat Anfragen von dieser IP blockiert; warten Sie eine Weile oder wechseln Sie Netzwerk oder Proxy",
|
||||
|
||||
@@ -1158,10 +1158,12 @@
|
||||
r.tries.forEach((tr, i) => {
|
||||
items.push([tryWhy(r, i), tr.done ? (tryOk(tr) ? "ok" : "bad") : "wait"]);
|
||||
// each model Copilot's Auto picked for it, where from, and whether
|
||||
// Copilot refused it (#256)
|
||||
// Copilot refused it, on which API and with Auto's session token or
|
||||
// without (#256)
|
||||
for (const p of tr.auto || []) {
|
||||
let s = t("Copilot's Auto picked {model} ({via})", { model: p.model, via: p.via === "fallback" ? t("the model the account may pick by hand") : p.via });
|
||||
if (p.skipped) s += " · " + t("not from /auto: {why}", { why: p.skipped });
|
||||
if (p.api) s += " · " + t(p.session ? "sent to {api} with Auto's session token" : "sent to {api}", { api: p.api });
|
||||
if (p.refused) s += " · " + t("Copilot refused it: {error}", { error: p.refused });
|
||||
items.push([s, "aside"]);
|
||||
}
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
// route logged each refused try as "auto", so which model Copilot refused,
|
||||
// and whether /auto or /models/session picked it, wasn't told. Each try's
|
||||
// line now names the model Auto picked, and a line under it says where the
|
||||
// pick came from, why not /auto when it didn't, and Copilot's refusal.
|
||||
// pick came from, why not /auto when it didn't, and Copilot's refusal; and
|
||||
// (v0.1.921's report) which API it went to and whether Auto's session token
|
||||
// went with it.
|
||||
const assert = require("node:assert/strict");
|
||||
const fs = require("node:fs/promises");
|
||||
const path = require("node:path");
|
||||
@@ -20,9 +22,9 @@ const routes = [{
|
||||
id: 100, seq: 100, time: at(0), agent: "opencode", model: "copilot/auto", provider: "copilot", order: [key],
|
||||
tries: [
|
||||
{ id: key.id, model: "auto", start: at(0), done: true, status: 400, ms: 300, error: refused,
|
||||
auto: [{ model: "gpt-5.3-codex", via: "/auto", refused }] },
|
||||
auto: [{ model: "gpt-5.3-codex", via: "/auto", refused, api: "/responses", session: true }] },
|
||||
{ id: key.id, model: "auto", start: at(0), done: true, status: 200, ms: 400,
|
||||
auto: [{ model: "gpt-4.1", via: "fallback", skipped: "picked gpt-5.3-codex, which the account was refused" }] },
|
||||
auto: [{ model: "gpt-4.1", via: "fallback", skipped: "picked gpt-5.3-codex, which the account was refused", api: "/chat/completions" }] },
|
||||
],
|
||||
done: true, status: 200, ms: 700,
|
||||
}];
|
||||
@@ -55,12 +57,12 @@ function serve(lang) {
|
||||
|
||||
const want = {
|
||||
en: [
|
||||
"Copilot's Auto picked gpt-5.3-codex (/auto) · Copilot refused it: " + refused,
|
||||
"Copilot's Auto picked gpt-4.1 (the model the account may pick by hand) · not from /auto: picked gpt-5.3-codex, which the account was refused",
|
||||
"Copilot's Auto picked gpt-5.3-codex (/auto) · sent to /responses with Auto's session token · Copilot refused it: " + refused,
|
||||
"Copilot's Auto picked gpt-4.1 (the model the account may pick by hand) · not from /auto: picked gpt-5.3-codex, which the account was refused · sent to /chat/completions",
|
||||
],
|
||||
zh: [
|
||||
"Copilot 的 Auto 选了 gpt-5.3-codex(/auto) · Copilot 拒绝了它:" + refused,
|
||||
"Copilot 的 Auto 选了 gpt-4.1(账号可手动选的模型) · 未取自 /auto:picked gpt-5.3-codex, which the account was refused",
|
||||
"Copilot 的 Auto 选了 gpt-5.3-codex(/auto) · 发往 /responses,带 Auto 的会话令牌 · Copilot 拒绝了它:" + refused,
|
||||
"Copilot 的 Auto 选了 gpt-4.1(账号可手动选的模型) · 未取自 /auto:picked gpt-5.3-codex, which the account was refused · 发往 /chat/completions",
|
||||
],
|
||||
};
|
||||
|
||||
|
||||
@@ -1235,6 +1235,10 @@ func copilotAccept(ctx context.Context, app copilotApp, s copilotSession, model
|
||||
}
|
||||
|
||||
// copilotModels asks Copilot which chat models this account may use.
|
||||
// copilotBaseModel is Copilot's base model, billed to no allowance on any
|
||||
// plan.
|
||||
const copilotBaseModel = "gpt-4.1"
|
||||
|
||||
func copilotModels(ctx context.Context, app copilotApp) ([]catalog.Model, error) {
|
||||
s, err := app.session(ctx)
|
||||
if err != nil {
|
||||
@@ -1306,18 +1310,23 @@ func copilotModels(ctx context.Context, app copilotApp) ([]catalog.Model, error)
|
||||
case m.Policy != nil && m.Policy.State == "enabled", m.Policy == nil && m.Picker:
|
||||
picks = append(picks, m.ID)
|
||||
// Copilot's base model (VS Code's copilot-base: the list's
|
||||
// is_chat_fallback), then its default, then one billed to no
|
||||
// premium allowance: what a plan that may pick little (a
|
||||
// Student's) is likeliest to be served
|
||||
// is_chat_fallback), then its default, then gpt-4.1, which
|
||||
// the list no longer flags as either but is the one a
|
||||
// Student plan is served picked by hand (#256: it was last
|
||||
// of seven picks, the six before it refused), then one billed
|
||||
// to no premium allowance: what a plan that may pick little
|
||||
// is likeliest to be served
|
||||
switch {
|
||||
case m.Fallback:
|
||||
rank[m.ID] = 0
|
||||
case m.Default:
|
||||
rank[m.ID] = 1
|
||||
case m.Billing != nil && !m.Billing.Premium:
|
||||
case m.ID == copilotBaseModel:
|
||||
rank[m.ID] = 2
|
||||
default:
|
||||
case m.Billing != nil && !m.Billing.Premium:
|
||||
rank[m.ID] = 3
|
||||
default:
|
||||
rank[m.ID] = 4
|
||||
}
|
||||
case m.Policy != nil && m.Policy.Terms != "":
|
||||
waiting[m.ID] = true
|
||||
|
||||
@@ -83,25 +83,31 @@ const AutoFallback = "fallback"
|
||||
// AutoPick is a model Copilot's Auto picked for a try of a request: where
|
||||
// the pick came from, why not from /auto when it didn't, and Copilot's
|
||||
// refusal of it, which the route shows (#256: a Student plan's picks were
|
||||
// refused, the tries logged as "auto").
|
||||
// refused, the tries logged as "auto"). API is the path it was sent to
|
||||
// (/responses, /chat/completions), Session whether Auto's session token
|
||||
// went with it.
|
||||
type AutoPick struct {
|
||||
Model string `json:"model"`
|
||||
Via string `json:"via"`
|
||||
Skipped string `json:"skipped,omitempty"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
API string `json:"api,omitempty"`
|
||||
Session bool `json:"session,omitempty"`
|
||||
}
|
||||
|
||||
type autoPicks struct {
|
||||
mu sync.Mutex
|
||||
list []AutoPick
|
||||
mu sync.Mutex
|
||||
list []AutoPick
|
||||
onPick func(model string)
|
||||
}
|
||||
|
||||
type autoPicksKey struct{}
|
||||
|
||||
// WithAutoPicks has the Auto picks a request is sent with noted: the func
|
||||
// returns them.
|
||||
func WithAutoPicks(ctx context.Context) (context.Context, func() []AutoPick) {
|
||||
p := &autoPicks{}
|
||||
// returns them. onPick, when set, is told each model the request goes as,
|
||||
// for the reply to name the one it went as last.
|
||||
func WithAutoPicks(ctx context.Context, onPick func(model string)) (context.Context, func() []AutoPick) {
|
||||
p := &autoPicks{onPick: onPick}
|
||||
return context.WithValue(ctx, autoPicksKey{}, p), func() []AutoPick {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
@@ -109,18 +115,22 @@ func WithAutoPicks(ctx context.Context) (context.Context, func() []AutoPick) {
|
||||
}
|
||||
}
|
||||
|
||||
// notePick notes the request is sent with a: once, while it isn't refused.
|
||||
func notePick(ctx context.Context, a copilotAutoSession) {
|
||||
// notePick notes req is sent with a: once, while it isn't refused.
|
||||
func notePick(ctx context.Context, a copilotAutoSession, req *http.Request) {
|
||||
p, ok := ctx.Value(autoPicksKey{}).(*autoPicks)
|
||||
if !ok || a.Model == "" {
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if n := len(p.list); n > 0 && p.list[n-1].Model == a.Model && p.list[n-1].Via == a.Via && p.list[n-1].Refused == "" {
|
||||
pick := AutoPick{Model: a.Model, Via: a.Via, Skipped: a.Skipped, API: req.URL.Path, Session: req.Header.Get("Copilot-Session-Token") != ""}
|
||||
if n := len(p.list); n > 0 && p.list[n-1].Model == a.Model && p.list[n-1].Via == a.Via && p.list[n-1].API == pick.API && p.list[n-1].Refused == "" {
|
||||
return
|
||||
}
|
||||
p.list = append(p.list, AutoPick{Model: a.Model, Via: a.Via, Skipped: a.Skipped})
|
||||
p.list = append(p.list, pick)
|
||||
if p.onPick != nil {
|
||||
p.onPick(a.Model)
|
||||
}
|
||||
if a.Skipped != "" {
|
||||
log.Printf("copilot auto: %s picked %s (/auto: %s)", a.Via, a.Model, a.Skipped)
|
||||
}
|
||||
@@ -379,7 +389,7 @@ func copilotAutoSign(ctx context.Context, app copilotApp, req *http.Request, bod
|
||||
if a.Token != "" {
|
||||
req.Header.Set("Copilot-Session-Token", a.Token)
|
||||
}
|
||||
notePick(ctx, a)
|
||||
notePick(ctx, a, req)
|
||||
return model, nil
|
||||
}
|
||||
// Auto's pick sent again once the account was refused it
|
||||
@@ -403,7 +413,7 @@ func copilotAutoSign(ctx context.Context, app copilotApp, req *http.Request, bod
|
||||
if a.Token != "" {
|
||||
req.Header.Set("Copilot-Session-Token", a.Token)
|
||||
}
|
||||
notePick(ctx, a)
|
||||
notePick(ctx, a, req)
|
||||
return a.Model, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user