diff --git a/internal/gateway/affinity.go b/internal/gateway/affinity.go index 13920974..1f80724a 100644 --- a/internal/gateway/affinity.go +++ b/internal/gateway/affinity.go @@ -56,6 +56,7 @@ type Affinity struct { type stick struct { rest string who string // the key or account, however many were on + model string // the model it answered as: a group may have several on one account turn int at time.Time cacheRead int @@ -116,11 +117,19 @@ func affine(scope, mode string, rotate bool, in http.Header, from provider.Proto if had { a.Last, a.LastTurn, a.At, a.CacheRead = st.rest, st.turn, st.at, st.cacheRead } + // the account and the model that answered: a group with Opus and + // Sonnet both on one Claude account, a rule sending the turn to Opus, + // had the turn's next request kept to the account and so to Sonnet, + // its first member there; the account alone when the model is gone at := -1 - for i, c := range cs { - if had && c.who() == st.who { - at, a.Last = i, c.rest // as it goes by now - break + for _, same := range []func(candidate) bool{ + func(c candidate) bool { return c.who() == st.who && c.model == st.model }, + func(c candidate) bool { return c.who() == st.who }, + } { + for i, c := range cs { + if had && at < 0 && same(c) { + at, a.Last = i, c.rest // as it goes by now + } } } switch { @@ -192,7 +201,7 @@ func answered(key string, c candidate, turn, cacheRead int) { now := time.Now() sticks.Lock() defer sticks.Unlock() - sticks.m[key] = stick{rest: c.rest, who: c.who(), turn: turn, at: now, cacheRead: cacheRead} + sticks.m[key] = stick{rest: c.rest, who: c.who(), model: c.model, turn: turn, at: now, cacheRead: cacheRead} if len(sticks.m) > 4096 { for k, st := range sticks.m { if now.Sub(st.at) > stickKeep { diff --git a/internal/gateway/decide.go b/internal/gateway/decide.go index cc28be7d..40fb0c29 100644 --- a/internal/gateway/decide.go +++ b/internal/gateway/decide.go @@ -113,7 +113,7 @@ func effortAsk(prev before) string { // readJev is the verdict in a System One reply to jevBody, the answer // without "none of these" counting when the intents are levels. -func readJev(b []byte, intents []string, levels bool) (verdict, error) { +func readJev(b []byte, intents []string, levels bool, prev before) (verdict, error) { var out struct { Model string `json:"model"` Answers map[string]struct { @@ -130,18 +130,21 @@ func readJev(b []byte, intents []string, levels bool) (verdict, error) { return verdict{}, answerError{fmt.Errorf("not a System One answer")} } var v verdict - q := "intent" - if levels { - q = "level" // every message is at one of them - } - if a, ok := out.Answers[q]; ok && len(intents) > 0 { + if a, ok := out.Answers["intent"]; ok && len(intents) > 0 && !levels { v.Sure = a.Confidence if a.Choice != noIntent && a.Confidence >= jevSure { - for _, in := range intents { - if in == a.Choice { - v.Intent = in - } - } + v.Intent = intentNamed(intents, a.Choice) + } + } + if a, ok := out.Answers["level"]; ok && len(intents) > 0 && levels { + // every message is at one of them, however unsure Jev is: when it + // is torn, the conversation stays at the level of the turn before + // (继续,再加一个基准测试 after a 复杂任务 came out 0.54 to 0.46), + // and a turn with none before takes the likelier + v.Sure = a.Confidence + v.Intent = intentNamed(intents, a.Choice) + if was := intentNamed(intents, prev.Intent); a.Confidence < jevSure && was != "" { + v.Intent = was } } if a, ok := out.Answers["effort"]; ok { @@ -152,6 +155,16 @@ func readJev(b []byte, intents []string, levels bool) (verdict, error) { return v, nil } +// intentNamed is the one of intents named, or "". +func intentNamed(intents []string, name string) string { + for _, in := range intents { + if in == name { + return in + } + } + return "" +} + // levelled is what Jev said of each set of intents: whether they are // levels every message is at (jevLevelled). It holds as long as the // intents do. @@ -216,7 +229,7 @@ func (s *Server) askJev(p provider.Provider, model string, intents []string, pre if err != nil { return verdict{}, err } - return readJev(b, intents, levels) + return readJev(b, intents, levels, prev) } // systemOne posts body to p's System One API and returns the answer, diff --git a/internal/gateway/decide_test.go b/internal/gateway/decide_test.go index 84b51998..77b7b1b8 100644 --- a/internal/gateway/decide_test.go +++ b/internal/gateway/decide_test.go @@ -232,3 +232,39 @@ func TestJevIsToldTheTurnBefore(t *testing.T) { t.Errorf("first turn: %v %q", got.State, got.Questions["intent"].Instructions) } } + +// Levels have every message at one of them however torn Jev is: a torn +// one stays at the turn before's level, or with none before takes the +// likelier. +func TestJevTornLevel(t *testing.T) { + in := []string{"简单任务", "复杂任务"} + b := []byte(`{"model":"jev-1.13.0","answers":{` + + `"intent":{"type":"choice","choice":"复杂任务","confidence":0.43},` + + `"level":{"type":"choice","choice":"复杂任务","confidence":0.09}}}`) + for _, c := range []struct { + prev before + want string + }{ + {before{}, "复杂任务"}, + {before{Intent: "复杂任务"}, "复杂任务"}, + {before{Intent: "简单任务"}, "简单任务"}, + {before{Intent: "gone"}, "复杂任务"}, + } { + if v, err := readJev(b, in, true, c.prev); err != nil || v.Intent != c.want || v.Sure != 0.09 { + t.Fatalf("after %q: %+v %v, want %s", c.prev.Intent, v, err, c.want) + } + } + // a sure level is taken over the turn before's + sure := []byte(`{"answers":{"level":{"type":"choice","choice":"简单任务","confidence":0.86}}}`) + if v, _ := readJev(sure, in, true, before{Intent: "复杂任务"}); v.Intent != "简单任务" { + t.Fatalf("sure: %+v", v) + } + // topics still need Jev sure, and may be none + if v, _ := readJev(b, in, false, before{Intent: "复杂任务"}); v.Intent != "复杂任务" { + t.Fatalf("topic sure enough: %+v", v) + } + unsure := []byte(`{"answers":{"intent":{"type":"choice","choice":"复杂任务","confidence":0.2}}}`) + if v, _ := readJev(unsure, in, false, before{Intent: "复杂任务"}); v.Intent != "" { + t.Fatalf("topic unsure: %+v", v) + } +} diff --git a/internal/gateway/rules_test.go b/internal/gateway/rules_test.go index 05b57a64..56b82765 100644 --- a/internal/gateway/rules_test.go +++ b/internal/gateway/rules_test.go @@ -559,3 +559,49 @@ func TestImageRuleFallbackOmitsOlderImages(t *testing.T) { t.Fatalf("the text-only member was sent the image %d times", a.seen) } } + +// A group may have two of its models on one account (Opus and Sonnet on +// one Claude sign-in): the turn a rule sent to one stays on that one for +// its tool rounds, not on the account's first member. +func TestRuleTurnHoldsItsModelOnASharedAccount(t *testing.T) { + fresh(t) + var mu sync.Mutex + var got []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var q struct { + Model string `json:"model"` + } + b, _ := io.ReadAll(r.Body) + json.Unmarshal(b, &q) + mu.Lock() + got = append(got, q.Model) + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"id":"x","choices":[{"index":0,"message":{"role":"assistant","content":"from `+q.Model+`"},"finish_reason":"stop"}],`+ + `"usage":{"prompt_tokens":3000,"completion_tokens":5,"prompt_tokens_details":{"cached_tokens":2500}}}`) + })) + t.Cleanup(srv.Close) + if err := provider.Save(provider.Provider{ID: "c", Name: "C", Key: "kc", Models: []string{"small", "big"}, Chat: srv.URL + "/v1"}); err != nil { + t.Fatal(err) + } + if err := provider.SaveGroup(provider.Group{Name: "R", Members: []string{"c/small", "c/big"}, Routing: provider.Ordered, + Rules: []provider.Rule{{Use: "c/big", Tokens: 20000}}}); err != nil { + t.Fatal(err) + } + s := New() + if code, out := postAs(t, s, "sess", chat("hello", nil, 0, "")); code != 200 || !strings.Contains(out, "from small") { + t.Fatalf("turn 1: %d %s", code, out) + } + if code, out := postAs(t, s, "sess", chat("hello", []string{long(25000)}, 0, "")); code != 200 || !strings.Contains(out, "from big") { + t.Fatalf("turn 2: %d %s", code, out) + } + for n := 1; n <= 2; n++ { + code, out := postAs(t, s, "sess", chat("hello", []string{long(25000)}, n, "")) + if r := lastRoute(s); code != 200 || !strings.Contains(out, "from big") || !r.Rule.Held || !r.Affinity.Kept { + t.Fatalf("turn 2 round %d: %d %s %+v %+v", n, code, out, r.Rule, r.Affinity) + } + } + if strings.Join(got, ",") != "small,big,big,big" { + t.Fatalf("asked %v", got) + } +}