diff --git a/internal/gateway/decide.go b/internal/gateway/decide.go index 68413ad2..45b5f729 100644 --- a/internal/gateway/decide.go +++ b/internal/gateway/decide.go @@ -354,7 +354,7 @@ func (s *Server) systemOne(ctx context.Context, p provider.Provider, model strin // they are; a transport error is named with p. func (s *Server) postDecide(ctx context.Context, p provider.Provider, model string, body []byte) (int, []byte, string, error) { ctx = p.Via(ctx) - u, err := p.DecideURL(ctx) + u, err := p.DecideModelURL(ctx, model) if err != nil { return 0, nil, "", err } @@ -488,7 +488,8 @@ func decideSeat(p provider.Provider, model string) Weighed { // decideAsk is a System One request (body) as via takes it: Vercel's // TypeSafe API takes it as it is; its evaluation models name the model in // a header and ask a noul as a boolean; Workers AI takes the state and -// questions as the input of a run of the model. +// questions as the input of a run of the model, and a Clef there takes +// the request as it is, its model named as Clef names itself. func decideAsk(via, model string, body []byte) []byte { var q map[string]any if via == provider.ViaSystemOne || via == provider.ViaVercel || json.Unmarshal(body, &q) != nil { @@ -506,6 +507,11 @@ func decideAsk(via, model string, body []byte) []byte { b, _ := json.Marshal(q) return b case provider.ViaCloudflare: + if provider.CloudflareClef(model) { + q["model"] = strings.TrimPrefix(model, "@cf/cloudflare/") + b, _ := json.Marshal(q) + return b + } b, _ := json.Marshal(map[string]any{"model": model, "input": q}) return b } diff --git a/internal/gateway/decide_test.go b/internal/gateway/decide_test.go index e28e0155..ac5c8c29 100644 --- a/internal/gateway/decide_test.go +++ b/internal/gateway/decide_test.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "net/http/httptest" + "slices" "strings" "sync" "testing" @@ -733,3 +734,42 @@ func TestJevConfidence(t *testing.T) { } } } + +// A Clef on Workers AI (ARNO on Discord) is asked at its own run, with +// the System One request as it is and the model named as Clef names +// itself, and answers in Cloudflare's envelope. +func TestClefOnCloudflare(t *testing.T) { + j := &jevUp{choice: "crashes", sure: 0.9, score: 1} + var models []string + var mu sync.Mutex + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/client/v4/accounts/acc7/ai/run/@cf/cloudflare/clef-flash" || r.Header.Get("Authorization") != "Bearer kj" { + http.Error(w, `{"success":false,"errors":[{"message":"No route for that URI"}]}`, 400) + return + } + b, _ := io.ReadAll(r.Body) + var q map[string]any + json.Unmarshal(b, &q) + if _, ok := q["input"]; ok || q["questions"] == nil { + http.Error(w, `{"success":false,"errors":[{"message":"bad input"}]}`, 400) + return + } + mu.Lock() + models = append(models, fmt.Sprint(q["model"])) + mu.Unlock() + rec := httptest.NewRecorder() + j.ServeHTTP(rec, httptest.NewRequest("POST", "/v1/systemone", bytes.NewReader(b))) + w.Write([]byte(`{"success":true,"errors":[],"messages":[],"result":` + rec.Body.String() + `}`)) + })) + defer up.Close() + s := jevAt(t, up.URL+"/client/v4/accounts/acc7/ai/run", "jv/@cf/cloudflare/clef-flash", provider.Rule{Use: "b/big", Intent: "crashes"}) + out, r := postOK(t, s, "s1", chat("why does this crash?", nil, 0, `,"reasoning_effort":"low"`)) + if c := r.Rule.Classified; !strings.Contains(out, "from kb") || c.Intent != "crashes" || c.Sure != 0.9 { + t.Fatalf("%s %+v %+v", out, r.Rule, c) + } + mu.Lock() + defer mu.Unlock() + if len(models) == 0 || slices.ContainsFunc(models, func(m string) bool { return m != "clef-flash" }) { + t.Fatalf("Clef asked as %v", models) + } +} diff --git a/internal/provider/decide.go b/internal/provider/decide.go index 88e535ff..7a88b57c 100644 --- a/internal/provider/decide.go +++ b/internal/provider/decide.go @@ -71,7 +71,30 @@ func (p Provider) DecideOnly() bool { // DecidesModel distinguishes Jev from the conversation models a gateway // also serves. A dedicated decision API may use any model name. func (p Provider) DecidesModel(model string) bool { - return p.Decides() && (p.DecideOnly() || jevID(model)) + return p.Decides() && (p.DecideOnly() || jevID(model) || p.DecideVia() == ViaCloudflare && CloudflareClef(model)) +} + +// Cloudflare's own decision models on Workers AI (ARNO on Discord): Clef +// and Clef Flash answer System One's questions as Jev does, but each is +// run at its own address (…/ai/run/@cf/cloudflare/clef) and takes the +// request as it is, its model named "clef" or "clef-flash", where Jev is +// a run of typesafe/jev with the questions as its input. +const ( + CloudflareJev = "typesafe/jev" + CloudflareClefModel = "@cf/cloudflare/clef" + CloudflareClefFlash = "@cf/cloudflare/clef-flash" +) + +// CloudflareClef reports whether model is one of Cloudflare's Clef models. +func CloudflareClef(model string) bool { + return model == CloudflareClefModel || model == CloudflareClefFlash +} + +// cloudflareDecideModels are the decision models Workers AI serves. +var cloudflareDecideModels = []catalog.Model{ + {ID: CloudflareJev, Name: "Jev"}, + {ID: CloudflareClefModel, Name: "Clef"}, + {ID: CloudflareClefFlash, Name: "Clef Flash"}, } // The ways a decision API is asked, by where it is (DecideVia): TypeSafe's @@ -111,7 +134,7 @@ func (p Provider) Jev() string { case ViaVercel, ViaVercelEval: return "typesafe-ai/jev" case ViaCloudflare: - return "typesafe/jev" + return CloudflareJev } if own := p.ownDecideModels(); len(own) > 0 { return own[0].ID @@ -155,7 +178,10 @@ func (p Provider) decideModels() []catalog.Model { } return out } - if p.DecideVia() != ViaSystemOne { + switch p.DecideVia() { + case ViaCloudflare: + return slices.Clone(cloudflareDecideModels) + case ViaVercel, ViaVercelEval: return []catalog.Model{{ID: p.Jev(), Name: "Jev"}} } if own := p.ownDecideModels(); len(own) > 0 { @@ -201,6 +227,16 @@ func (p Provider) DecideURL(ctx context.Context) (string, error) { return p.Decide + "/systemone", nil } +// DecideModelURL is where a question for model is posted: DecideURL, or, +// for a Clef on Workers AI, its own run under it. +func (p Provider) DecideModelURL(ctx context.Context, model string) (string, error) { + u, err := p.DecideURL(ctx) + if err == nil && p.DecideVia() == ViaCloudflare && CloudflareClef(model) { + u += "/" + model + } + return u, err +} + // cloudflareBase splits p's Workers AI base into Cloudflare's API // (…/client/v4) and the account it names, if any. Cloudflare's docs give // …/client/v4/accounts/$CLOUDFLARE_ACCOUNT_ID/ai/run, and a token made @@ -555,7 +591,7 @@ func (p Provider) decideAsked(ctx context.Context) ([]catalog.Model, error) { // AskSystemOne sends model the smallest System One question at p's // decision API, and says why it wasn't answered. func (p Provider) AskSystemOne(ctx context.Context, model string) error { - u, err := p.DecideURL(ctx) + u, err := p.DecideModelURL(ctx, model) if err != nil { return err } diff --git a/internal/provider/decide_test.go b/internal/provider/decide_test.go index ae972037..45e00436 100644 --- a/internal/provider/decide_test.go +++ b/internal/provider/decide_test.go @@ -533,3 +533,51 @@ func TestTinyStreamsStreamOnly(t *testing.T) { p.Account = nil } } + +// Workers AI's own decision models, Clef and Clef Flash, are offered +// beside Jev, routed to by name, and each asked at its own run. +func TestCloudflareClef(t *testing.T) { + h := t.TempDir() + t.Setenv("HOME", h) + t.Setenv("USERPROFILE", h) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(h, ".config")) + t.Setenv("XDG_CACHE_HOME", filepath.Join(h, ".cache")) + p, err := FromPreset("cloudflare-jev") + if err != nil { + t.Fatal(err) + } + p.Key, p.Decide = "k", "https://api.cloudflare.com/client/v4/accounts/acc7/ai/run" + var ids []string + for _, m := range p.Available() { + ids = append(ids, m.ID) + } + if strings.Join(ids, " ") != "typesafe/jev @cf/cloudflare/clef @cf/cloudflare/clef-flash" || p.Jev() != "typesafe/jev" { + t.Fatalf("offered %v, default %s", ids, p.Jev()) + } + if err := Save(p); err != nil { + t.Fatal(err) + } + for _, name := range []string{"cloudflare-jev/@cf/cloudflare/clef-flash", "@cf/cloudflare/clef"} { + got, model, err := RouteDecider(name) + if err != nil || got.ID != "cloudflare-jev" || !strings.HasSuffix(name, model) { + t.Fatalf("%s: %s %s %v", name, got.ID, model, err) + } + } + if !IsDecider("cloudflare-jev/@cf/cloudflare/clef") { + t.Fatal("Clef is not a classifier") + } + for model, want := range map[string]string{ + "typesafe/jev": "https://api.cloudflare.com/client/v4/accounts/acc7/ai/run", + "@cf/cloudflare/clef": "https://api.cloudflare.com/client/v4/accounts/acc7/ai/run/@cf/cloudflare/clef", + "@cf/cloudflare/clef-flash": "https://api.cloudflare.com/client/v4/accounts/acc7/ai/run/@cf/cloudflare/clef-flash", + } { + if u, err := p.DecideModelURL(context.Background(), model); err != nil || u != want { + t.Errorf("%s at %s %v", model, u, err) + } + } + // another System One provider's model named so is no Clef + o := Provider{ID: "o", Name: "O", Key: "k", Decide: "https://api.typesafe.ai/v1", Chat: "https://api.typesafe.ai/v1"} + if o.DecidesModel("@cf/cloudflare/clef") { + t.Fatal("a Clef id outside Workers AI decides") + } +} diff --git a/internal/provider/presets.go b/internal/provider/presets.go index ec921de0..71aec2a5 100644 --- a/internal/provider/presets.go +++ b/internal/provider/presets.go @@ -447,7 +447,9 @@ var presets = []PresetDef{ Decide: "https://ai-gateway.vercel.sh/typesafe", Note: "routes groups · picks model and effort", Website: "https://vercel.com/ai-gateway/models/jev", KeysURL: "https://vercel.com/d?to=%2F%5Bteam%5D%2F%7E%2Fai-gateway%2Fapi-keys"}, - {ID: "cloudflare-jev", Name: "Jev · Cloudflare Workers AI", Icon: "cloudflare-color", Kind: KindRelay, + // Workers AI also serves Cloudflare's own decision models, Clef and + // Clef Flash, on the same key + {ID: "cloudflare-jev", Name: "Cloudflare Workers AI · Jev, Clef", Icon: "cloudflare-color", Kind: KindRelay, Decide: "https://api.cloudflare.com/client/v4", Note: "routes groups · picks model and effort", Website: "https://developers.cloudflare.com/ai/models/typesafe/jev/", KeysURL: "https://dash.cloudflare.com/profile/api-tokens"},