agents: omp, Pi, Cline and dsh are no longer told a model writes more than its window holds (#338)
models.dev lists some models' output above their context (deepseek-chat: 384000 out, 128000 in), and magpie handed that on as maxTokens. The limit is now kept within the contextWindow written beside it; a model whose window isn't known keeps its output as before. Follow-up to the review of
This commit is contained in:
1 parent
a9ff84fd80
commit
b60cfa6787
6 files changed
+39
-4
No files matched your search
@@ -292,7 +292,7 @@ func magpieProviderJSONAt(shape, catalog, gw string) any {
|
||||
// without it Pi caps every reply at 16384 tokens, a model
|
||||
// that can write 128K included
|
||||
if m.Output > 0 {
|
||||
e["maxTokens"] = m.Output
|
||||
e["maxTokens"] = maxTokens(m)
|
||||
}
|
||||
ms = append(ms, e)
|
||||
}
|
||||
|
||||
@@ -375,7 +375,7 @@ func clineModels(model string) map[string]any {
|
||||
e["contextWindow"] = m.Context
|
||||
}
|
||||
if m.Output > 0 {
|
||||
e["maxTokens"] = m.Output
|
||||
e["maxTokens"] = maxTokens(m)
|
||||
}
|
||||
ms[m.ID] = e
|
||||
}
|
||||
|
||||
@@ -405,7 +405,7 @@ func dshProviderLines(modern bool, effort string) []string {
|
||||
lines = append(lines, fmt.Sprintf(" contextWindow: %d", m.Context))
|
||||
}
|
||||
if m.Output > 0 {
|
||||
lines = append(lines, fmt.Sprintf(" maxTokens: %d", m.Output))
|
||||
lines = append(lines, fmt.Sprintf(" maxTokens: %d", maxTokens(m)))
|
||||
}
|
||||
if m.Images {
|
||||
lines = append(lines, " inputModalities: [text, image]")
|
||||
|
||||
@@ -184,7 +184,7 @@ type ompProviderEntry struct {
|
||||
func ompProvider() ompProviderEntry {
|
||||
ms := []ompModel{}
|
||||
for _, m := range magpieModels("omp") {
|
||||
e := ompModel{ID: m.ID, Name: m.Name, Context: m.Context, MaxTokens: m.Output}
|
||||
e := ompModel{ID: m.ID, Name: m.Name, Context: m.Context, MaxTokens: maxTokens(m)}
|
||||
mode := "effort"
|
||||
switch {
|
||||
case slices.Contains(m.APIs, string(provider.Responses)):
|
||||
|
||||
@@ -86,6 +86,17 @@ func magpieModels(agent string) []catalog.Model {
|
||||
return out
|
||||
}
|
||||
|
||||
// maxTokens is the output limit an agent is handed for m, kept within the
|
||||
// context window it is handed with it: models.dev lists some models' output
|
||||
// above their window (deepseek-chat's 384000 against 128000). An unknown
|
||||
// window leaves the output as it is.
|
||||
func maxTokens(m catalog.Model) int {
|
||||
if m.Context > 0 && m.Output > m.Context {
|
||||
return m.Context
|
||||
}
|
||||
return m.Output
|
||||
}
|
||||
|
||||
// group tags every option with a group name.
|
||||
func group(name string, opts []Option) []Option {
|
||||
for i := range opts {
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/yetone/magpie/internal/edit"
|
||||
"github.com/yetone/magpie/internal/provider"
|
||||
"github.com/yetone/magpie/internal/settings"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// syncHome is a sandbox home with a models.dev catalog that knows glm-4.6's
|
||||
@@ -103,6 +104,29 @@ func TestPiModelsCarryMaxTokens(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// An output limit above the model's window (models.dev lists deepseek-chat's
|
||||
// 384000 against 128000 of context) is cut to the window for every agent
|
||||
// magpie hands maxTokens; one whose window isn't known keeps its output.
|
||||
func TestMaxTokensWithinContextWindow(t *testing.T) {
|
||||
syncHome(t)
|
||||
check := func(limit, want string) {
|
||||
t.Helper()
|
||||
os.WriteFile(catalog.CachePath(), []byte(`{"zai":{"models":{"glm-4.6":{"id":"glm-4.6","name":"GLM-4.6","limit":{`+limit+`}}}}}`), 0o644)
|
||||
catalog.Reset()
|
||||
pi, _ := json.Marshal(magpieProviderJSON("pi"))
|
||||
cline, _ := json.Marshal(clineModels(""))
|
||||
omp, _ := yaml.Marshal(ompProvider())
|
||||
dsh := strings.Join(dshProviderLines(true, ""), "\n")
|
||||
if !strings.Contains(string(pi), `"maxTokens":`+want) || !strings.Contains(string(cline), `"maxTokens":`+want) ||
|
||||
!strings.Contains(string(omp), "maxTokens: "+want) || !strings.Contains(dsh, "maxTokens: "+want) {
|
||||
t.Fatalf("want maxTokens %s:\npi %s\ncline %s\nomp %s\ndsh %s", want, pi, cline, omp, dsh)
|
||||
}
|
||||
}
|
||||
check(`"context":128000,"output":384000`, "128000")
|
||||
check(`"output":384000`, "384000")
|
||||
check(`"context":204800,"output":131072`, "131072")
|
||||
}
|
||||
|
||||
// A provider added after a magpie model was picked reaches the lists agents
|
||||
// keep of magpie's models; a file magpie wrote nothing into stays as it is.
|
||||
func TestSyncCatalogRewritesAgentLists(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user