gui tests: TestSetAnswersWhileDevinAndProvidersHang tells that the set didn't wait by order, not a 1.5s clock, and its fake devin ends with the test (#1375)
The 1.5s wall-clock bound failed at 1.6s under a loaded full run. The fake devin CLI now takes 20s and touches a marker when its answer lands; after the requests the test says no marker is there, which no request that didn't wait can see, whatever the load. The bounds are 10s: slack against load on one side and against the fake's 20s on the other, and they still stop a set that waits on the never-answering provider. As ggbdpq proposed on the issue, with one change: their fake ran `sleep 20` and outlived the test (LESSONS: a fake a test starts ends when the test does). This one sleeps in 0.1s steps, stops on a file the test's cleanup writes, and the cleanup waits (5s at most, else an error) until every run of it has ended. Seen: 2 runs alive at cleanup, 0 after. Ran: -count=3 -v pass (sets 5-111ms); -race -count=10 pass; the whole internal/gui package pass; gofmt -l clean; go vet, GOOS=linux and GOOS=windows go vet -tags nogui ./internal/gui clean. Broken by hand (DevinFamilies ignoring DevinNoWait): FAIL, "set claude-opus-5-5 took 21.6s" and "a devin answer landed before the requests were done".
This commit is contained in:
1 parent
3cd0b5f78d
commit
2c36d3f282
1 file changed
+44
-6
@@ -23,9 +23,13 @@ import (
|
||||
// models list` again (several times: its model's family, its options, its
|
||||
// efforts), each taking seconds, and a CLI that gave no list was asked anew
|
||||
// on the next click — the pick took 5–10s to show. Here the devin CLI takes
|
||||
// 4s to answer and a provider's model endpoint never answers; the set, and
|
||||
// 20s to answer and a provider's model endpoint never answers; the set, and
|
||||
// the state after it, must not wait on either, and Claude Code's config is
|
||||
// written as before.
|
||||
// written as before. That they didn't wait is told by order, not a clock:
|
||||
// the fake marks each answer it gives, and no answer may have landed when
|
||||
// the requests are done. The clock bound only stops a set that waits from
|
||||
// running into the fake's 20s; a 1.5s one failed at 1.6s under a loaded
|
||||
// full run (#1375).
|
||||
func TestSetAnswersWhileDevinAndProvidersHang(t *testing.T) {
|
||||
h := t.TempDir()
|
||||
t.Setenv("HOME", h)
|
||||
@@ -38,9 +42,34 @@ func TestSetAnswersWhileDevinAndProvidersHang(t *testing.T) {
|
||||
os.MkdirAll(bin, 0o755)
|
||||
t.Setenv("PATH", bin+string(os.PathListSeparator)+"/usr/bin"+string(os.PathListSeparator)+"/bin")
|
||||
|
||||
// a devin CLI that takes 4s over every answer
|
||||
// a devin CLI that takes 20s over every answer and marks it landed.
|
||||
// It sleeps in short steps and ends as soon as the test does: the
|
||||
// background asks it is given outlive the requests, and would otherwise
|
||||
// outlive the test (each run of it keeps a file in running/ till it ends)
|
||||
devin := filepath.Join(bin, "devin")
|
||||
testenv.Program(t, devin, "#!/bin/sh\nexec sleep 4\n")
|
||||
answered := filepath.Join(h, "devin-answered")
|
||||
stop := filepath.Join(h, "devin-stop")
|
||||
running := filepath.Join(h, "devin-running")
|
||||
os.MkdirAll(running, 0o755)
|
||||
testenv.Program(t, devin, "#!/bin/sh\n"+
|
||||
"touch '"+running+"'/$$\n"+
|
||||
"i=0\n"+
|
||||
"while [ $i -lt 200 ] && [ ! -e '"+stop+"' ]; do sleep 0.1; i=$((i+1)); done\n"+
|
||||
"[ -e '"+stop+"' ] || touch '"+answered+"'\n"+
|
||||
"rm -f '"+running+"'/$$\n")
|
||||
t.Cleanup(func() {
|
||||
os.WriteFile(stop, nil, 0o644)
|
||||
for deadline := time.Now().Add(5 * time.Second); ; time.Sleep(50 * time.Millisecond) {
|
||||
left, _ := os.ReadDir(running)
|
||||
if len(left) == 0 {
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Errorf("%d devin fakes still running after the test", len(left))
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
old := provider.DevinExecutable
|
||||
provider.DevinExecutable = func() string { return devin }
|
||||
t.Cleanup(func() { provider.DevinExecutable = old })
|
||||
@@ -107,7 +136,10 @@ func TestSetAnswersWhileDevinAndProvidersHang(t *testing.T) {
|
||||
for _, v := range []string{"claude-opus-5-5", "slow/slow-1", "claude-sonnet-5-5"} {
|
||||
took, s := set(v)
|
||||
t.Logf("set %s: %v", v, took)
|
||||
if took > 1500*time.Millisecond {
|
||||
// a fast set is milliseconds; one that waits takes the fake's 20s
|
||||
// or, on the provider, never comes back: the bound has slack on
|
||||
// both sides, and the marker below is what pins the guarantee
|
||||
if took > 10*time.Second {
|
||||
t.Errorf("set %s took %v: it waited on the devin CLI or the provider", v, took)
|
||||
}
|
||||
if got := shown(s); got != v {
|
||||
@@ -119,11 +151,17 @@ func TestSetAnswersWhileDevinAndProvidersHang(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res.Body.Close()
|
||||
if took := time.Since(start); took > 1500*time.Millisecond {
|
||||
if took := time.Since(start); took > 10*time.Second {
|
||||
t.Errorf("the state after set %s took %v", v, took)
|
||||
}
|
||||
}
|
||||
|
||||
// no devin answer landed while the requests ran: one that waited on
|
||||
// the CLI came back only after an answer had, whatever the load
|
||||
if _, err := os.Stat(answered); err == nil {
|
||||
t.Error("a devin answer landed before the requests were done: a set or the state waited on the devin CLI")
|
||||
}
|
||||
|
||||
// what is written is what it was: the last pick, Anthropic's own
|
||||
// model, off magpie's gateway
|
||||
if m, _ := edit.GetJSON(settingsPath, "model"); m != "claude-sonnet-5-5" {
|
||||
|
||||
Reference in new issue
Block a user