769 lines
22 KiB
Go
769 lines
22 KiB
Go
package control
|
|
|
|
import (
|
|
"context"
|
|
"reflect"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"gpt-load/internal/dialect"
|
|
"gpt-load/internal/health"
|
|
"gpt-load/internal/protocol"
|
|
"gpt-load/internal/state"
|
|
)
|
|
|
|
type autoWeightWrite struct {
|
|
keyID uint
|
|
weight int
|
|
}
|
|
|
|
type fakeAutoWeightRegistry struct {
|
|
mu sync.Mutex
|
|
activeIDs []uint
|
|
reject map[uint]bool
|
|
writes []autoWeightWrite
|
|
wrote chan autoWeightWrite
|
|
}
|
|
|
|
func newFakeAutoWeightRegistry(activeIDs ...uint) *fakeAutoWeightRegistry {
|
|
return &fakeAutoWeightRegistry{
|
|
activeIDs: append([]uint(nil), activeIDs...),
|
|
reject: make(map[uint]bool),
|
|
wrote: make(chan autoWeightWrite, 32),
|
|
}
|
|
}
|
|
|
|
func (registry *fakeAutoWeightRegistry) ActiveKeyIDs() []uint {
|
|
registry.mu.Lock()
|
|
defer registry.mu.Unlock()
|
|
return append([]uint(nil), registry.activeIDs...)
|
|
}
|
|
|
|
func (registry *fakeAutoWeightRegistry) SetAutoWeight(keyID uint, weight int) bool {
|
|
write := autoWeightWrite{keyID: keyID, weight: weight}
|
|
registry.mu.Lock()
|
|
registry.writes = append(registry.writes, write)
|
|
rejected := registry.reject[keyID]
|
|
registry.mu.Unlock()
|
|
registry.wrote <- write
|
|
return !rejected
|
|
}
|
|
|
|
func (registry *fakeAutoWeightRegistry) snapshotWrites() []autoWeightWrite {
|
|
registry.mu.Lock()
|
|
defer registry.mu.Unlock()
|
|
return append([]autoWeightWrite(nil), registry.writes...)
|
|
}
|
|
|
|
type fakeRuntimeTicker struct {
|
|
ticks chan time.Time
|
|
stopped chan struct{}
|
|
stopOnce sync.Once
|
|
}
|
|
|
|
func newFakeRuntimeTicker() *fakeRuntimeTicker {
|
|
return &fakeRuntimeTicker{
|
|
ticks: make(chan time.Time, 8),
|
|
stopped: make(chan struct{}),
|
|
}
|
|
}
|
|
|
|
func (ticker *fakeRuntimeTicker) C() <-chan time.Time {
|
|
return ticker.ticks
|
|
}
|
|
|
|
func (ticker *fakeRuntimeTicker) Stop() {
|
|
ticker.stopOnce.Do(func() { close(ticker.stopped) })
|
|
}
|
|
|
|
type fakeValidationSweep struct {
|
|
started chan struct{}
|
|
returned chan struct{}
|
|
block bool
|
|
once sync.Once
|
|
}
|
|
|
|
func newFakeValidationSweep(block bool) *fakeValidationSweep {
|
|
return &fakeValidationSweep{
|
|
started: make(chan struct{}),
|
|
returned: make(chan struct{}),
|
|
block: block,
|
|
}
|
|
}
|
|
|
|
func (sweep *fakeValidationSweep) Validate(ctx context.Context) {
|
|
sweep.once.Do(func() { close(sweep.started) })
|
|
if sweep.block {
|
|
<-ctx.Done()
|
|
}
|
|
close(sweep.returned)
|
|
}
|
|
|
|
type fakeRuntimeClock struct {
|
|
mu sync.Mutex
|
|
now time.Time
|
|
}
|
|
|
|
type barrierRuntimeMutationCoordinator struct {
|
|
entered chan struct{}
|
|
releaseEntry chan struct{}
|
|
callbackDone chan struct{}
|
|
releaseExit chan struct{}
|
|
}
|
|
|
|
func (coordinator *barrierRuntimeMutationCoordinator) Do(_ uint, fn func()) {
|
|
close(coordinator.entered)
|
|
<-coordinator.releaseEntry
|
|
fn()
|
|
close(coordinator.callbackDone)
|
|
<-coordinator.releaseExit
|
|
}
|
|
|
|
func (clock *fakeRuntimeClock) set(now time.Time) {
|
|
clock.mu.Lock()
|
|
clock.now = now
|
|
clock.mu.Unlock()
|
|
}
|
|
|
|
func (clock *fakeRuntimeClock) current() time.Time {
|
|
clock.mu.Lock()
|
|
defer clock.mu.Unlock()
|
|
return clock.now
|
|
}
|
|
|
|
type controlledRequestLogCleaner struct {
|
|
calls chan time.Time
|
|
release chan struct{}
|
|
returned chan struct{}
|
|
active atomic.Int64
|
|
maxActive atomic.Int64
|
|
ignoreCancel bool
|
|
}
|
|
|
|
func newControlledRequestLogCleaner(ignoreCancel bool) *controlledRequestLogCleaner {
|
|
return &controlledRequestLogCleaner{
|
|
calls: make(chan time.Time, 8),
|
|
release: make(chan struct{}, 8),
|
|
returned: make(chan struct{}, 8),
|
|
ignoreCancel: ignoreCancel,
|
|
}
|
|
}
|
|
|
|
func (cleaner *controlledRequestLogCleaner) Sweep(ctx context.Context, now time.Time) {
|
|
active := cleaner.active.Add(1)
|
|
for {
|
|
maxActive := cleaner.maxActive.Load()
|
|
if active <= maxActive || cleaner.maxActive.CompareAndSwap(maxActive, active) {
|
|
break
|
|
}
|
|
}
|
|
cleaner.calls <- now
|
|
if cleaner.ignoreCancel {
|
|
<-cleaner.release
|
|
} else {
|
|
select {
|
|
case <-cleaner.release:
|
|
case <-ctx.Done():
|
|
}
|
|
}
|
|
cleaner.active.Add(-1)
|
|
cleaner.returned <- struct{}{}
|
|
}
|
|
|
|
type controlledOperationRecovery struct {
|
|
started chan struct{}
|
|
returned chan struct{}
|
|
}
|
|
|
|
func (recovery *controlledOperationRecovery) RunOperationRecovery(ctx context.Context) {
|
|
close(recovery.started)
|
|
<-ctx.Done()
|
|
close(recovery.returned)
|
|
}
|
|
|
|
func TestRuntimeRunsOperationRecoveryUntilCancellation(t *testing.T) {
|
|
recovery := &controlledOperationRecovery{
|
|
started: make(chan struct{}),
|
|
returned: make(chan struct{}),
|
|
}
|
|
runtime, _, _, created := newRuntimeHarness(
|
|
newFakeAutoWeightRegistry(1),
|
|
health.NewStatsStore(),
|
|
newFakeValidationSweep(false),
|
|
time.Now,
|
|
)
|
|
runtime.operationRecovery = recovery
|
|
|
|
cancel, done := startRuntime(t, runtime)
|
|
awaitTickers(t, created)
|
|
awaitSignal(t, recovery.started)
|
|
cancel()
|
|
awaitSignal(t, recovery.returned)
|
|
awaitSignal(t, done)
|
|
}
|
|
|
|
func TestRuntimeSweepsRequestLogsImmediatelyAndHourlyWithoutOverlap(t *testing.T) {
|
|
base := time.Date(2026, time.July, 24, 12, 0, 0, 0, time.UTC)
|
|
clock := &fakeRuntimeClock{now: base}
|
|
cleaner := newControlledRequestLogCleaner(false)
|
|
autoTicker := newFakeRuntimeTicker()
|
|
validationTicker := newFakeRuntimeTicker()
|
|
retentionTicker := newFakeRuntimeTicker()
|
|
created := make(chan time.Duration, 3)
|
|
runtime := newTestRuntime(
|
|
newFakeAutoWeightRegistry(1),
|
|
health.NewStatsStore(),
|
|
newFakeValidationSweep(false),
|
|
autoTicker,
|
|
validationTicker,
|
|
created,
|
|
clock.current,
|
|
)
|
|
runtime.requestLogCleaner = cleaner
|
|
runtime.newTicker = func(interval time.Duration) runtimeTicker {
|
|
created <- interval
|
|
switch interval {
|
|
case 30 * time.Second:
|
|
return autoTicker
|
|
case 32 * time.Minute:
|
|
return validationTicker
|
|
case time.Hour:
|
|
return retentionTicker
|
|
default:
|
|
testingPanic("unexpected ticker interval", interval)
|
|
return nil
|
|
}
|
|
}
|
|
|
|
cancel, done := startRuntime(t, runtime)
|
|
if interval := awaitValue(t, created); interval != 30*time.Second {
|
|
t.Fatalf("auto-weight ticker interval = %v, want 30s", interval)
|
|
}
|
|
if interval := awaitValue(t, created); interval != 32*time.Minute {
|
|
t.Fatalf("validation ticker interval = %v, want 32m", interval)
|
|
}
|
|
if interval := awaitValue(t, created); interval != time.Hour {
|
|
t.Fatalf("retention ticker interval = %v, want 1h", interval)
|
|
}
|
|
if got := awaitValue(t, cleaner.calls); !got.Equal(base) {
|
|
t.Fatalf("immediate Sweep time = %v, want %v", got, base)
|
|
}
|
|
|
|
clock.set(base.Add(time.Hour))
|
|
retentionTicker.ticks <- base.Add(99 * time.Hour)
|
|
select {
|
|
case got := <-cleaner.calls:
|
|
t.Fatalf("overlapping Sweep started at %v before first returned", got)
|
|
case <-time.After(25 * time.Millisecond):
|
|
}
|
|
cleaner.release <- struct{}{}
|
|
awaitSignal(t, cleaner.returned)
|
|
if got := awaitValue(t, cleaner.calls); !got.Equal(base.Add(time.Hour)) {
|
|
t.Fatalf("hourly Sweep time = %v, want injected clock %v", got, base.Add(time.Hour))
|
|
}
|
|
cleaner.release <- struct{}{}
|
|
awaitSignal(t, cleaner.returned)
|
|
|
|
stopRuntime(t, cancel, done)
|
|
awaitSignal(t, autoTicker.stopped)
|
|
awaitSignal(t, validationTicker.stopped)
|
|
awaitSignal(t, retentionTicker.stopped)
|
|
if got := cleaner.maxActive.Load(); got != 1 {
|
|
t.Fatalf("maximum concurrent Sweeps = %d, want 1", got)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeCancellationWaitsForRetentionSweep(t *testing.T) {
|
|
cleaner := newControlledRequestLogCleaner(true)
|
|
autoTicker := newFakeRuntimeTicker()
|
|
validationTicker := newFakeRuntimeTicker()
|
|
retentionTicker := newFakeRuntimeTicker()
|
|
created := make(chan time.Duration, 3)
|
|
runtime := newTestRuntime(
|
|
newFakeAutoWeightRegistry(1),
|
|
health.NewStatsStore(),
|
|
newFakeValidationSweep(false),
|
|
autoTicker,
|
|
validationTicker,
|
|
created,
|
|
time.Now,
|
|
)
|
|
runtime.requestLogCleaner = cleaner
|
|
runtime.newTicker = func(interval time.Duration) runtimeTicker {
|
|
created <- interval
|
|
switch interval {
|
|
case 30 * time.Second:
|
|
return autoTicker
|
|
case 32 * time.Minute:
|
|
return validationTicker
|
|
case time.Hour:
|
|
return retentionTicker
|
|
default:
|
|
testingPanic("unexpected ticker interval", interval)
|
|
return nil
|
|
}
|
|
}
|
|
|
|
cancel, done := startRuntime(t, runtime)
|
|
awaitTickers(t, created)
|
|
if interval := awaitValue(t, created); interval != time.Hour {
|
|
t.Fatalf("retention ticker interval = %v, want 1h", interval)
|
|
}
|
|
_ = awaitValue(t, cleaner.calls)
|
|
cancel()
|
|
select {
|
|
case <-done:
|
|
t.Fatal("Runtime.Run returned before active retention Sweep completed")
|
|
case <-time.After(25 * time.Millisecond):
|
|
}
|
|
cleaner.release <- struct{}{}
|
|
awaitSignal(t, cleaner.returned)
|
|
awaitSignal(t, done)
|
|
awaitSignal(t, retentionTicker.stopped)
|
|
}
|
|
|
|
func TestRuntimeCreatesAutoWeightAndJitteredValidationTickers(t *testing.T) {
|
|
registry := newFakeAutoWeightRegistry(1)
|
|
autoTicker := newFakeRuntimeTicker()
|
|
validationTicker := newFakeRuntimeTicker()
|
|
created := make(chan time.Duration, 2)
|
|
runtime := newTestRuntime(registry, health.NewStatsStore(), newFakeValidationSweep(false), autoTicker, validationTicker, created, time.Now)
|
|
runtime.validationJitter = func() time.Duration { return 2 * time.Minute }
|
|
|
|
cancel, done := startRuntime(t, runtime)
|
|
if interval := awaitValue(t, created); interval != 30*time.Second {
|
|
t.Fatalf("auto-weight ticker interval = %v, want 30s", interval)
|
|
}
|
|
if interval := awaitValue(t, created); interval != 32*time.Minute {
|
|
t.Fatalf("validation ticker interval = %v, want 32m", interval)
|
|
}
|
|
stopRuntime(t, cancel, done)
|
|
awaitSignal(t, autoTicker.stopped)
|
|
awaitSignal(t, validationTicker.stopped)
|
|
}
|
|
|
|
func TestRuntimeWaitsForTickBeforeRecompute(t *testing.T) {
|
|
registry := newFakeAutoWeightRegistry(1)
|
|
runtime, autoTicker, _, created := newRuntimeHarness(registry, health.NewStatsStore(), newFakeValidationSweep(false), time.Now)
|
|
|
|
cancel, done := startRuntime(t, runtime)
|
|
awaitTickers(t, created)
|
|
if got := registry.snapshotWrites(); len(got) != 0 {
|
|
t.Fatalf("writes before first tick = %v, want none", got)
|
|
}
|
|
stopRuntime(t, cancel, done)
|
|
awaitSignal(t, autoTicker.stopped)
|
|
}
|
|
|
|
func TestRuntimeRecomputesEveryActiveKey(t *testing.T) {
|
|
registry := newFakeAutoWeightRegistry(3, 7, 9)
|
|
runtime, autoTicker, _, created := newRuntimeHarness(registry, health.NewStatsStore(), newFakeValidationSweep(false), time.Now)
|
|
|
|
cancel, done := startRuntime(t, runtime)
|
|
awaitTickers(t, created)
|
|
for tick := 0; tick < 2; tick++ {
|
|
autoTicker.ticks <- time.Now()
|
|
for _, keyID := range []uint{3, 7, 9} {
|
|
if got, want := awaitValue(t, registry.wrote), (autoWeightWrite{keyID: keyID, weight: state.DefaultWeight}); got != want {
|
|
t.Fatalf("tick %d write = %#v, want %#v", tick+1, got, want)
|
|
}
|
|
}
|
|
}
|
|
stopRuntime(t, cancel, done)
|
|
|
|
want := []autoWeightWrite{
|
|
{keyID: 3, weight: 50}, {keyID: 7, weight: 50}, {keyID: 9, weight: 50},
|
|
{keyID: 3, weight: 50}, {keyID: 7, weight: 50}, {keyID: 9, weight: 50},
|
|
}
|
|
if got := registry.snapshotWrites(); !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("writes = %#v, want %#v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeResetsExpiredStatsToDefaultWeight(t *testing.T) {
|
|
base := time.Date(2026, time.July, 22, 12, 0, 0, 0, time.UTC)
|
|
stats := health.NewStatsStore()
|
|
for sample := 0; sample < 10; sample++ {
|
|
stats.Record(1, true, base)
|
|
}
|
|
registry := newFakeAutoWeightRegistry(1)
|
|
clock := &fakeRuntimeClock{now: base}
|
|
runtime, autoTicker, _, created := newRuntimeHarness(registry, stats, newFakeValidationSweep(false), clock.current)
|
|
|
|
cancel, done := startRuntime(t, runtime)
|
|
awaitTickers(t, created)
|
|
autoTicker.ticks <- base
|
|
if got, want := awaitValue(t, registry.wrote), (autoWeightWrite{keyID: 1, weight: 92}); got != want {
|
|
t.Fatalf("write with populated window = %#v, want %#v", got, want)
|
|
}
|
|
|
|
clock.set(base.Add(5 * time.Minute))
|
|
autoTicker.ticks <- base.Add(5 * time.Minute)
|
|
if got, want := awaitValue(t, registry.wrote), (autoWeightWrite{keyID: 1, weight: state.DefaultWeight}); got != want {
|
|
t.Fatalf("write after window expiry = %#v, want %#v", got, want)
|
|
}
|
|
stopRuntime(t, cancel, done)
|
|
}
|
|
|
|
func TestRuntimeContinuesWhenKeyDisappears(t *testing.T) {
|
|
registry := newFakeAutoWeightRegistry(1, 2, 3)
|
|
registry.reject[2] = true
|
|
runtime, autoTicker, _, created := newRuntimeHarness(registry, health.NewStatsStore(), newFakeValidationSweep(false), time.Now)
|
|
|
|
cancel, done := startRuntime(t, runtime)
|
|
awaitTickers(t, created)
|
|
autoTicker.ticks <- time.Now()
|
|
for _, keyID := range []uint{1, 2, 3} {
|
|
if got, want := awaitValue(t, registry.wrote), (autoWeightWrite{keyID: keyID, weight: state.DefaultWeight}); got != want {
|
|
t.Fatalf("write = %#v, want %#v", got, want)
|
|
}
|
|
}
|
|
stopRuntime(t, cancel, done)
|
|
}
|
|
|
|
func TestRuntimeUpdatesRegistryWeightSeenByCandidateCollection(t *testing.T) {
|
|
base := time.Date(2026, time.July, 22, 12, 0, 0, 0, time.UTC)
|
|
registry := state.NewKeyRegistry()
|
|
if err := registry.Replace([]state.KeyEntry{{
|
|
ID: 1, GroupID: 10, Status: state.KeyStatusActive, EncryptedValue: "cipher-one",
|
|
}}); err != nil {
|
|
t.Fatalf("Replace() error = %v", err)
|
|
}
|
|
stats := health.NewStatsStore()
|
|
for sample := 0; sample < 10; sample++ {
|
|
stats.Record(1, true, base)
|
|
}
|
|
|
|
runtime := &Runtime{
|
|
registry: registry, stats: stats, mutations: health.NewMutationCoordinator(),
|
|
}
|
|
runtime.recompute(base)
|
|
candidates := registry.CollectCandidates([]uint{10}, nil, base)
|
|
if len(candidates) != 1 || candidates[0].WeightAuto != 92 {
|
|
t.Fatalf("CollectCandidates() = %#v, want one candidate with WeightAuto 92", candidates)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeCoordinatesStatsSnapshotAndAutoWeightWrite(t *testing.T) {
|
|
now := time.Date(2026, time.July, 27, 12, 0, 0, 0, time.UTC)
|
|
registry := newFakeAutoWeightRegistry(1)
|
|
stats := health.NewStatsStore()
|
|
for range 10 {
|
|
stats.Record(1, true, now)
|
|
}
|
|
coordinator := &barrierRuntimeMutationCoordinator{
|
|
entered: make(chan struct{}),
|
|
releaseEntry: make(chan struct{}),
|
|
callbackDone: make(chan struct{}),
|
|
releaseExit: make(chan struct{}),
|
|
}
|
|
runtime := &Runtime{registry: registry, stats: stats, mutations: coordinator}
|
|
|
|
done := make(chan struct{})
|
|
go func() {
|
|
runtime.recompute(now)
|
|
close(done)
|
|
}()
|
|
awaitSignal(t, coordinator.entered)
|
|
select {
|
|
case write := <-registry.wrote:
|
|
t.Fatalf("auto-weight write before coordinator callback = %#v", write)
|
|
default:
|
|
}
|
|
stats.Record(1, false, now)
|
|
close(coordinator.releaseEntry)
|
|
awaitSignal(t, coordinator.callbackDone)
|
|
|
|
if got, want := awaitValue(t, registry.wrote), (autoWeightWrite{keyID: 1, weight: 42}); got != want {
|
|
t.Fatalf("auto-weight write = %#v, want %#v", got, want)
|
|
}
|
|
select {
|
|
case <-done:
|
|
t.Fatal("recompute returned before coordinator interval was released")
|
|
default:
|
|
}
|
|
close(coordinator.releaseExit)
|
|
awaitSignal(t, done)
|
|
}
|
|
|
|
func TestRuntimeCoordinatesAutoWeightWithValidationRecovery(t *testing.T) {
|
|
base := time.Date(2026, time.July, 22, 12, 0, 0, 0, time.UTC)
|
|
stats := health.NewStatsStore()
|
|
for range 10 {
|
|
stats.Record(1, true, base)
|
|
}
|
|
registry := newInterleavingRegistry()
|
|
probePassed := make(chan struct{})
|
|
probes := &validationProbeRecorder{probe: func(context.Context, protocol.Protocol, string, string) error {
|
|
close(probePassed)
|
|
return nil
|
|
}}
|
|
mutations := health.NewMutationCoordinator()
|
|
runtime := &Runtime{registry: registry, stats: stats, mutations: mutations}
|
|
worker := &validationWorker{
|
|
snapshots: &validationSnapshotRecorder{snapshot: validationSnapshot(map[uint]state.GroupView{
|
|
1: validationGroup([]protocol.Protocol{protocol.OpenAI}, "model", nil),
|
|
})},
|
|
registry: registry,
|
|
stats: stats,
|
|
mutations: mutations,
|
|
decryptor: validationDecryptor{},
|
|
dialects: dialect.Set{protocol.OpenAI: &validationTestDialect{protocol: protocol.OpenAI, probes: probes}},
|
|
}
|
|
|
|
autoDone := make(chan struct{})
|
|
go func() {
|
|
runtime.recompute(base)
|
|
close(autoDone)
|
|
}()
|
|
awaitSignal(t, registry.autoBlocked)
|
|
|
|
validationDone := make(chan struct{})
|
|
go func() {
|
|
worker.Validate(context.Background())
|
|
close(validationDone)
|
|
}()
|
|
awaitSignal(t, probePassed)
|
|
close(registry.releaseAuto)
|
|
awaitSignal(t, autoDone)
|
|
awaitSignal(t, validationDone)
|
|
|
|
if got := registry.weight(); got != state.DefaultWeight {
|
|
t.Fatalf("final weight = %d, want recovered default %d", got, state.DefaultWeight)
|
|
}
|
|
if got := registry.recoveryCount(); got != 1 {
|
|
t.Fatalf("conditional recoveries = %d, want 1", got)
|
|
}
|
|
}
|
|
|
|
func TestRuntimeWaitsForValidationTick(t *testing.T) {
|
|
registry := newFakeAutoWeightRegistry(1)
|
|
validator := newFakeValidationSweep(false)
|
|
runtime, _, validationTicker, created := newRuntimeHarness(registry, health.NewStatsStore(), validator, time.Now)
|
|
|
|
cancel, done := startRuntime(t, runtime)
|
|
awaitTickers(t, created)
|
|
select {
|
|
case <-validator.started:
|
|
t.Fatal("validation ran before its first tick")
|
|
default:
|
|
}
|
|
validationTicker.ticks <- time.Now()
|
|
awaitSignal(t, validator.started)
|
|
awaitSignal(t, validator.returned)
|
|
stopRuntime(t, cancel, done)
|
|
}
|
|
|
|
func TestRuntimeKeepsRecomputingWhileValidationIsBlocked(t *testing.T) {
|
|
registry := newFakeAutoWeightRegistry(1)
|
|
validator := newFakeValidationSweep(true)
|
|
runtime, autoTicker, validationTicker, created := newRuntimeHarness(registry, health.NewStatsStore(), validator, time.Now)
|
|
|
|
cancel, done := startRuntime(t, runtime)
|
|
awaitTickers(t, created)
|
|
validationTicker.ticks <- time.Now()
|
|
awaitSignal(t, validator.started)
|
|
autoTicker.ticks <- time.Now()
|
|
if got, want := awaitValue(t, registry.wrote), (autoWeightWrite{keyID: 1, weight: state.DefaultWeight}); got != want {
|
|
t.Fatalf("auto-weight write while validation is blocked = %#v, want %#v", got, want)
|
|
}
|
|
stopRuntime(t, cancel, done)
|
|
awaitSignal(t, validator.returned)
|
|
}
|
|
|
|
func TestRuntimeCancellationStopsBothTickersAndWaitsForValidation(t *testing.T) {
|
|
registry := newFakeAutoWeightRegistry(1)
|
|
validator := newFakeValidationSweep(true)
|
|
runtime, autoTicker, validationTicker, created := newRuntimeHarness(registry, health.NewStatsStore(), validator, time.Now)
|
|
|
|
cancel, done := startRuntime(t, runtime)
|
|
awaitTickers(t, created)
|
|
validationTicker.ticks <- time.Now()
|
|
awaitSignal(t, validator.started)
|
|
cancel()
|
|
awaitSignal(t, validator.returned)
|
|
awaitSignal(t, done)
|
|
awaitSignal(t, autoTicker.stopped)
|
|
awaitSignal(t, validationTicker.stopped)
|
|
}
|
|
|
|
func TestRuntimeStopsOnContextCancellation(t *testing.T) {
|
|
registry := newFakeAutoWeightRegistry(1)
|
|
runtime, autoTicker, validationTicker, created := newRuntimeHarness(registry, health.NewStatsStore(), newFakeValidationSweep(false), time.Now)
|
|
|
|
cancel, done := startRuntime(t, runtime)
|
|
awaitTickers(t, created)
|
|
stopRuntime(t, cancel, done)
|
|
awaitSignal(t, autoTicker.stopped)
|
|
awaitSignal(t, validationTicker.stopped)
|
|
|
|
autoTicker.ticks <- time.Now()
|
|
if got := registry.snapshotWrites(); len(got) != 0 {
|
|
t.Fatalf("writes after cancellation = %v, want none", got)
|
|
}
|
|
}
|
|
|
|
func newRuntimeHarness(
|
|
registry autoWeightRegistry,
|
|
stats *health.StatsStore,
|
|
validator validationSweep,
|
|
now func() time.Time,
|
|
) (*Runtime, *fakeRuntimeTicker, *fakeRuntimeTicker, <-chan time.Duration) {
|
|
autoTicker := newFakeRuntimeTicker()
|
|
validationTicker := newFakeRuntimeTicker()
|
|
created := make(chan time.Duration, 2)
|
|
runtime := newTestRuntime(registry, stats, validator, autoTicker, validationTicker, created, now)
|
|
return runtime, autoTicker, validationTicker, created
|
|
}
|
|
|
|
func newTestRuntime(
|
|
registry autoWeightRegistry,
|
|
stats *health.StatsStore,
|
|
validator validationSweep,
|
|
autoTicker *fakeRuntimeTicker,
|
|
validationTicker *fakeRuntimeTicker,
|
|
created chan<- time.Duration,
|
|
now func() time.Time,
|
|
) *Runtime {
|
|
return &Runtime{
|
|
registry: registry,
|
|
stats: stats,
|
|
mutations: health.NewMutationCoordinator(),
|
|
validator: validator,
|
|
autoWeightInterval: 30 * time.Second,
|
|
validationInterval: 30 * time.Minute,
|
|
validationJitter: func() time.Duration { return 2 * time.Minute },
|
|
now: now,
|
|
newTicker: func(interval time.Duration) runtimeTicker {
|
|
created <- interval
|
|
switch interval {
|
|
case 30 * time.Second:
|
|
return autoTicker
|
|
case 32 * time.Minute:
|
|
return validationTicker
|
|
default:
|
|
testingPanic("unexpected ticker interval", interval)
|
|
return nil
|
|
}
|
|
},
|
|
}
|
|
}
|
|
|
|
func testingPanic(message string, value time.Duration) {
|
|
panic(message + ": " + value.String())
|
|
}
|
|
|
|
func awaitTickers(t *testing.T, created <-chan time.Duration) {
|
|
t.Helper()
|
|
if interval := awaitValue(t, created); interval != 30*time.Second {
|
|
t.Fatalf("auto-weight ticker interval = %v, want 30s", interval)
|
|
}
|
|
if interval := awaitValue(t, created); interval != 32*time.Minute {
|
|
t.Fatalf("validation ticker interval = %v, want 32m", interval)
|
|
}
|
|
}
|
|
|
|
func startRuntime(t *testing.T, runtime *Runtime) (context.CancelFunc, <-chan struct{}) {
|
|
t.Helper()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan struct{})
|
|
go func() {
|
|
defer close(done)
|
|
runtime.Run(ctx)
|
|
}()
|
|
t.Cleanup(func() {
|
|
cancel()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(time.Second):
|
|
t.Errorf("Runtime.Run did not return during cleanup")
|
|
}
|
|
})
|
|
return cancel, done
|
|
}
|
|
|
|
func stopRuntime(t *testing.T, cancel context.CancelFunc, done <-chan struct{}) {
|
|
t.Helper()
|
|
cancel()
|
|
awaitSignal(t, done)
|
|
}
|
|
|
|
func awaitValue[T any](t *testing.T, channel <-chan T) T {
|
|
t.Helper()
|
|
select {
|
|
case value := <-channel:
|
|
return value
|
|
case <-time.After(time.Second):
|
|
t.Fatal("timed out waiting for channel value")
|
|
var zero T
|
|
return zero
|
|
}
|
|
}
|
|
|
|
func awaitSignal(t *testing.T, channel <-chan struct{}) {
|
|
t.Helper()
|
|
select {
|
|
case <-channel:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("timed out waiting for signal")
|
|
}
|
|
}
|
|
|
|
type interleavingRegistry struct {
|
|
mu sync.Mutex
|
|
autoBlocked chan struct{}
|
|
releaseAuto chan struct{}
|
|
blockAutoOnce sync.Once
|
|
currentWeight int
|
|
recoveries int
|
|
}
|
|
|
|
func newInterleavingRegistry() *interleavingRegistry {
|
|
return &interleavingRegistry{
|
|
autoBlocked: make(chan struct{}),
|
|
releaseAuto: make(chan struct{}),
|
|
currentWeight: 17,
|
|
}
|
|
}
|
|
|
|
func (*interleavingRegistry) ActiveKeyIDs() []uint {
|
|
return []uint{1}
|
|
}
|
|
|
|
func (registry *interleavingRegistry) SetAutoWeight(_ uint, weight int) bool {
|
|
if weight == 92 {
|
|
registry.blockAutoOnce.Do(func() { close(registry.autoBlocked) })
|
|
<-registry.releaseAuto
|
|
}
|
|
registry.mu.Lock()
|
|
registry.currentWeight = weight
|
|
registry.mu.Unlock()
|
|
return true
|
|
}
|
|
|
|
func (*interleavingRegistry) BlacklistedKeys() []state.KeyRef {
|
|
return []state.KeyRef{{ID: 1, GroupID: 1, EncryptedValue: "cipher-one"}}
|
|
}
|
|
|
|
func (registry *interleavingRegistry) RecoverIfMatch(_ state.KeyRef, weight int) bool {
|
|
registry.mu.Lock()
|
|
registry.currentWeight = weight
|
|
registry.recoveries++
|
|
registry.mu.Unlock()
|
|
return true
|
|
}
|
|
|
|
func (registry *interleavingRegistry) weight() int {
|
|
registry.mu.Lock()
|
|
defer registry.mu.Unlock()
|
|
return registry.currentWeight
|
|
}
|
|
|
|
func (registry *interleavingRegistry) recoveryCount() int {
|
|
registry.mu.Lock()
|
|
defer registry.mu.Unlock()
|
|
return registry.recoveries
|
|
}
|