Files
Piik/internal/server/route/controller_test.go
T
AAA 1332473513 feat: rename product and runtime to Piik
Rename the active product, Go module, binaries, configuration, protocols and documentation to Piik while retaining the TV mascot and existing media/room behavior.

Use one current private contract without old-name aliases. Preserve SQLite authority and copy compatible local settings once; other settings start fresh. Git-message cleanup and the infrastructure name cutover are coordinated with this phase, with original history and deployment data archived externally.

Validated locally with Web and Go checks, Windows capture compilation, matched Client/Server builds, packaged Client control handshake and history-rewrite integrity checks. Skip duplicate hosted CI for this owner-authorized local release.

[skip ci]
2026-09-10 11:10:12 +08:00

2896 lines
135 KiB
Go

package route
// Oracle port of tests/room-route-controller.test.ts. Each test keeps the TS
// title and line as a comment; every nowMs is the explicit literal the TS
// used, so nothing here needs a clock.
import (
"encoding/json"
"fmt"
"regexp"
"slices"
"strings"
"testing"
"github.com/TNTcraftHIM/Piik/internal/server/protocol"
)
// TS 359: keeps one diagnostic label until a departed relay is finally pruned
func TestKeepsOneDiagnosticLabelUntilDepartedRelayIsPruned(t *testing.T) {
routes := newController(2, Options{})
addViewer(routes, A, 1, nil)
addViewer(routes, B, 0, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(A, "b_from_a"))
label := routes.DiagnosticParticipantLabel(A)
if !regexp.MustCompile(`^viewer-\d+$`).MatchString(label) {
t.Fatalf("label %q is not viewer-<n>", label)
}
routes.UpsertParticipant(viewerInput(A, "a_replacement_session", 1), nil)
eq(t, routes.DiagnosticParticipantLabel(A), label)
eq(t, routes.DisconnectSession(A, "a_replacement_session"), true)
eq(t, routes.DiagnosticParticipantLabel(A), label)
eq(t, routes.ConfirmDeparture(A, ms(0)), true)
eq(t, routes.DiagnosticParticipantLabel(A), label)
repair := must(t, routes.Reconcile(1).Operation)
eq(t, repair.ChildPeerID, B)
eq(t, repair.Reason, DemandParentDeparted)
commitCurrent(t, routes, 2, "b_from_host")
pruned := routes.Reconcile(4)
if !slices.Contains(pruned.RemovedPeerIDs, A) {
t.Fatalf("removedPeerIds %v does not contain A", pruned.RemovedPeerIDs)
}
eq(t, routes.DiagnosticParticipantLabel(A), "viewer-unknown")
addViewer(routes, C, 0, nil)
if routes.DiagnosticParticipantLabel(C) == label {
t.Fatalf("C reused label %q", label)
}
}
// TS 394: clears an unknown sender run without rearming its healthy baseline
func TestClearsUnknownSenderRunWithoutRearmingHealthyBaseline(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
for _, seed := range [][2]string{{A, "a_from_host"}, {B, "b_from_host"}} {
eq(t, observeSenderState(routes, senderState{
childPeerID: seed[0], connectionID: seed[1], state: SenderQualityHealthy, acceptedAtMs: 0,
}).Accepted, true)
}
eq(t, observeSenderState(routes, senderState{
childPeerID: A, connectionID: "a_from_host", state: SenderQualityDegraded, acceptedAtMs: 1,
}).Accepted, true)
eq(t, routes.ObserveSenderQualityEvidence(unknownSample(HOST, A, routes.Snapshot().Revision, "a_from_host", 2), nil).Accepted, true)
for _, acceptedAtMs := range []int64{3, 4, 5} {
eq(t, observeSenderState(routes, senderState{
childPeerID: A, connectionID: "a_from_host", state: SenderQualityDegraded, acceptedAtMs: acceptedAtMs,
}).Accepted, true)
}
operation := must(t, routes.Reconcile(6).Operation)
eq(t, operation.ChildPeerID, A)
eq(t, operation.Reason, DemandQualityConvergence)
}
// TS 448: consumes a quality candidate rejected at preparation instead of rebuilding it
func TestConsumesQualityCandidateRejectedAtPreparation(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
observePersistentDegraded(t, routes, A, "a_from_host", 0)
first := must(t, routes.Reconcile(10).Operation)
eq(t, first.ChildPeerID, A)
eq(t, first.Reason, DemandQualityConvergence)
rejected := first.Candidates[first.Cursor].Tuple
operation := first
for round := 0; operation != nil && round < 4; round++ {
if round > 0 && slices.Contains(tuplesFrom(operation, operation.Cursor), rejected) {
t.Fatalf("round %d rebuilt the rejected tuple %+v", round, rejected)
}
eq(t, routes.SkipCurrentCandidate(cursorGuard(operation), int64(11+round), RejectionStale).Accepted, true)
operation = routes.Reconcile(int64(12 + round)).Operation
}
noOperation(t, operation)
}
// TS 480: lets a newly committed availability edge report persistent degradation before healthy
func TestNewlyCommittedEdgeReportsPersistentDegradationBeforeHealthy(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, B, 2, nil)
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
addViewer(routes, A, 2, ms(0))
joined := must(t, routes.Reconcile(0).Operation)
eq(t, joined.ChildPeerID, A)
commitCurrent(t, routes, 1, "a_from_host")
for _, acceptedAtMs := range []int64{2, 3, 4} {
eq(t, observeSenderState(routes, senderState{
childPeerID: A, connectionID: "a_from_host", state: SenderQualityDegraded, acceptedAtMs: acceptedAtMs,
}).Accepted, true)
}
operation := must(t, routes.Reconcile(5).Operation)
eq(t, operation.ChildPeerID, A)
eq(t, operation.Reason, DemandQualityConvergence)
}
// TS 506: starts a fresh degradation run when an availability sender identity changes
func TestStartsFreshDegradationRunWhenSenderIdentityChanges(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, B, 2, nil)
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
addViewer(routes, A, 2, ms(0))
routes.Reconcile(0)
commitCurrent(t, routes, 1, "a_from_host")
observe := func(identity string, at int64) {
t.Helper()
eq(t, observeSenderState(routes, senderState{
childPeerID: A, connectionID: "a_from_host", senderIdentity: identity, state: SenderQualityDegraded, acceptedAtMs: at,
}).Accepted, true)
}
observe("old-rtp\x00track", 2)
observe("old-rtp\x00track", 3)
observe("new-rtp\x00track", 4)
noOperation(t, routes.Reconcile(4).Operation)
observe("new-rtp\x00track", 5)
observe("new-rtp\x00track", 6)
operation := must(t, routes.Reconcile(7).Operation)
eq(t, operation.ChildPeerID, A)
eq(t, operation.Reason, DemandQualityConvergence)
}
// TS 566: lets a reset sender begin a new degradation run without a healthy sample
func TestResetSenderBeginsNewDegradationRunWithoutHealthySample(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, B, 2, nil)
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
addViewer(routes, A, 2, ms(0))
routes.Reconcile(0)
commitCurrent(t, routes, 1, "a_from_host")
eq(t, observeSenderState(routes, senderState{
childPeerID: A, connectionID: "a_from_host", state: SenderQualityHealthy, acceptedAtMs: 2,
}).Accepted, true)
routes.ResetSenderQuality(HOST, "host_session", 3)
for _, acceptedAtMs := range []int64{4, 5, 6} {
eq(t, observeSenderState(routes, senderState{
childPeerID: A, connectionID: "a_from_host", state: SenderQualityDegraded, acceptedAtMs: acceptedAtMs,
}).Accepted, true)
}
operation := must(t, routes.Reconcile(7).Operation)
eq(t, operation.ChildPeerID, A)
eq(t, operation.Reason, DemandQualityConvergence)
}
// TS 599: regenerates a persistently degraded edge on the same parent
func TestRegeneratesPersistentlyDegradedEdgeOnSameParent(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
observePersistentDegraded(t, routes, A, "a_from_host", 10)
operation := must(t, routes.Reconcile(14).Operation)
eq(t, len(operation.Candidates), 1)
eq(t, operation.Candidates[0].Tuple, regenerateTuple(HOST))
expectTransition(t, operation.Candidates[0].EndpointTransition, TransitionNone, HOST)
begin := beginOperation(t, routes, BeginInput{NowMs: 15, ConnectionID: "a_regenerated", Reservation: direct()})
edge := edgeOf(t, routes, A)
eq(t, edge.ParentPeerID, HOST)
eq(t, edge.ConnectionID, "a_from_host")
eq(t, edge.Usable, true)
eq(t, edge.PhysicalActive, true)
guard := guardFor(A, begin.Current.Revision, "a_regenerated")
ready := routes.CandidateReady(guard, 16, nil, CandidateProof{RelativeQualityApproved: true})
eq(t, ready.Accepted, true)
eq(t, ready.Committed, false)
eq(t, routes.ObserveSenderQualityEvidence(senderSample(
HOST, A, begin.Current.Revision, "a_regenerated", "a-regenerated-rtp\x00track", SenderQualityHealthy, 17), nil).Committed, true)
edge = edgeOf(t, routes, A)
eq(t, edge.Kind, UpstreamPeer)
eq(t, edge.ParentPeerID, HOST)
eq(t, edge.ConnectionID, "a_regenerated")
eq(t, observeSenderState(routes, senderState{
childPeerID: A, connectionID: "a_regenerated", senderIdentity: "a-regenerated-rtp\x00track",
state: SenderQualityHealthy, acceptedAtMs: 18,
}).Accepted, true)
for _, acceptedAtMs := range []int64{19, 20, 21} {
eq(t, observeSenderState(routes, senderState{
childPeerID: A, connectionID: "a_regenerated", senderIdentity: "a-regenerated-rtp\x00track",
state: SenderQualityDegraded, acceptedAtMs: acceptedAtMs,
}).Accepted, true)
}
eq(t, must(t, routes.Reconcile(22).Operation).Candidates[0].Tuple, regenerateTuple(HOST))
}
// TS 686: keeps same-edge regeneration damped while allowing a different parent
func TestKeepsSameEdgeRegenerationDampedWhileAllowingDifferentParent(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
observePersistentDegraded(t, routes, A, "a_from_host", 10)
routes.Reconcile(14)
begin := beginOperation(t, routes, BeginInput{NowMs: 15, ConnectionID: "a_regenerated", Reservation: direct()})
guard := guardFor(A, begin.Current.Revision, "a_regenerated")
routes.CandidateReady(guard, 16, nil, CandidateProof{RelativeQualityApproved: true})
eq(t, routes.ObserveSenderQualityEvidence(senderSample(
HOST, A, begin.Current.Revision, "a_regenerated", "a-regenerated-rtp\x00track", SenderQualityHealthy, 17), nil).Committed, true)
for _, acceptedAtMs := range []int64{18, 19, 20} {
eq(t, observeSenderState(routes, senderState{
childPeerID: A, connectionID: "a_regenerated", senderIdentity: "a-regenerated-rtp\x00track",
state: SenderQualityDegraded, acceptedAtMs: acceptedAtMs,
}).Accepted, true)
}
noOperation(t, routes.Reconcile(21).Operation)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
eq(t, observeSenderState(routes, senderState{
childPeerID: B, connectionID: "b_from_host", state: SenderQualityHealthy, acceptedAtMs: 22,
}).Accepted, true)
withAlternate := must(t, routes.Reconcile(23).Operation)
eq(t, withAlternate.ChildPeerID, A)
eq(t, withAlternate.Reason, DemandQualityConvergence)
eqSlice(t, tuples(withAlternate), []CandidateTuple{peerTuple(B)})
}
// TS 754: uses the existing one-slot overlap for same-parent regeneration
func TestUsesExistingOneSlotOverlapForSameParentRegeneration(t *testing.T) {
routes := newController(1, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 1, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
observePersistentDegraded(t, routes, A, "a_from_host", 10)
operation := must(t, routes.Reconcile(14).Operation)
expectTransition(t, operation.Candidates[0].EndpointTransition, TransitionOverlap, HOST)
begin := beginOperation(t, routes, BeginInput{NowMs: 15, ConnectionID: "a_regenerated", Reservation: directOverlap("a_overlap")})
guard := guardFor(A, begin.Current.Revision, "a_regenerated")
routes.CandidateReady(guard, 16, nil, CandidateProof{RelativeQualityApproved: true})
committed := routes.ObserveSenderQualityEvidence(senderSample(
HOST, A, begin.Current.Revision, "a_regenerated", "a-regenerated-rtp\x00track", SenderQualityHealthy, 17), nil)
eq(t, committed.Committed, true)
containsLabels(t, committed.Released, "a_overlap")
}
// TS 794: keeps one quality experiment through inconclusive current windows
func TestKeepsOneQualityExperimentThroughInconclusiveWindows(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
observeSenderState(routes, senderState{childPeerID: B, connectionID: "b_from_host", state: SenderQualityHealthy, acceptedAtMs: 9})
observePersistentDegraded(t, routes, A, "a_from_host", 10)
operation := must(t, routes.Reconcile(14).Operation)
eqSlice(t, tuples(operation), []CandidateTuple{peerTuple(B), regenerateTuple(HOST)})
beginCandidate(t, routes, BeginInput{NowMs: 15, ConnectionID: "a_from_b_failed", Reservation: direct()})
currentRevision := routes.Snapshot().Revision
eq(t, routes.ObserveSenderQualityEvidence(unknownSample(HOST, A, currentRevision, "a_from_host", 16), nil).Accepted, true)
pending := must(t, routes.Snapshot().Operation)
eq(t, pending.Reason, DemandQualityConvergence)
eq(t, pending.Cursor, 0)
afterFirst := routes.OperationExpired(must(t, routes.Snapshot().Operation).WakeAtMs)
eq(t, afterFirst.Accepted, true)
pending = must(t, routes.Snapshot().Operation)
eq(t, pending.Reason, DemandQualityConvergence)
eq(t, pending.Cursor, 1)
regeneration := beginOperation(t, routes, BeginInput{
NowMs: 20, ConnectionID: "a_regeneration_failed", Reservation: directOverlap("a_regeneration_overlap"),
})
eq(t, routes.CandidateFailed(guardFor(A, regeneration.Current.Revision, "a_regeneration_failed"), 21).Accepted, true)
noOperation(t, routes.Snapshot().Operation)
for _, acceptedAtMs := range []int64{22, 23, 24} {
eq(t, observeSenderState(routes, senderState{
childPeerID: A, connectionID: "a_from_host", state: SenderQualityDegraded, acceptedAtMs: acceptedAtMs,
}).Accepted, true)
}
noOperation(t, routes.Reconcile(25).Operation)
routes.TouchExternalFacts()
noOperation(t, routes.Reconcile(26).Operation)
addViewer(routes, C, 2, nil)
routes.hydrateEdge(C, peerEdge(B, "c_from_b"))
observeSenderState(routes, senderState{
childPeerID: C, connectionID: "c_from_b", parentPeerID: B, state: SenderQualityHealthy, acceptedAtMs: 27,
})
newOpportunity := must(t, routes.Reconcile(28).Operation)
eq(t, newOpportunity.ChildPeerID, A)
eq(t, newOpportunity.Reason, DemandQualityConvergence)
eqSlice(t, tuples(newOpportunity), []CandidateTuple{peerTuple(C)})
}
// TS 902: does not restart a quality candidate set after its total deadline
func TestDoesNotRestartQualityCandidateSetAfterTotalDeadline(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
observeSenderState(routes, senderState{childPeerID: B, connectionID: "b_from_host", state: SenderQualityHealthy, acceptedAtMs: 9})
observePersistentDegraded(t, routes, A, "a_from_host", 10)
operation := must(t, routes.Reconcile(14).Operation)
beginCandidate(t, routes, BeginInput{NowMs: 15, ConnectionID: "a_from_b_pending", Reservation: direct()})
eq(t, routes.OperationExpired(operation.DeadlineAtMs).Accepted, true)
noOperation(t, routes.Snapshot().Operation)
noOperation(t, routes.Reconcile(operation.DeadlineAtMs+1).Operation)
routes.TouchExternalFacts()
noOperation(t, routes.Reconcile(operation.DeadlineAtMs+2).Operation)
observeSenderState(routes, senderState{
childPeerID: A, connectionID: "a_from_host", state: SenderQualityHealthy, acceptedAtMs: operation.DeadlineAtMs + 3,
})
for _, acceptedAtMs := range []int64{operation.DeadlineAtMs + 4, operation.DeadlineAtMs + 5, operation.DeadlineAtMs + 6} {
observeSenderState(routes, senderState{
childPeerID: A, connectionID: "a_from_host", state: SenderQualityDegraded, acceptedAtMs: acceptedAtMs,
})
}
eq(t, must(t, routes.Reconcile(operation.DeadlineAtMs+7).Operation).Reason, DemandQualityConvergence)
}
// TS 951: keeps isolated limitation windows diagnostic
func TestKeepsIsolatedLimitationWindowsDiagnostic(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
observeSenderState(routes, senderState{childPeerID: B, connectionID: "b_from_host", state: SenderQualityHealthy, acceptedAtMs: 99})
observeSenderState(routes, senderState{childPeerID: A, connectionID: "a_from_host", state: SenderQualityHealthy, acceptedAtMs: 99})
observeSenderState(routes, senderState{childPeerID: A, connectionID: "a_from_host", state: SenderQualityDegraded, acceptedAtMs: 100})
repeated := senderSample(HOST, A, routes.Snapshot().Revision, "a_from_host", "a_from_host-rtp\x00track", SenderQualityDegraded, 101)
repeated.SampleTimestampMs = ms(100) // TS: sampleTimestampMs 100 with acceptedAtMs 101
eq(t, routes.ObserveSenderQualityEvidence(repeated, nil).Accepted, true)
noOperation(t, routes.Reconcile(101).Operation)
for _, acceptedAtMs := range []int64{102} {
observeSenderState(routes, senderState{childPeerID: A, connectionID: "a_from_host", state: SenderQualityDegraded, acceptedAtMs: acceptedAtMs})
noOperation(t, routes.Reconcile(acceptedAtMs).Operation)
}
observeSenderState(routes, senderState{childPeerID: A, connectionID: "a_from_host", state: SenderQualityDegraded, acceptedAtMs: 103})
operation := must(t, routes.Reconcile(104).Operation)
eq(t, operation.ChildPeerID, A)
eq(t, operation.Reason, DemandQualityConvergence)
}
// TS 1010: never treats one degraded Host edge as SFU fanout pressure
func TestNeverTreatsOneDegradedHostEdgeAsSfuFanoutPressure(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true, QualityConvergenceEnabled: true})
addViewer(routes, A, 1, nil)
addViewer(routes, B, 0, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateHostPublication("publication_generation_12345678", res("publication_resource"), "")
routes.hydrateEdge(B, sfuEdge("publication_generation_12345678", "b_from_sfu", "b_subscription"))
observePersistentDegraded(t, routes, A, "a_from_host", 99)
observePersistentSfuHealthy(t, routes, "publication_generation_12345678", routes.Snapshot().Revision, 103)
operation := must(t, routes.Reconcile(104).Operation)
eq(t, operation.Reason, DemandQualityConvergence)
eq(t, len(operation.Candidates), 1)
eq(t, operation.Candidates[0].Tuple, regenerateTuple(HOST))
edge := edgeOf(t, routes, A)
eq(t, edge.Kind, UpstreamPeer)
eq(t, edge.ParentPeerID, HOST)
}
// TS 1052: commits only after the Viewer approves a P2P receive improvement
func TestCommitsOnlyAfterViewerApprovesP2PReceiveImprovement(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
eq(t, observeSenderState(routes, senderState{
childPeerID: B, connectionID: "b_from_host", state: SenderQualityHealthy, acceptedAtMs: 100,
}).Accepted, true)
observePersistentDegraded(t, routes, A, "a_from_host", 99)
operation := must(t, routes.Reconcile(103).Operation)
eq(t, operation.Reason, DemandQualityConvergence)
eq(t, operation.Candidates[0].Tuple, peerTuple(B))
begin := beginCandidate(t, routes, BeginInput{NowMs: 104, ConnectionID: "a_from_b", Reservation: direct()})
current := must(t, begin.Operation).Current
guard := guardFor(A, current.Revision, "a_from_b")
edge := edgeOf(t, routes, A)
eq(t, edge.Kind, UpstreamPeer)
eq(t, edge.ParentPeerID, HOST)
eq(t, edge.ConnectionID, "a_from_host")
ready := routes.CandidateReady(guard, 105, nil, CandidateProof{RelativeQualityApproved: true})
eq(t, ready.Accepted, true)
eq(t, ready.Committed, false)
eq(t, routes.ObserveSenderQualityEvidence(senderSample(
B, A, current.Revision, "a_from_b", "a-from-b-rtp\x00a-track", SenderQualityHealthy, 106), nil).Committed, true)
edge = edgeOf(t, routes, A)
eq(t, edge.Kind, UpstreamPeer)
eq(t, edge.ParentPeerID, B)
eq(t, edge.ConnectionID, "a_from_b")
migratedRevision := routes.Snapshot().Revision
observeMigrated := func(state SenderQualityState, at int64) SenderQualityEvidenceResult {
return routes.ObserveSenderQualityEvidence(senderSample(
B, A, migratedRevision, "a_from_b", "a-from-b-rtp\x00a-track", state, at), nil)
}
for _, acceptedAtMs := range []int64{107, 108, 109} {
eq(t, observeMigrated(SenderQualityDegraded, acceptedAtMs).Accepted, true)
}
noOperation(t, routes.Reconcile(110).Operation)
routes.TouchExternalFacts()
noOperation(t, routes.Reconcile(110).Operation)
eq(t, observeMigrated(SenderQualityHealthy, 111).Accepted, true)
eq(t, observeMigrated(SenderQualityDegraded, 112).Accepted, true)
eq(t, observeMigrated(SenderQualityDegraded, 113).Accepted, true)
eq(t, observeMigrated(SenderQualityDegraded, 114).Accepted, true)
eq(t, must(t, routes.Reconcile(115).Operation).Reason, DemandQualityConvergence)
}
// TS 1147: uses one healthy serial operation to distribute a newly committed Host root
func TestUsesOneHealthySerialOperationToDistributeNewHostRoot(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, B, 2, nil)
addViewer(routes, C, 0, nil)
addViewer(routes, D, 0, nil)
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
routes.hydrateEdge(C, peerEdge(B, "c_from_b"))
routes.hydrateEdge(D, peerEdge(B, "d_from_b"))
addViewer(routes, A, 2, ms(10))
joined := must(t, routes.Reconcile(10).Operation)
eq(t, joined.Candidates[0].Tuple, peerTuple(HOST))
commitCurrent(t, routes, 11, "a_from_host")
convergence := must(t, routes.Reconcile(13).Operation)
eq(t, convergence.Reason, DemandRootConvergence)
eq(t, len(convergence.Candidates), 1)
eq(t, convergence.Candidates[0].Tuple, peerTuple(A))
eq(t, edgeOf(t, routes, convergence.ChildPeerID).ParentPeerID, B)
prepared := beginOperation(t, routes, BeginInput{NowMs: 14, ConnectionID: "root_convergence_candidate", Reservation: direct()})
guard := CandidateGuard{
ChildPeerID: convergence.ChildPeerID,
ChildSessionID: convergence.ChildSessionID,
Revision: prepared.Current.Revision,
ConnectionID: "root_convergence_candidate",
}
ready := routes.CandidateReady(guard, 15, nil, CandidateProof{})
eq(t, ready.Accepted, true)
eq(t, ready.Committed, false)
eq(t, routes.ObserveSenderQualityEvidence(senderSample(
A, convergence.ChildPeerID, prepared.Current.Revision, "root_convergence_candidate", "root-rtp\x00track",
SenderQualityHealthy, 16), nil).Committed, true)
eq(t, edgeOf(t, routes, convergence.ChildPeerID).ParentPeerID, A)
migratedChildPeerID := convergence.ChildPeerID
for _, acceptedAtMs := range []int64{17, 18, 19} {
eq(t, observeSenderState(routes, senderState{
childPeerID: migratedChildPeerID, connectionID: "root_convergence_candidate", parentPeerID: A,
state: SenderQualityDegraded, acceptedAtMs: acceptedAtMs,
}).Accepted, true)
}
noOperation(t, routes.Reconcile(20).Operation)
eq(t, observeSenderState(routes, senderState{
childPeerID: migratedChildPeerID, connectionID: "root_convergence_candidate", parentPeerID: A,
state: SenderQualityHealthy, acceptedAtMs: 21,
}).Accepted, true)
for _, acceptedAtMs := range []int64{22, 23, 24} {
eq(t, observeSenderState(routes, senderState{
childPeerID: migratedChildPeerID, connectionID: "root_convergence_candidate", parentPeerID: A,
state: SenderQualityDegraded, acceptedAtMs: acceptedAtMs,
}).Accepted, true)
}
eq(t, must(t, routes.Reconcile(25).Operation).Reason, DemandQualityConvergence)
}
// TS 1247: advances a silent Peer only when another bounded candidate remains
func TestAdvancesSilentPeerOnlyWhenAnotherBoundedCandidateRemains(t *testing.T) {
routes := newController(2, Options{OperationTimeoutMs: 20_000})
addViewer(routes, B, 2, nil)
addViewer(routes, C, 2, nil)
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
routes.hydrateEdge(C, peerEdge(HOST, "c_from_host"))
addViewer(routes, A, 0, ms(0))
operation := must(t, routes.Reconcile(0).Operation)
beginCandidate(t, routes, BeginInput{NowMs: 100, ConnectionID: "first_silent_candidate", Reservation: direct()})
eq(t, must(t, routes.Snapshot().Operation).WakeAtMs, 5_100)
eq(t, routes.OperationExpired(5_100).Accepted, true)
beginCandidate(t, routes, BeginInput{NowMs: 5_200, ConnectionID: "second_silent_candidate", Reservation: direct()})
eq(t, must(t, routes.Snapshot().Operation).WakeAtMs, operation.DeadlineAtMs)
eq(t, must(t, routes.Snapshot().Operation).DeadlineAtMs, operation.DeadlineAtMs)
eq(t, routes.OperationExpired(10_200).Accepted, false)
eq(t, must(t, routes.Snapshot().Operation).Current.ConnectionID, "second_silent_candidate")
}
// TS 1281: retains a peer-only direct attempt through the total operation deadline
func TestRetainsPeerOnlyDirectAttemptThroughTotalDeadline(t *testing.T) {
routes := newController(1, Options{OperationTimeoutMs: 20_000})
addViewer(routes, A, 0, ms(0))
operation := must(t, routes.Reconcile(0).Operation)
beginCandidate(t, routes, BeginInput{NowMs: 100, ConnectionID: "only_direct_candidate", Reservation: direct()})
eq(t, must(t, routes.Snapshot().Operation).WakeAtMs, operation.DeadlineAtMs)
eq(t, routes.OperationExpired(5_100).Accepted, false)
eq(t, must(t, routes.Snapshot().Operation).Current.ConnectionID, "only_direct_candidate")
expired := routes.OperationExpired(operation.DeadlineAtMs)
eq(t, expired.Accepted, true)
eqSlice(t, expired.FailedPeerIDs, []string{A})
}
// TS 1305: keeps only fresh exact-edge quality shadow aggregates
func TestKeepsOnlyFreshExactEdgeQualityShadowAggregates(t *testing.T) {
routes := newController(2, Options{})
addViewer(routes, A, 1, ms(0))
routes.Reconcile(10)
commitCurrent(t, routes, 20, "a_quality_connection")
active := routes.Snapshot()
eq(t, routes.ObserveQualityEvidence(qualityWindow(
A, active.Revision, "a_quality_connection", peerUpstream(HOST), 1, 1_000, qualityMetrics())), QualityEvidenceObserved)
eq(t, routes.Snapshot().Revision, active.Revision)
eq(t, routes.Snapshot().FactVersion, active.FactVersion)
addViewer(routes, B, 0, ms(1_100))
routes.Reconcile(1_200)
commitCurrent(t, routes, 1_210, "b_quality_connection")
quality := routes.RouteDiagnosticSnapshot(1_300).Children[0].Quality
if quality == nil {
t.Fatal("expected quality aggregates")
}
eq(t, *quality, protocol.RouteDiagnosticQuality{
EligibleWindows: 1,
EligibleDurationMs: 2_000,
FreezeWindows: 1,
FreezeCount: 1,
FreezeDurationMs: 250,
PauseCount: 0,
PauseDurationMs: 0,
})
if routes.RouteDiagnosticSnapshot(5_999).Children[0].Quality == nil {
t.Fatal("quality expired before 6_000")
}
if routes.RouteDiagnosticSnapshot(6_000).Children[0].Quality != nil {
t.Fatal("quality did not expire at 6_000")
}
nextRevision := routes.Snapshot().Revision
nullDecoded := qualityMetrics()
nullDecoded.FramesDecodedDelta = nil
eq(t, routes.ObserveQualityEvidence(qualityWindow(
A, nextRevision, "a_quality_connection", peerUpstream(HOST), 2, 7_000, nullDecoded)), QualityEvidenceAccepted)
if routes.RouteDiagnosticSnapshot(7_100).Children[0].Quality != nil {
t.Fatal("expected no quality after an accepted-only window")
}
eq(t, routes.ObserveQualityEvidence(qualityWindow(
A, nextRevision, "a_quality_connection", peerUpstream(HOST), 2, 7_100, qualityMetrics())), QualityEvidenceObserved)
eq(t, routes.ObserveQualityEvidence(qualityWindow(
A, nextRevision, "a_quality_connection", peerUpstream(HOST), 1, 7_200, qualityMetrics())), QualityEvidenceRejected)
routes.SetPaused(true, ms(7_300))
if routes.RouteDiagnosticSnapshot(7_301).Children[0].Quality != nil {
t.Fatal("expected no quality while paused")
}
}
// TS 1390: waits for a fresh healthy candidate sender after relative proof
func TestWaitsForFreshHealthyCandidateSenderAfterRelativeProof(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
eq(t, observeSenderState(routes, senderState{
childPeerID: B, connectionID: "b_from_host", state: SenderQualityHealthy, acceptedAtMs: 100,
}).Accepted, true)
observePersistentDegraded(t, routes, A, "a_from_host", 99)
routes.Reconcile(103)
begin := beginCandidate(t, routes, BeginInput{NowMs: 104, ConnectionID: "a_from_b", Reservation: direct()})
current := must(t, begin.Operation).Current
eq(t, routes.CandidateTransportConnected(guardFor(A, current.Revision, "a_from_b"), 105).Accepted, true)
eq(t, routes.ObserveSenderQualityEvidence(senderSample(
B, A, current.Revision, "a_from_b", "a-from-b-rtp\x00track", SenderQualityDegraded, 105), nil).Accepted, true)
eq(t, routes.CandidateReady(guardFor(A, current.Revision, "a_from_b"), 106, nil, CandidateProof{RelativeQualityApproved: true}).Committed, false)
eq(t, routes.ObserveSenderQualityEvidence(senderSample(
B, A, current.Revision, "a_from_b", "a-from-b-rtp\x00track", SenderQualityHealthy, 107), nil).Committed, true)
}
// TS 1464: does not commit a relative proof after its freshness window expires
func TestDoesNotCommitRelativeProofAfterFreshnessWindowExpires(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
observePersistentDegraded(t, routes, A, "a_from_host", 99)
routes.Reconcile(103)
begin := beginCandidate(t, routes, BeginInput{NowMs: 104, ConnectionID: "a_from_b_stale_proof", Reservation: direct()})
current := must(t, begin.Operation).Current
guard := guardFor(A, current.Revision, "a_from_b_stale_proof")
eq(t, routes.CandidateTransportConnected(guard, 105).Accepted, true)
eq(t, routes.CandidateReady(guard, 105, nil, CandidateProof{RelativeQualityApproved: true}).Committed, false)
eq(t, routes.CandidateReady(guard, 109, nil, CandidateProof{RelativeQualityApproved: true}).Committed, false)
for _, acceptedAtMs := range []int64{5_105, 5_106, 5_107} {
eq(t, observeSenderState(routes, senderState{
childPeerID: A, connectionID: "a_from_host", state: SenderQualityDegraded, acceptedAtMs: acceptedAtMs,
}).Accepted, true)
}
settled := routes.ObserveSenderQualityEvidence(senderSample(
B, A, current.Revision, guard.ConnectionID, "stale-proof-rtp\x00track", SenderQualityHealthy, 5_111), nil)
eq(t, settled.Accepted, true)
eq(t, settled.Committed, false)
operation := must(t, routes.Snapshot().Operation)
eq(t, operation.Reason, DemandQualityConvergence)
eq(t, operation.Cursor, 1)
edge := edgeOf(t, routes, A)
eq(t, edge.ParentPeerID, HOST)
eq(t, edge.ConnectionID, "a_from_host")
}
// TS 1528: abandons a pending quality move when the incumbent recovers
func TestAbandonsPendingQualityMoveWhenIncumbentRecovers(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
observePersistentDegraded(t, routes, A, "a_from_host", 99)
routes.Reconcile(103)
begin := beginCandidate(t, routes, BeginInput{NowMs: 104, ConnectionID: "a_from_b_incumbent_recovery", Reservation: direct()})
current := must(t, begin.Operation).Current
guard := guardFor(A, current.Revision, "a_from_b_incumbent_recovery")
eq(t, routes.CandidateReady(guard, 105, nil, CandidateProof{RelativeQualityApproved: true}).Committed, false)
eq(t, observeSenderState(routes, senderState{
childPeerID: A, connectionID: "a_from_host", state: SenderQualityHealthy, acceptedAtMs: 106,
}).Accepted, true)
noOperation(t, routes.Snapshot().Operation)
eq(t, routes.ObserveSenderQualityEvidence(senderSample(
B, A, current.Revision, guard.ConnectionID, "late-healthy-rtp\x00track", SenderQualityHealthy, 107), nil).Accepted, false)
edge := edgeOf(t, routes, A)
eq(t, edge.ParentPeerID, HOST)
eq(t, edge.ConnectionID, "a_from_host")
}
// TS 2086-2095: when candidateReady abandons a quality move because the
// incumbent is no longer degraded, activeRevision is the revision that was
// active before the abort (the TS literal reads this.revision before
// abortOperation advances it), even though the abort itself advances it.
func TestReportsPreAbortRevisionWhenCandidateReadyAbandonsQualityMove(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
observePersistentDegraded(t, routes, A, "a_from_host", 99)
routes.Reconcile(103)
begin := beginCandidate(t, routes, BeginInput{NowMs: 104, ConnectionID: "a_from_b", Reservation: direct()})
current := must(t, begin.Operation).Current
before := routes.Snapshot().Revision
eq(t, routes.ObserveSenderQualityEvidence(unknownSample(HOST, A, before, "a_from_host", 105), nil).Accepted, true)
settled := routes.CandidateReady(guardFor(A, current.Revision, "a_from_b"), 106, nil, CandidateProof{})
eq(t, settled.Accepted, true)
eq(t, settled.Committed, false)
noOperation(t, routes.Snapshot().Operation)
eq(t, settled.ActiveRevision, before)
if routes.Snapshot().Revision <= before {
t.Fatalf("expected the abort to advance the revision past %d, got %d", before, routes.Snapshot().Revision)
}
}
// TS 1582: persists relative proof while candidate sender evidence is unknown
func TestPersistsRelativeProofWhileCandidateSenderEvidenceIsUnknown(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
observeSenderState(routes, senderState{childPeerID: B, connectionID: "b_from_host", state: SenderQualityHealthy, acceptedAtMs: 99})
observePersistentDegraded(t, routes, A, "a_from_host", 99)
routes.Reconcile(103)
begin := beginCandidate(t, routes, BeginInput{NowMs: 104, ConnectionID: "a_from_b", Reservation: direct()})
current := must(t, begin.Operation).Current
eq(t, routes.ObserveSenderQualityEvidence(unknownSample(B, A, current.Revision, "a_from_b", 105), nil).Accepted, true)
eq(t, routes.CandidateReady(guardFor(A, current.Revision, "a_from_b"), 106, nil, CandidateProof{RelativeQualityApproved: true}).Committed, false)
eq(t, routes.ObserveSenderQualityEvidence(senderSample(
B, A, current.Revision, "a_from_b", "a-from-b-rtp\x00track", SenderQualityHealthy, 107), nil).Committed, true)
edge := edgeOf(t, routes, A)
eq(t, edge.Kind, UpstreamPeer)
eq(t, edge.ParentPeerID, B)
eq(t, edge.ConnectionID, "a_from_b")
}
// TS 1648: advances past a persistently limited quality candidate
func TestAdvancesPastPersistentlyLimitedQualityCandidate(t *testing.T) {
routes := newController(3, Options{QualityConvergenceEnabled: true})
for _, seed := range [][2]string{{A, "a_from_host"}, {B, "b_from_host"}, {C, "c_from_host"}} {
addViewer(routes, seed[0], 3, nil)
routes.hydrateEdge(seed[0], peerEdge(HOST, seed[1]))
}
observeSenderState(routes, senderState{childPeerID: B, connectionID: "b_from_host", state: SenderQualityHealthy, acceptedAtMs: 99})
observeSenderState(routes, senderState{childPeerID: C, connectionID: "c_from_host", state: SenderQualityHealthy, acceptedAtMs: 99})
observePersistentDegraded(t, routes, A, "a_from_host", 99)
operation := must(t, routes.Reconcile(103).Operation)
deadlineAtMs := operation.DeadlineAtMs
eq(t, operation.Candidates[0].Tuple.Kind, UpstreamPeer)
firstParent := operation.Candidates[0].Tuple.ParentPeerID
first := beginOperation(t, routes, BeginInput{NowMs: 104, ConnectionID: "first_quality_candidate", Reservation: direct()})
var lastResult SenderQualityEvidenceResult
for _, acceptedAtMs := range []int64{105, 106, 107} {
lastResult = routes.ObserveSenderQualityEvidence(senderSample(
firstParent, A, first.Current.Revision, "first_quality_candidate", "first-quality-rtp\x00track",
SenderQualityDegraded, acceptedAtMs), nil)
eq(t, lastResult.Accepted, true)
}
eq(t, lastResult.Committed, false)
edge := edgeOf(t, routes, A)
eq(t, edge.ParentPeerID, HOST)
eq(t, edge.ConnectionID, "a_from_host")
pending := must(t, routes.Snapshot().Operation)
eq(t, pending.Cursor, 1)
eq(t, pending.DeadlineAtMs, deadlineAtMs)
if pending.Current != nil {
t.Fatal("expected no live attempt")
}
secondParent := pending.Candidates[1].Tuple.ParentPeerID
if secondParent == firstParent {
t.Fatalf("second parent %q repeats the first", secondParent)
}
second := beginOperation(t, routes, BeginInput{NowMs: 108, ConnectionID: "second_quality_candidate", Reservation: direct()})
eq(t, routes.ObserveSenderQualityEvidence(unknownSample(
secondParent, A, second.Current.Revision, "second_quality_candidate", 109), nil).Accepted, true)
eq(t, routes.CandidateReady(guardFor(A, second.Current.Revision, "second_quality_candidate"), 110, nil,
CandidateProof{RelativeQualityApproved: true}).Committed, false)
eq(t, routes.ObserveSenderQualityEvidence(senderSample(
secondParent, A, second.Current.Revision, "second_quality_candidate", "second-quality-rtp\x00track",
SenderQualityHealthy, 111), nil).Committed, true)
edge = edgeOf(t, routes, A)
eq(t, edge.ParentPeerID, secondParent)
eq(t, edge.ConnectionID, "second_quality_candidate")
}
// TS 1768: preempts quality work for a Viewer without media
func TestPreemptsQualityWorkForViewerWithoutMedia(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
observeSenderState(routes, senderState{childPeerID: B, connectionID: "b_from_host", state: SenderQualityHealthy, acceptedAtMs: 100})
observePersistentDegraded(t, routes, A, "a_from_host", 99)
eq(t, must(t, routes.Reconcile(103).Operation).Reason, DemandQualityConvergence)
addViewer(routes, C, 0, ms(104))
join := must(t, routes.Reconcile(105).Operation)
eq(t, join.ChildPeerID, C)
eq(t, join.Reason, DemandJoin)
commitCurrent(t, routes, 106, "c_join_connection")
observeSenderState(routes, senderState{childPeerID: B, connectionID: "b_from_host", state: SenderQualityHealthy, acceptedAtMs: 107})
observePersistentDegraded(t, routes, A, "a_from_host", 107)
eq(t, must(t, routes.Reconcile(111).Operation).Reason, DemandQualityConvergence)
edge := edgeOf(t, routes, A)
eq(t, edge.ParentPeerID, HOST)
eq(t, edge.ConnectionID, "a_from_host")
}
// TS 1806: keeps a quality operation reason immutable until availability preempts it
func TestKeepsQualityOperationReasonImmutableUntilAvailabilityPreempts(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
observeSenderState(routes, senderState{childPeerID: B, connectionID: "b_from_host", state: SenderQualityHealthy, acceptedAtMs: 99})
observePersistentDegraded(t, routes, A, "a_from_host", 99)
eq(t, must(t, routes.Reconcile(103).Operation).Reason, DemandQualityConvergence)
eq(t, routes.InvalidateEdge(edgeGuard(A, "host_session", routes.Snapshot().Revision, "a_from_host"), ms(104)), true)
eq(t, must(t, routes.Snapshot().Operation).Reason, DemandQualityConvergence)
preempted := must(t, routes.Reconcile(105).Operation)
eq(t, preempted.ChildPeerID, A)
eq(t, preempted.Reason, DemandEdgeUnavailable)
}
// TS 1842: keeps a direct convergence reason immutable until availability preempts it
func TestKeepsDirectConvergenceReasonImmutableUntilAvailabilityPreempts(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
addViewer(routes, B, 2, nil)
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
routes.hydrateHostPublication("publication", res("publication_resource"), "")
addViewer(routes, A, 0, ms(0))
acquisition := must(t, routes.Reconcile(0).Operation)
beginCandidate(t, routes, BeginInput{NowMs: 1, ConnectionID: "a_head_start", Reservation: direct()})
routes.OperationExpired(acquisition.WakeAtMs)
sfu := beginOperation(t, routes, BeginInput{NowMs: acquisition.WakeAtMs + 1, ConnectionID: "a_sfu", Reservation: sfuReuse("a_subscription")})
eq(t, routes.CandidateReady(guardFor(A, sfu.Current.Revision, "a_sfu"), acquisition.WakeAtMs+2, nil, CandidateProof{}).Committed, true)
eq(t, must(t, routes.Reconcile(acquisition.WakeAtMs+3).Operation).Reason, DemandDirectConvergence)
eq(t, routes.InvalidateEdge(edgeGuard(A, "", routes.Snapshot().Revision, "a_sfu"), ms(acquisition.WakeAtMs+4)), true)
eq(t, must(t, routes.Snapshot().Operation).Reason, DemandDirectConvergence)
preempted := must(t, routes.Reconcile(acquisition.WakeAtMs+5).Operation)
eq(t, preempted.ChildPeerID, A)
eq(t, preempted.Reason, DemandEdgeUnavailable)
}
// TS 1893: keeps a root convergence reason immutable until availability preempts it
func TestKeepsRootConvergenceReasonImmutableUntilAvailabilityPreempts(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, B, 2, nil)
addViewer(routes, C, 0, nil)
addViewer(routes, D, 0, nil)
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
routes.hydrateEdge(C, peerEdge(B, "c_from_b"))
routes.hydrateEdge(D, peerEdge(B, "d_from_b"))
addViewer(routes, A, 2, ms(10))
routes.Reconcile(10)
commitCurrent(t, routes, 11, "a_from_host")
convergence := must(t, routes.Reconcile(13).Operation)
eq(t, convergence.Reason, DemandRootConvergence)
oldEdge := edgeOf(t, routes, convergence.ChildPeerID)
eq(t, oldEdge.Kind, UpstreamPeer)
eq(t, routes.InvalidateEdge(EdgeGuard{
ChildPeerID: convergence.ChildPeerID,
ChildSessionID: convergence.ChildSessionID,
ParentSessionID: oldEdge.ParentSessionID,
RouteRevision: routes.Snapshot().Revision,
ConnectionID: oldEdge.ConnectionID,
}, ms(14)), true)
eq(t, must(t, routes.Snapshot().Operation).Reason, DemandRootConvergence)
preempted := must(t, routes.Reconcile(15).Operation)
eq(t, preempted.ChildPeerID, convergence.ChildPeerID)
eq(t, preempted.Reason, DemandEdgeUnavailable)
}
// TS 1931: keeps a usable parent eligible before native state becomes clear
func TestKeepsUsableParentEligibleBeforeNativeStateBecomesClear(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
observePersistentDegraded(t, routes, A, "a_from_host", 99)
operation := must(t, routes.Reconcile(103).Operation)
eq(t, operation.ChildPeerID, A)
eq(t, operation.Reason, DemandQualityConvergence)
eqSlice(t, tuples(operation), []CandidateTuple{peerTuple(B), regenerateTuple(HOST)})
}
// TS 1957: waits for a fresh candidate sender after prior evidence expires
func TestWaitsForFreshCandidateSenderAfterPriorEvidenceExpires(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
observeCurrent := func(childPeerID, connectionID string, state SenderQualityState, acceptedAtMs int64) SenderQualityEvidenceResult {
return routes.ObserveSenderQualityEvidence(senderSample(
HOST, childPeerID, routes.Snapshot().Revision, connectionID, connectionID+"-rtp\x00track", state, acceptedAtMs), nil)
}
eq(t, observeCurrent(B, "b_from_host", SenderQualityHealthy, 100).Accepted, true)
eq(t, observeCurrent(A, "a_from_host", SenderQualityHealthy, 99).Accepted, true)
eq(t, observeCurrent(A, "a_from_host", SenderQualityDegraded, 100).Accepted, true)
eq(t, observeCurrent(A, "a_from_host", SenderQualityDegraded, 101).Accepted, true)
eq(t, observeCurrent(A, "a_from_host", SenderQualityDegraded, 102).Accepted, true)
routes.Reconcile(103)
begin := beginCandidate(t, routes, BeginInput{NowMs: 104, ConnectionID: "a_from_b_stale", Reservation: direct()})
current := must(t, begin.Operation).Current
eq(t, routes.CandidateTransportConnected(guardFor(A, current.Revision, "a_from_b_stale"), 105).Accepted, true)
eq(t, routes.ObserveSenderQualityEvidence(senderSample(
B, A, current.Revision, "a_from_b_stale", "a-stale-rtp\x00a-track", SenderQualityHealthy, 104), nil).Accepted, true)
for _, acceptedAtMs := range []int64{5_101, 5_102, 5_103} {
eq(t, observeCurrent(A, "a_from_host", SenderQualityDegraded, acceptedAtMs).Accepted, true)
}
eq(t, routes.CandidateReady(guardFor(A, current.Revision, "a_from_b_stale"), 5_104, nil,
CandidateProof{RelativeQualityApproved: true}).Committed, false)
eq(t, routes.ObserveSenderQualityEvidence(senderSample(
B, A, current.Revision, "a_from_b_stale", "a-stale-rtp\x00a-track", SenderQualityHealthy, 5_105), nil).Committed, true)
edge := edgeOf(t, routes, A)
eq(t, edge.ParentPeerID, B)
eq(t, edge.ConnectionID, "a_from_b_stale")
}
// TS 2063: keeps a usable candidate when its native source category expires
func TestKeepsUsableCandidateWhenNativeSourceCategoryExpires(t *testing.T) {
routes := newController(2, Options{QualityConvergenceEnabled: true})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
observeSenderState(routes, senderState{childPeerID: B, connectionID: "b_from_host", state: SenderQualityHealthy, acceptedAtMs: 0})
observePersistentDegraded(t, routes, A, "a_from_host", 0)
eq(t, must(t, routes.Reconcile(4).Operation).Reason, DemandQualityConvergence)
begin := beginCandidate(t, routes, BeginInput{NowMs: 5_001, ConnectionID: "expired_candidate", Reservation: direct()})
eq(t, begin.Accepted, true)
eq(t, must(t, begin.Operation).Current.ConnectionID, "expired_candidate")
}
// TS 2086: keeps bounded SFU after direct candidates when every Host root is degraded
func TestKeepsBoundedSfuAfterDirectCandidatesWhenEveryHostRootIsDegraded(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true, QualityConvergenceEnabled: true})
addViewer(routes, A, 1, nil)
addViewer(routes, B, 0, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
observePersistentDegraded(t, routes, A, "a_from_host", 99)
firstRegeneration := must(t, routes.Reconcile(103).Operation)
eq(t, firstRegeneration.Candidates[0].Tuple, regenerateTuple(HOST))
firstAttempt := beginOperation(t, routes, BeginInput{
NowMs: 104, ConnectionID: "a_regeneration_failed", Reservation: directOverlap("a_regeneration_overlap"),
})
eq(t, routes.CandidateFailed(guardFor(A, firstAttempt.Current.Revision, "a_regeneration_failed"), 105).Accepted, true)
eq(t, routes.SetEffectiveCapacity(B, sessionOf(B), 1, nil), true)
observePersistentDegraded(t, routes, B, "b_from_host", 104)
operation := must(t, routes.Reconcile(108).Operation)
childPeerID := operation.ChildPeerID
eq(t, childPeerID, A)
eqSlice(t, tuples(operation), []CandidateTuple{peerTuple(B), sfuTuple(PublicationCreate)})
directAttempt := beginOperation(t, routes, BeginInput{NowMs: 109, ConnectionID: "quality_peer_candidate", Reservation: direct()})
failed := routes.CandidateFailed(guardFor(childPeerID, directAttempt.Current.Revision, "quality_peer_candidate"), 110)
eq(t, failed.Accepted, true)
eqSlice(t, failed.FailedPeerIDs, []string{})
begin := beginCandidate(t, routes, BeginInput{
NowMs: 111,
ConnectionID: "quality_sfu_candidate",
Reservation: sfuCreateOverlap("sfu-edge", "sfu-publication", "host-overlap"),
PublicationGeneration: "publication_generation_12345678",
PublicationConnectionID: "publication_connection_12345678",
HostSessionID: "host_session",
})
current := must(t, begin.Operation).Current
guard := guardFor(childPeerID, current.Revision, "quality_sfu_candidate")
ready := routes.CandidateReady(guard, 112, nil, CandidateProof{})
eq(t, ready.Accepted, true)
eq(t, ready.Committed, false)
for _, acceptedAtMs := range []int64{113, 114} {
observed := routes.ObserveSfuPublisherQualityEvidence(publisherSample(
"publication_generation_12345678", current.Revision, SenderQualityHealthy, acceptedAtMs), nil)
eq(t, observed.Accepted, true)
eq(t, observed.Committed, false)
ready = routes.CandidateReady(guard, acceptedAtMs, nil, CandidateProof{})
eq(t, ready.Accepted, true)
eq(t, ready.Committed, false)
}
eq(t, routes.ObserveSfuPublisherQualityEvidence(publisherSample(
"publication_generation_12345678", current.Revision, SenderQualityHealthy, 115), nil).Committed, true)
edge := edgeOf(t, routes, childPeerID)
eq(t, edge.Kind, UpstreamSfu)
eq(t, edge.ConnectionID, "quality_sfu_candidate")
}
// TS 2201: rejects an SFU quality candidate on degraded publisher evidence
func TestRejectsSfuQualityCandidateOnDegradedPublisherEvidence(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true, QualityConvergenceEnabled: true})
addViewer(routes, A, 1, nil)
addViewer(routes, B, 0, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
observePersistentDegraded(t, routes, A, "a_from_host", 99)
firstRegeneration := must(t, routes.Reconcile(103).Operation)
eq(t, firstRegeneration.Candidates[0].Tuple, regenerateTuple(HOST))
firstAttempt := beginOperation(t, routes, BeginInput{
NowMs: 104, ConnectionID: "a_regeneration_failed", Reservation: directOverlap("a_regeneration_overlap"),
})
eq(t, routes.CandidateFailed(guardFor(A, firstAttempt.Current.Revision, "a_regeneration_failed"), 105).Accepted, true)
eq(t, routes.SetEffectiveCapacity(B, sessionOf(B), 1, nil), true)
observePersistentDegraded(t, routes, B, "b_from_host", 104)
operation := must(t, routes.Reconcile(108).Operation)
childPeerID := operation.ChildPeerID
eq(t, childPeerID, A)
peerAttempt := beginOperation(t, routes, BeginInput{NowMs: 109, ConnectionID: "quality_peer_candidate", Reservation: direct()})
eq(t, routes.CandidateFailed(guardFor(childPeerID, peerAttempt.Current.Revision, "quality_peer_candidate"), 110).Accepted, true)
sfuAttempt := beginOperation(t, routes, BeginInput{
NowMs: 111,
ConnectionID: "quality_sfu_degraded",
Reservation: sfuCreateOverlap("sfu-edge", "sfu-publication", "host-overlap"),
PublicationGeneration: "publication_generation_degraded",
PublicationConnectionID: "publication_connection_degraded",
HostSessionID: "host_session",
})
current := sfuAttempt.Current
ready := routes.CandidateReady(guardFor(childPeerID, current.Revision, current.ConnectionID), 112, nil, CandidateProof{})
eq(t, ready.Accepted, true)
eq(t, ready.Committed, false)
degraded := routes.ObserveSfuPublisherQualityEvidence(publisherSample(
"publication_generation_degraded", current.Revision, SenderQualityDegraded, 113), nil)
eq(t, degraded.Accepted, true)
eq(t, degraded.Committed, false)
noOperation(t, routes.Reconcile(114).Operation)
edge := edgeOf(t, routes, childPeerID)
eq(t, edge.Kind, UpstreamPeer)
eq(t, edge.ParentPeerID, HOST)
}
// TS 2297: abandons Host relief when another Host root recovers during the canary
func TestAbandonsHostReliefWhenAnotherHostRootRecoversDuringCanary(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true, QualityConvergenceEnabled: true})
addViewer(routes, A, 1, nil)
addViewer(routes, B, 0, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
observePersistentDegraded(t, routes, A, "a_from_host", 99)
observePersistentDegraded(t, routes, B, "b_from_host", 99)
routes.Reconcile(103)
begin := beginCandidate(t, routes, BeginInput{
NowMs: 104,
ConnectionID: "a_from_sfu",
Reservation: sfuCreateOverlap("sfu-edge", "sfu-publication", "host-overlap"),
PublicationGeneration: "publication_generation_12345678",
PublicationConnectionID: "publication_connection_12345678",
HostSessionID: "host_session",
})
current := must(t, begin.Operation).Current
eq(t, observeSenderState(routes, senderState{
childPeerID: B, connectionID: "b_from_host", state: SenderQualityHealthy, acceptedAtMs: 105,
}).Accepted, true)
eq(t, routes.CandidateReady(guardFor(A, current.Revision, "a_from_sfu"), 106, nil, CandidateProof{}).Committed, false)
noOperation(t, routes.Snapshot().Operation)
edge := edgeOf(t, routes, A)
eq(t, edge.Kind, UpstreamPeer)
eq(t, edge.ParentPeerID, HOST)
eq(t, edge.ConnectionID, "a_from_host")
}
// TS 2350: uses SFU reuse only for multi-edge Host relief with healthy ingress
func TestUsesSfuReuseOnlyForMultiEdgeHostReliefWithHealthyIngress(t *testing.T) {
routes := newController(3, Options{SfuEnabled: true, QualityConvergenceEnabled: true})
addViewer(routes, A, 0, nil)
addViewer(routes, B, 0, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
routes.hydrateHostPublication("publication_generation_12345678", res("publication_resource"), "")
revision := routes.Snapshot().Revision
observePersistentDegraded(t, routes, A, "a_from_host", 99)
observePersistentDegraded(t, routes, B, "b_from_host", 99)
noOperation(t, routes.Reconcile(103).Operation)
observePersistentSfuHealthy(t, routes, "publication_generation_12345678", revision, 104)
eq(t, must(t, routes.Reconcile(105).Operation).Candidates[0].Tuple, sfuTuple(PublicationReuse))
begin := beginCandidate(t, routes, BeginInput{NowMs: 106, ConnectionID: "a_from_sfu_reuse", Reservation: sfuReuse("a_subscription")})
current := must(t, begin.Operation).Current
unknown := publisherSample("publication_generation_12345678", revision, SenderQualityUnknown, 107)
unknown.SampleTimestampMs = nil
eq(t, routes.ObserveSfuPublisherQualityEvidence(unknown, nil).Accepted, true)
eq(t, routes.CandidateReady(guardFor(A, current.Revision, "a_from_sfu_reuse"), 108, nil, CandidateProof{}).Committed, false)
for _, acceptedAtMs := range []int64{2_100, 4_100, 6_100} {
observeSenderState(routes, senderState{childPeerID: A, connectionID: "a_from_host", state: SenderQualityDegraded, acceptedAtMs: acceptedAtMs})
observeSenderState(routes, senderState{childPeerID: B, connectionID: "b_from_host", state: SenderQualityDegraded, acceptedAtMs: acceptedAtMs})
settled := routes.ObserveSfuPublisherQualityEvidence(publisherSample(
"publication_generation_12345678", revision, SenderQualityHealthy, acceptedAtMs), nil)
eq(t, settled.Committed, acceptedAtMs == 6_100)
}
edge := edgeOf(t, routes, A)
eq(t, edge.Kind, UpstreamSfu)
eq(t, edge.ConnectionID, "a_from_sfu_reuse")
}
// TS 2435: accepts an SFU-fed Peer parent from decoded progress without freeze stats
func TestAcceptsSfuFedPeerParentFromDecodedProgressWithoutFreezeStats(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true, QualityConvergenceEnabled: true})
addViewer(routes, A, 1, nil)
addViewer(routes, B, 2, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateHostPublication("publication_generation_12345678", res("publication_resource"), "")
routes.hydrateEdge(B, sfuEdge("publication_generation_12345678", "b_from_sfu", "b_subscription"))
revision := routes.Snapshot().Revision
observePersistentSfuHealthy(t, routes, "publication_generation_12345678", revision, 100)
metrics := qualityMetrics()
metrics.FreezeCountDelta = nil
metrics.FreezeDurationMsDelta = nil
metrics.PauseCountDelta = nil
metrics.PauseDurationMsDelta = nil
eq(t, routes.ObserveQualityEvidence(qualityWindow(B, revision, "b_from_sfu", sfuUpstream(), 1, 100, metrics)), QualityEvidenceAccepted)
observePersistentDegraded(t, routes, A, "a_from_host", 99)
eq(t, must(t, routes.Reconcile(103).Operation).Candidates[0].Tuple, peerTuple(B))
}
// TS 2489: clears quality shadow for a relay subtree when its source changes
func TestClearsQualityShadowForRelaySubtreeWhenSourceChanges(t *testing.T) {
routes := newController(2, Options{})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
addViewer(routes, C, 0, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(A, "b_from_a"))
routes.hydrateEdge(C, peerEdge(B, "c_from_b"))
revision := routes.Snapshot().Revision
relays := [][3]string{{B, A, "b_from_a"}, {C, B, "c_from_b"}}
for _, relay := range relays {
eq(t, routes.ObserveQualityEvidence(qualityWindow(
relay[0], revision, relay[2], peerUpstream(relay[1]), 1, 1_000, qualityMetrics())), QualityEvidenceObserved)
}
withQuality := func(nowMs int64) int {
count := 0
for _, child := range routes.RouteDiagnosticSnapshot(nowMs).Children {
if child.Quality != nil {
count++
}
}
return count
}
eq(t, withQuality(1_001), 2)
eq(t, routes.AdoptDirectConnection(AdoptDirectConnectionInput{
EdgeGuard: edgeGuard(A, "host_session", revision, "a_from_host"),
NewConnectionID: "a_from_host_recovered",
}, 1_002).Accepted, true)
eq(t, withQuality(1_002), 0)
for _, step := range []struct {
at int64
expected RouteQualityEvidenceResult
}{{1_100, QualityEvidenceAccepted}, {3_100, QualityEvidenceObserved}} {
for _, relay := range relays {
eq(t, routes.ObserveQualityEvidence(qualityWindow(
relay[0], revision, relay[2], peerUpstream(relay[1]), 1, step.at, qualityMetrics())), step.expected)
}
}
eq(t, withQuality(3_101), 2)
}
func TestActiveRebuildRetiresBootstrapReservationWithoutLosingWaitingDemand(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
for _, child := range []string{B, C} {
addViewer(routes, child, 0, nil)
routes.hydrateEdge(child, peerEdge(HOST, child+"_from_host"))
}
addViewer(routes, A, 0, ms(0))
bootstrap := must(t, routes.Reconcile(0).Operation)
eq(t, bootstrap.Reason, DemandSfuBootstrap)
retained := edgeOf(t, routes, bootstrap.ChildPeerID)
attempt := beginOperation(t, routes, BeginInput{
NowMs: 1, ConnectionID: "carrier_sfu", PublicationGeneration: "publication",
PublicationConnectionID: "publication_connection",
Reservation: sfuCreateOverlap("subscription", "publication", "overlap"),
})
replaced := routes.AdoptDirectConnection(AdoptDirectConnectionInput{
EdgeGuard: edgeGuard(bootstrap.ChildPeerID, "host_session", routes.Revision(), retained.ConnectionID),
NewConnectionID: "recovered_connection",
}, 2)
eq(t, replaced.Accepted, true)
eqLabels(t, replaced.Released, "subscription", "publication", "overlap")
noOperation(t, routes.Snapshot().Operation)
eq(t, edgeOf(t, routes, bootstrap.ChildPeerID).ConnectionID, "recovered_connection")
eq(t, routes.CandidateReady(guardFor(bootstrap.ChildPeerID, attempt.Current.Revision, "carrier_sfu"), 3, nil, CandidateProof{}).Accepted, false)
next := must(t, routes.Reconcile(4).Operation)
eq(t, next.Reason, DemandSfuBootstrap)
eq(t, next.DemandPeerID, A)
}
// TS 2569: retains one latest timing sample for a 20-Viewer burst
func TestRetainsOneLatestTimingSampleForTwentyViewerBurst(t *testing.T) {
routes := newController(2, Options{})
viewerPeerIDs := make([]string, 20)
for index := range viewerPeerIDs {
viewerPeerIDs[index] = fmt.Sprintf("burst_%02d_12345678", index)
}
for _, peerID := range viewerPeerIDs {
addViewer(routes, peerID, 2, ms(0))
}
for index, peerID := range viewerPeerIDs {
operationStartedAtMs := int64(index+1) * 10
eq(t, must(t, routes.Reconcile(operationStartedAtMs).Operation).ChildPeerID, peerID)
commitCurrent(t, routes, operationStartedAtMs+1, peerID+"_connection")
}
snapshot := routes.RouteDiagnosticSnapshot(250)
eq(t, len(snapshot.Children), 20)
for index, child := range snapshot.Children {
eqDeep(t, child.QueueWaitMs, optInt(int64(index+1)*10))
eq(t, child.FinalRoute, string(FinalRouteDirect))
}
}
// TS 2598: projects latest route timing through snapshot-local ordinals
func TestProjectsLatestRouteTimingThroughSnapshotLocalOrdinals(t *testing.T) {
routes := newController(2, Options{})
addViewer(routes, A, 1, ms(100))
eqDeep(t, routes.RouteDiagnosticSnapshot(150), protocol.RouteDiagnosticSnapshot{
Children: []protocol.RouteDiagnosticChild{{
Ordinal: 1,
Parent: protocol.RouteDiagnosticParent{Kind: "none"},
EffectiveCapacity: 1,
ChildCount: 0,
DemandAgeMs: optInt(50),
QueueWaitMs: nil,
CandidateStartMs: nil,
FirstDecodedFrameMs: nil,
FinalMs: nil,
FinalRoute: "waiting",
RejectionBucket: "none",
Quality: nil,
}},
Operation: nil,
})
operation := must(t, routes.Reconcile(160).Operation)
eqDeep(t, routes.RouteDiagnosticSnapshot(165).Operation, &protocol.RouteDiagnosticOperation{
ChildOrdinal: 1,
Reason: "join",
Stage: "admission",
Cursor: 0,
CandidateCount: protocol.Int(len(operation.Candidates)),
})
prepared := beginOperation(t, routes, BeginInput{NowMs: 170, ConnectionID: "a_connection", Reservation: direct()})
pending := routes.RouteDiagnosticSnapshot(180)
eq(t, len(pending.Children), 1)
eqDeep(t, pending.Children[0].QueueWaitMs, optInt(60))
eqDeep(t, pending.Children[0].CandidateStartMs, optInt(70))
eqDeep(t, pending.Children[0].FirstDecodedFrameMs, (*protocol.Int)(nil))
eqDeep(t, pending.Children[0].FinalMs, (*protocol.Int)(nil))
if pending.Operation == nil {
t.Fatal("expected a diagnostic operation")
}
eq(t, pending.Operation.ChildOrdinal, 1)
eq(t, pending.Operation.Stage, "first-frame")
eq(t, routes.CandidateReady(guardFor(A, prepared.Current.Revision, "a_connection"), 230, nil, CandidateProof{}).Accepted, true)
settled := routes.RouteDiagnosticSnapshot(240)
eq(t, len(settled.Children), 1)
eq(t, settled.Children[0].Parent, protocol.RouteDiagnosticParent{Kind: "host"})
eqDeep(t, settled.Children[0].QueueWaitMs, optInt(60))
eqDeep(t, settled.Children[0].CandidateStartMs, optInt(70))
eqDeep(t, settled.Children[0].FirstDecodedFrameMs, optInt(130))
eqDeep(t, settled.Children[0].FinalMs, optInt(130))
eq(t, settled.Children[0].FinalRoute, "direct")
eq(t, settled.Children[0].RejectionBucket, "none")
if settled.Operation != nil {
t.Fatal("expected no diagnostic operation")
}
serialized, err := json.Marshal(settled)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(serialized), A) {
t.Fatalf("diagnostic snapshot leaks the peer ID: %s", serialized)
}
if strings.Contains(string(serialized), "a_connection") {
t.Fatalf("diagnostic snapshot leaks the connection ID: %s", serialized)
}
eq(t, routes.InvalidateEdge(edgeGuard(A, "host_session", routes.Snapshot().Revision, "a_connection"), ms(300)), true)
waiting := routes.RouteDiagnosticSnapshot(320).Children[0]
eqDeep(t, waiting.DemandAgeMs, optInt(20))
eqDeep(t, waiting.QueueWaitMs, (*protocol.Int)(nil))
eqDeep(t, waiting.CandidateStartMs, (*protocol.Int)(nil))
eqDeep(t, waiting.FirstDecodedFrameMs, (*protocol.Int)(nil))
eqDeep(t, waiting.FinalMs, (*protocol.Int)(nil))
eq(t, waiting.FinalRoute, "waiting")
eq(t, waiting.RejectionBucket, "none")
eq(t, routes.ConfirmDeparture(A, ms(330)), true)
departed := routes.RouteDiagnosticSnapshot(330)
eq(t, len(departed.Children), 0)
if departed.Operation != nil {
t.Fatal("expected no diagnostic operation")
}
}
// TS 2704: records a bounded candidate failure without retaining route identity
func TestRecordsBoundedCandidateFailureWithoutRetainingRouteIdentity(t *testing.T) {
routes := newController(1, Options{})
addViewer(routes, A, 0, ms(0))
routes.Reconcile(10)
prepared := beginOperation(t, routes, BeginInput{NowMs: 20, ConnectionID: "failed_candidate", Reservation: direct()})
failed := routes.CandidateFailed(guardFor(A, prepared.Current.Revision, "failed_candidate"), 30)
eq(t, failed.Accepted, true)
eqSlice(t, failed.FailedPeerIDs, []string{A})
child := routes.RouteDiagnosticSnapshot(30).Children[0]
eqDeep(t, child.FinalMs, optInt(30))
eq(t, child.FinalRoute, "failed")
eq(t, child.RejectionBucket, "candidate-failed")
routes.Dispose()
disposed := routes.RouteDiagnosticSnapshot(40)
eq(t, len(disposed.Children), 0)
if disposed.Operation != nil {
t.Fatal("expected no diagnostic operation after dispose")
}
}
// TS 2737: reports a current child that has no usable candidate
func TestReportsCurrentChildThatHasNoUsableCandidate(t *testing.T) {
routes := newController(1, Options{})
addViewer(routes, A, 0, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_connection"))
addViewer(routes, B, 0, ms(0))
result := routes.Reconcile(10)
noOperation(t, result.Operation)
eqSlice(t, result.FailedPeerIDs, []string{B})
child := routes.RouteDiagnosticSnapshot(10).Children[1]
eqDeep(t, child.FinalMs, optInt(10))
eq(t, child.FinalRoute, "failed")
eq(t, child.RejectionBucket, "candidate-failed")
}
// TS 2753: records SFU admission waiting without advancing the route cursor
func TestRecordsSfuAdmissionWaitingWithoutAdvancingRouteCursor(t *testing.T) {
routes := newController(1, Options{SfuEnabled: true})
addViewer(routes, A, 0, ms(0))
directOp := must(t, routes.Reconcile(10).Operation)
eqSlice(t, tupleKinds(directOp), []UpstreamKind{UpstreamPeer, UpstreamSfu})
eq(t, routes.SkipCurrentCandidate(cursorGuard(directOp), 20, RejectionCandidateFailed).Accepted, true)
sfu := must(t, routes.Snapshot().Operation)
eq(t, routes.NoteCurrentCandidateRejection(cursorGuard(sfu), RejectionSfuAdmission), true)
snapshot := routes.RouteDiagnosticSnapshot(30)
eq(t, len(snapshot.Children), 1)
eq(t, snapshot.Children[0].FinalRoute, "waiting")
eq(t, snapshot.Children[0].RejectionBucket, "sfu-admission")
if snapshot.Operation == nil {
t.Fatal("expected a diagnostic operation")
}
eq(t, *snapshot.Operation, protocol.RouteDiagnosticOperation{
ChildOrdinal: 1, Reason: "join", Stage: "admission", Cursor: 1, CandidateCount: 2,
})
}
// TS 2791: distinguishes first-frame timeout from the total operation deadline
func TestDistinguishesFirstFrameTimeoutFromTotalOperationDeadline(t *testing.T) {
routes := newController(1, Options{SfuEnabled: true})
addViewer(routes, A, 0, ms(0))
directOp := must(t, routes.Reconcile(10).Operation)
beginCandidate(t, routes, BeginInput{NowMs: 20, ConnectionID: "silent_direct", Reservation: direct()})
eq(t, routes.OperationExpired(directOp.WakeAtMs).Accepted, true)
snapshot := routes.RouteDiagnosticSnapshot(directOp.WakeAtMs)
eq(t, len(snapshot.Children), 1)
eqDeep(t, snapshot.Children[0].CandidateStartMs, (*protocol.Int)(nil))
eqDeep(t, snapshot.Children[0].FinalMs, (*protocol.Int)(nil))
eq(t, snapshot.Children[0].FinalRoute, "waiting")
eq(t, snapshot.Children[0].RejectionBucket, "first-frame-timeout")
if snapshot.Operation == nil {
t.Fatal("expected a diagnostic operation")
}
eq(t, snapshot.Operation.Cursor, 1)
eq(t, snapshot.Operation.Stage, "admission")
deadline := must(t, routes.Snapshot().Operation).DeadlineAtMs
expired := routes.OperationExpired(deadline)
eq(t, expired.Accepted, true)
eqSlice(t, expired.FailedPeerIDs, []string{A})
child := routes.RouteDiagnosticSnapshot(deadline).Children[0]
eqDeep(t, child.FinalMs, optInt(deadline))
eq(t, child.FinalRoute, "failed")
eq(t, child.RejectionBucket, "operation-deadline")
}
// TS 2828: admits 20 Viewers through one bounded event-driven graph at C=%i
func TestAdmitsTwentyViewersThroughOneBoundedGraph(t *testing.T) {
for _, capacity := range []int{1, 2, 3} {
t.Run(fmt.Sprintf("C=%d", capacity), func(t *testing.T) {
routes := newController(capacity, Options{})
viewerPeerIDs := make([]string, 20)
for index := range viewerPeerIDs {
viewerPeerIDs[index] = fmt.Sprintf("viewer_%02d_12345678", index)
}
for index, peerID := range viewerPeerIDs {
addViewer(routes, peerID, capacity, nil)
operation := must(t, routes.Reconcile(int64(index*2)).Operation)
eq(t, operation.ChildPeerID, peerID)
commitCurrent(t, routes, int64(index*2+1), peerID+"_connection")
}
snapshot := routes.Snapshot()
eq(t, snapshot.UpstreamByViewer.Len(), 20)
for _, parentPeerID := range append([]string{HOST}, viewerPeerIDs...) {
childCount := 0
for _, edge := range snapshot.UpstreamByViewer.Values() {
if edge.Kind == UpstreamPeer && edge.PhysicalActive && edge.ParentPeerID == parentPeerID {
childCount++
}
}
if childCount > capacity {
t.Fatalf("parent %q has %d children, capacity %d", parentPeerID, childCount, capacity)
}
}
})
}
}
// TS 2856: admits waiting Viewers through existing SFU before direct convergence
func TestAdmitsWaitingViewersThroughExistingSfuBeforeDirectConvergence(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
addViewer(routes, B, 2, nil)
routes.hydrateHostPublication("publication_generation", res("publication_resource"), "")
routes.hydrateEdge(B, sfuEdge("publication_generation", "b_sfu", "b_subscription"))
addViewer(routes, C, 0, nil)
routes.hydrateEdge(C, peerEdge(HOST, "c_from_host"))
waiting := make([]string, 18)
for index := range waiting {
waiting[index] = fmt.Sprintf("silent_%02d_12345678", index)
}
for _, peerID := range waiting {
addViewer(routes, peerID, 0, nil)
}
nowMs := int64(0)
for index, peerID := range waiting {
directOperation := must(t, routes.Reconcile(nowMs).Operation)
eq(t, directOperation.ChildPeerID, peerID)
eq(t, directOperation.Candidates[0].Tuple.Kind, UpstreamPeer)
beginCandidate(t, routes, BeginInput{NowMs: nowMs + 1, ConnectionID: peerID + "_direct", Reservation: direct()})
routes.OperationExpired(directOperation.WakeAtMs)
sfuOperation := must(t, routes.Snapshot().Operation)
eq(t, sfuOperation.Candidates[sfuOperation.Cursor].Tuple, sfuTuple(PublicationReuse))
prepared := beginOperation(t, routes, BeginInput{
NowMs: directOperation.WakeAtMs + 1, ConnectionID: peerID + "_sfu", Reservation: sfuReuse(peerID + "_subscription"),
})
eq(t, routes.CandidateReady(guardFor(peerID, prepared.Current.Revision, peerID+"_sfu"),
directOperation.WakeAtMs+2, nil, CandidateProof{}).Accepted, true)
nowMs = directOperation.WakeAtMs + 3
bEdge := edgeOf(t, routes, B)
eq(t, bEdge.Kind, UpstreamSfu)
eq(t, bEdge.ConnectionID, "b_sfu")
cEdge := edgeOf(t, routes, C)
eq(t, cEdge.Kind, UpstreamPeer)
eq(t, cEdge.ParentPeerID, HOST)
eq(t, cEdge.ConnectionID, "c_from_host")
admitted := edgeOf(t, routes, peerID)
eq(t, admitted.Kind, UpstreamSfu)
eq(t, admitted.Usable, true)
eq(t, routes.Snapshot().UpstreamByViewer.Len(), index+3)
}
eq(t, routes.Snapshot().UpstreamByViewer.Len(), 20)
publication := routes.Snapshot().HostPublication
if publication == nil {
t.Fatal("expected a host publication")
}
eq(t, publication.Generation, "publication_generation")
eq(t, publication.Usable, true)
eq(t, must(t, routes.Reconcile(nowMs).Operation).Reason, DemandDirectConvergence)
}
// TS 2943: builds a deterministic acyclic route without a depth cap at C=%i
func TestBuildsDeterministicAcyclicRouteWithoutDepthCap(t *testing.T) {
for _, capacity := range []int{1, 2, 3} {
t.Run(fmt.Sprintf("C=%d", capacity), func(t *testing.T) {
routes := newController(capacity, Options{})
for _, peerID := range []string{A, B, C, D} {
addViewer(routes, peerID, capacity, nil)
result := routes.Reconcile(0)
eq(t, must(t, result.Operation).ChildPeerID, peerID)
commitCurrent(t, routes, 1, peerID+"_connection")
}
snapshot := routes.Snapshot()
eq(t, snapshot.UpstreamByViewer.Len(), 4)
for childPeerID, edge := range snapshot.UpstreamByViewer.All() {
eq(t, edge.Kind, UpstreamPeer)
if edge.ParentPeerID == childPeerID {
t.Fatalf("%q is its own parent", childPeerID)
}
}
if capacity == 1 {
// Pins the stablePairRank/depth ordering: at C=1 D hangs below C.
dEdge := edgeOf(t, routes, D)
eq(t, dEdge.Kind, UpstreamPeer)
eq(t, dEdge.ParentPeerID, C)
}
})
}
}
// TS 2969: keeps the old edge until exact candidate first-frame ready
func TestKeepsOldEdgeUntilExactCandidateFirstFrameReady(t *testing.T) {
routes := newController(2, Options{})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
routes.Reconcile(0)
commitCurrent(t, routes, 1, "old_connection")
routes.Reconcile(2)
commitCurrent(t, routes, 3, "b_connection")
revision := routes.Snapshot().Revision
eq(t, routes.InvalidateEdge(edgeGuard(A, "host_session", revision, "old_connection"), nil), true)
routes.Reconcile(100)
prepared := beginOperation(t, routes, BeginInput{NowMs: 101, ConnectionID: "new_connection", Reservation: direct()})
eq(t, edgeOf(t, routes, A).ConnectionID, "old_connection")
eq(t, routes.CandidateReady(guardFor(A, prepared.Current.Revision-1, "new_connection"), 102, nil, CandidateProof{}).Accepted, false)
eq(t, routes.CandidateReady(guardFor(A, prepared.Current.Revision, "new_connection"), 102, nil, CandidateProof{}).Accepted, true)
edge := edgeOf(t, routes, A)
eq(t, edge.ConnectionID, "new_connection")
eq(t, edge.ParentPeerID, B)
eq(t, edge.Usable, true)
}
// TS 3020: invalidates one exact active Peer edge from its parent idempotently
func TestInvalidatesOneExactActivePeerEdgeFromParentIdempotently(t *testing.T) {
routes := newController(2, Options{})
addViewer(routes, A, 0, nil)
addViewer(routes, B, 1, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_from_host"))
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
report := ParentEdgeGuard{
ParentPeerID: HOST,
ParentSessionID: "host_session",
RouteRevision: routes.Snapshot().Revision,
ConnectionID: "a_from_host",
}
eq(t, routes.InvalidateDirectEdgeFromParent(report, ms(10)), true)
afterFirst := routes.Snapshot()
aEdge, _ := afterFirst.UpstreamByViewer.Get(A)
eq(t, aEdge.Usable, false)
eq(t, aEdge.ConnectionID, "a_from_host")
bEdge, _ := afterFirst.UpstreamByViewer.Get(B)
eq(t, bEdge.Usable, true)
eq(t, bEdge.ConnectionID, "b_from_host")
eq(t, routes.InvalidateDirectEdgeFromParent(report, ms(11)), true)
eq(t, routes.Snapshot().FactVersion, afterFirst.FactVersion)
unknown := report
unknown.ConnectionID = "unknown_connection"
eq(t, routes.InvalidateDirectEdgeFromParent(unknown, nil), false)
operation := must(t, routes.Reconcile(12).Operation)
eq(t, operation.ChildPeerID, A)
eq(t, operation.Reason, DemandEdgeUnavailable)
eqSlice(t, tuples(operation), []CandidateTuple{peerTuple(B)})
}
// TS 3062: advances one candidate cursor without resetting the operation deadline
func TestAdvancesOneCandidateCursorWithoutResettingOperationDeadline(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
addViewer(routes, A, 2, nil)
operation := must(t, routes.Reconcile(1_000).Operation)
deadline := operation.DeadlineAtMs
first := beginOperation(t, routes, BeginInput{NowMs: 1_001, ConnectionID: "candidate_1", Reservation: direct()})
failed := routes.CandidateFailed(guardFor(A, first.Current.Revision, "candidate_1"), 1_001)
eq(t, failed.Accepted, true)
if failed.ActiveRevision <= first.Current.Revision {
t.Fatalf("activeRevision %d did not advance past %d", failed.ActiveRevision, first.Current.Revision)
}
eqSlice(t, failed.FailedPeerIDs, []string{})
eqLabels(t, failed.Released)
pending := must(t, routes.Snapshot().Operation)
eq(t, pending.Cursor, 1)
eq(t, pending.DeadlineAtMs, deadline)
second := beginOperation(t, routes, BeginInput{
NowMs: 1_002,
ConnectionID: "candidate_2",
PublicationGeneration: "publication_2",
PublicationConnectionID: "publication_connection_2",
Reservation: sfuCreate("candidate_2_edge", "candidate_2_publication"),
})
lateReady := routes.CandidateReady(guardFor(A, second.Current.Revision, "candidate_2"), deadline, nil, CandidateProof{})
eq(t, lateReady.Accepted, false)
eqSlice(t, lateReady.FailedPeerIDs, []string{A})
eqLabels(t, lateReady.Released, "candidate_2_edge", "candidate_2_publication")
noOperation(t, routes.Snapshot().Operation)
eq(t, routes.CandidateReady(guardFor(A, second.Current.Revision, "candidate_2"), deadline+1, nil, CandidateProof{}).Accepted, false)
noOperation(t, routes.Reconcile(deadline+1).Operation)
routes.TouchExternalFacts()
eq(t, must(t, routes.Reconcile(deadline+2).Operation).ChildPeerID, A)
}
// TS 3123: reserves one derived deadline stage for direct and SFU
func TestReservesOneDerivedDeadlineStageForDirectAndSfu(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
addViewer(routes, B, 2, nil)
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
addViewer(routes, A, 0, nil)
operation := must(t, routes.Reconcile(0).Operation)
eq(t, operation.DeadlineAtMs, 10_000)
eq(t, operation.WakeAtMs, 5_000)
for _, want := range []CandidateTuple{peerTuple(B), sfuTuple(PublicationCreate)} {
if !slices.Contains(tuples(operation), want) {
t.Fatalf("candidates %+v do not contain %+v", tuples(operation), want)
}
}
beginCandidate(t, routes, BeginInput{NowMs: 1, ConnectionID: "silent_direct", Reservation: direct()})
directExpired := routes.OperationExpired(operation.WakeAtMs)
eq(t, directExpired.Accepted, true)
eqSlice(t, directExpired.FailedPeerIDs, []string{})
pending := must(t, routes.Snapshot().Operation)
eq(t, pending.DeadlineAtMs, 10_000)
eq(t, pending.WakeAtMs, 10_000)
if pending.Current != nil {
t.Fatal("expected no live attempt")
}
eq(t, pending.Candidates[pending.Cursor].Tuple.Kind, UpstreamSfu)
prepared := beginOperation(t, routes, BeginInput{
NowMs: 7_000,
ConnectionID: "sfu_subscription",
PublicationGeneration: "publication_generation",
PublicationConnectionID: "publication_connection",
Reservation: sfuCreate("sfu_edge", "sfu_publication"),
})
eq(t, routes.CandidateReady(guardFor(A, prepared.Current.Revision, "sfu_subscription"), 9_000, nil, CandidateProof{}).Accepted, true)
edge := edgeOf(t, routes, A)
eq(t, edge.Kind, UpstreamSfu)
eq(t, edge.Usable, true)
}
// TS 3186: uses the same direct head-start after an early hard failure
func TestUsesSameDirectHeadStartAfterEarlyHardFailure(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
addViewer(routes, D, 0, nil)
routes.hydrateEdge(D, peerEdge(HOST, "host_full"))
routes.hydrateHostPublication("publication_1", res("publication_resource"), "")
for _, peerID := range []string{B, C} {
addViewer(routes, peerID, 1, nil)
routes.hydrateEdge(peerID, sfuEdge("publication_1", peerID+"_sfu", peerID+"_subscription"))
}
addViewer(routes, A, 0, nil)
operation := must(t, routes.Reconcile(0).Operation)
eqSlice(t, tupleKinds(operation), []UpstreamKind{UpstreamPeer, UpstreamSfu, UpstreamPeer})
first := beginOperation(t, routes, BeginInput{NowMs: 1, ConnectionID: "first_direct", Reservation: direct()})
eq(t, routes.CandidateFailed(guardFor(A, first.Current.Revision, "first_direct"), 2).Accepted, true)
pending := must(t, routes.Snapshot().Operation)
eq(t, pending.Cursor, 1)
eq(t, pending.WakeAtMs, operation.WakeAtMs)
eq(t, pending.Candidates[1].Tuple.Kind, UpstreamPeer)
}
// TS 3231: keeps a transport-connected direct candidate until the total deadline
func TestKeepsTransportConnectedDirectCandidateUntilTotalDeadline(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
addViewer(routes, D, 0, nil)
routes.hydrateEdge(D, peerEdge(HOST, "host_full"))
routes.hydrateHostPublication("publication", res("publication_resource"), "")
addViewer(routes, B, 1, nil)
routes.hydrateEdge(B, sfuEdge("publication", "b_sfu", "b_subscription"))
addViewer(routes, A, 0, nil)
operation := must(t, routes.Reconcile(0).Operation)
directOp := beginOperation(t, routes, BeginInput{NowMs: 1, ConnectionID: "a_direct", Reservation: direct()})
eq(t, routes.CandidateTransportConnected(guardFor(A, directOp.Current.Revision, "a_direct"), 1_000).Accepted, true)
eq(t, routes.CandidateTransportConnected(guardFor(A, directOp.Current.Revision, "a_direct"), 1_001).Accepted, true)
eq(t, routes.CandidateTransportConnected(guardFor(A, directOp.Current.Revision, "stale_direct"), 1_002).Accepted, false)
eq(t, routes.Snapshot().Revision, directOp.Current.Revision-1)
eq(t, must(t, routes.Snapshot().Operation).WakeAtMs, operation.DeadlineAtMs)
eq(t, routes.OperationExpired(operation.WakeAtMs).Accepted, false)
eq(t, must(t, routes.Snapshot().Operation).Current.ConnectionID, "a_direct")
eq(t, routes.CandidateReady(guardFor(A, directOp.Current.Revision, "a_direct"), operation.WakeAtMs+1, nil, CandidateProof{}).Accepted, true)
edge := edgeOf(t, routes, A)
eq(t, edge.Kind, UpstreamPeer)
eq(t, edge.ConnectionID, "a_direct")
}
// TS 3288: keeps SFU active while converging through an SFU-fed parent
func TestKeepsSfuActiveWhileConvergingThroughSfuFedParent(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
addViewer(routes, B, 2, nil)
routes.hydrateHostPublication("publication_generation", res("publication_resource"), "")
routes.hydrateEdge(B, sfuEdge("publication_generation", "b_sfu", "b_subscription"))
addViewer(routes, C, 0, nil)
routes.hydrateEdge(C, peerEdge(HOST, "c_from_host"))
addViewer(routes, A, 0, nil)
acquisition := must(t, routes.Reconcile(0).Operation)
eq(t, acquisition.Candidates[0].Tuple, peerTuple(B))
beginCandidate(t, routes, BeginInput{NowMs: 1, ConnectionID: "a_direct_head_start", Reservation: direct()})
routes.OperationExpired(acquisition.WakeAtMs)
prepared := beginOperation(t, routes, BeginInput{
NowMs: acquisition.WakeAtMs + 1, ConnectionID: "a_sfu_reuse", Reservation: sfuReuse("a_subscription"),
})
eq(t, routes.CandidateReady(guardFor(A, prepared.Current.Revision, "a_sfu_reuse"), acquisition.WakeAtMs+2, nil, CandidateProof{}).Accepted, true)
publication := routes.Snapshot().HostPublication
if publication == nil {
t.Fatal("expected a host publication")
}
eq(t, publication.Generation, "publication_generation")
eq(t, publication.Usable, true)
edge := edgeOf(t, routes, A)
eq(t, edge.Kind, UpstreamSfu)
eq(t, edge.PublicationGeneration, "publication_generation")
eq(t, edge.Usable, true)
convergence := must(t, routes.Reconcile(acquisition.WakeAtMs+3).Operation)
eq(t, convergence.ChildPeerID, A)
eq(t, convergence.Reason, DemandDirectConvergence)
eq(t, convergence.Candidates[0].Tuple, peerTuple(B))
directOp := beginOperation(t, routes, BeginInput{NowMs: acquisition.WakeAtMs + 4, ConnectionID: "a_from_b", Reservation: direct()})
eq(t, must(t, routes.Snapshot().Operation).WakeAtMs, convergence.DeadlineAtMs)
edge = edgeOf(t, routes, A)
eq(t, edge.Kind, UpstreamSfu)
eq(t, edge.ConnectionID, "a_sfu_reuse")
eq(t, routes.CandidateReady(guardFor(A, directOp.Current.Revision, "a_from_b"), acquisition.WakeAtMs+5, nil, CandidateProof{}).Accepted, true)
edge = edgeOf(t, routes, A)
eq(t, edge.Kind, UpstreamPeer)
eq(t, edge.ParentPeerID, B)
eq(t, edge.ConnectionID, "a_from_b")
}
// TS 3380: advances exact direct convergence failures without failing SFU
func TestAdvancesExactDirectConvergenceFailuresWithoutFailingSfu(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
addViewer(routes, D, 0, nil)
routes.hydrateEdge(D, peerEdge(HOST, "host_full"))
routes.hydrateHostPublication("publication_1", res("publication_resource"), "")
for _, peerID := range []string{B, C} {
addViewer(routes, peerID, 1, nil)
routes.hydrateEdge(peerID, sfuEdge("publication_1", peerID+"_sfu", peerID+"_subscription"))
}
addViewer(routes, A, 0, ms(0))
acquisition := must(t, routes.Reconcile(0).Operation)
headStartTuple := acquisition.Candidates[0].Tuple
eq(t, headStartTuple.Kind, UpstreamPeer)
headStartParent := headStartTuple.ParentPeerID
otherParent := B
if headStartParent == B {
otherParent = C
}
beginCandidate(t, routes, BeginInput{NowMs: 1, ConnectionID: "a_direct_head_start", Reservation: direct()})
routes.OperationExpired(acquisition.WakeAtMs)
sfu := beginOperation(t, routes, BeginInput{NowMs: acquisition.WakeAtMs + 1, ConnectionID: "a_sfu", Reservation: sfuReuse("a_subscription")})
eq(t, routes.CandidateReady(guardFor(A, sfu.Current.Revision, "a_sfu"), acquisition.WakeAtMs+2, nil, CandidateProof{}).Accepted, true)
first := must(t, routes.Reconcile(acquisition.WakeAtMs+3).Operation)
eq(t, first.Candidates[0].Tuple.Kind, UpstreamPeer)
eq(t, first.Candidates[0].Tuple.ParentPeerID, otherParent)
failed := beginOperation(t, routes, BeginInput{NowMs: acquisition.WakeAtMs + 4, ConnectionID: "a_direct_followup", Reservation: direct()})
eqSlice(t, routes.CandidateFailed(guardFor(A, failed.Current.Revision, "a_direct_followup"), acquisition.WakeAtMs+5).FailedPeerIDs, []string{})
edge := edgeOf(t, routes, A)
eq(t, edge.Kind, UpstreamSfu)
eq(t, edge.ConnectionID, "a_sfu")
retry := must(t, routes.Reconcile(acquisition.WakeAtMs+6).Operation)
eq(t, retry.Candidates[0].Tuple.Kind, UpstreamPeer)
eq(t, retry.Candidates[0].Tuple.ParentPeerID, headStartParent)
}
// TS 3451: round-robins background direct convergence between SFU Viewers
func TestRoundRobinsBackgroundDirectConvergenceBetweenSfuViewers(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
routes.hydrateHostPublication("publication", res("publication_resource"), "")
for _, peerID := range []string{B, C} {
addViewer(routes, peerID, 2, nil)
routes.hydrateEdge(peerID, sfuEdge("publication", peerID+"_sfu", peerID+"_subscription"))
}
nowMs := int64(0)
for _, peerID := range []string{A, D} {
addViewer(routes, peerID, 0, ms(nowMs))
acquisition := must(t, routes.Reconcile(nowMs).Operation)
beginCandidate(t, routes, BeginInput{NowMs: nowMs + 1, ConnectionID: peerID + "_head_start", Reservation: direct()})
routes.OperationExpired(acquisition.WakeAtMs)
sfu := beginOperation(t, routes, BeginInput{
NowMs: acquisition.WakeAtMs + 1, ConnectionID: peerID + "_sfu", Reservation: sfuReuse(peerID + "_subscription"),
})
eq(t, routes.CandidateReady(guardFor(peerID, sfu.Current.Revision, peerID+"_sfu"), acquisition.WakeAtMs+2, nil, CandidateProof{}).Accepted, true)
nowMs = acquisition.WakeAtMs + 3
}
first := must(t, routes.Reconcile(nowMs).Operation)
eq(t, first.ChildPeerID, A)
eq(t, first.Reason, DemandDirectConvergence)
firstAttempt := beginOperation(t, routes, BeginInput{NowMs: nowMs + 1, ConnectionID: "a_background_direct", Reservation: direct()})
eq(t, routes.CandidateFailed(guardFor(A, firstAttempt.Current.Revision, "a_background_direct"), nowMs+2).Accepted, true)
next := must(t, routes.Reconcile(nowMs+3).Operation)
eq(t, next.ChildPeerID, D)
eq(t, next.Reason, DemandDirectConvergence)
}
// TS 3516: preempts direct convergence when another Viewer needs a route
func TestPreemptsDirectConvergenceWhenAnotherViewerNeedsRoute(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
addViewer(routes, B, 2, nil)
routes.hydrateHostPublication("publication_1", res("publication_resource"), "")
routes.hydrateEdge(B, sfuEdge("publication_1", "b_sfu", "b_subscription"))
addViewer(routes, C, 0, nil)
routes.hydrateEdge(C, peerEdge(HOST, "c_from_host"))
addViewer(routes, A, 0, ms(0))
acquisition := must(t, routes.Reconcile(0).Operation)
beginCandidate(t, routes, BeginInput{NowMs: 1, ConnectionID: "a_direct_head_start", Reservation: direct()})
routes.OperationExpired(acquisition.WakeAtMs)
sfu := beginOperation(t, routes, BeginInput{NowMs: acquisition.WakeAtMs + 1, ConnectionID: "a_sfu", Reservation: sfuReuse("a_subscription")})
routes.CandidateReady(guardFor(A, sfu.Current.Revision, "a_sfu"), acquisition.WakeAtMs+2, nil, CandidateProof{})
eq(t, must(t, routes.Reconcile(acquisition.WakeAtMs+3).Operation).Reason, DemandDirectConvergence)
beginCandidate(t, routes, BeginInput{NowMs: acquisition.WakeAtMs + 4, ConnectionID: "a_direct_pending", Reservation: directOverlap("a_overlap")})
activeRevision := routes.Snapshot().Revision
eq(t, routes.InvalidateEdge(edgeGuard(C, "host_session", activeRevision, "c_from_host"), ms(acquisition.WakeAtMs+5)), true)
reconciled := routes.Reconcile(acquisition.WakeAtMs + 6)
eqLabels(t, reconciled.Released, "a_overlap")
edge := edgeOf(t, routes, A)
eq(t, edge.Kind, UpstreamSfu)
eq(t, edge.ConnectionID, "a_sfu")
preempted := must(t, reconciled.Operation)
eq(t, preempted.ChildPeerID, C)
eq(t, preempted.Reason, DemandEdgeUnavailable)
}
// TS 3579: owns Host-full SFU bootstrap failure by the waiting demand
func TestOwnsHostFullSfuBootstrapFailureByWaitingDemand(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
addViewer(routes, B, 2, nil)
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
addViewer(routes, C, 0, nil)
routes.hydrateEdge(C, peerEdge(HOST, "c_from_host"))
addViewer(routes, A, 0, nil)
addViewer(routes, D, 0, nil)
operation := must(t, routes.Reconcile(0).Operation)
eq(t, hasTuple(operation, isSfu), false)
eq(t, operation.WakeAtMs, 5_000)
beginCandidate(t, routes, BeginInput{NowMs: 1, ConnectionID: "a_silent_direct", Reservation: direct()})
blocked := routes.OperationExpired(operation.WakeAtMs)
eq(t, blocked.Accepted, true)
eqSlice(t, blocked.FailedPeerIDs, []string{})
eqLabels(t, blocked.Released)
noOperation(t, routes.Snapshot().Operation)
bootstrap := must(t, routes.Reconcile(operation.WakeAtMs+1).Operation)
eq(t, bootstrap.ChildPeerID, C)
eq(t, bootstrap.DemandPeerID, A)
eq(t, bootstrap.Candidates[0].Tuple, sfuTuple(PublicationCreate))
eq(t, bootstrap.Candidates[0].EndpointTransition.Kind, TransitionOverlap)
firstCarrier := beginOperation(t, routes, BeginInput{
NowMs: operation.WakeAtMs + 2,
ConnectionID: "c_bootstrap_sfu",
PublicationGeneration: "c_bootstrap_publication",
PublicationConnectionID: "c_bootstrap_ingress",
Reservation: sfuCreateOverlap("c_bootstrap_subscription", "c_bootstrap_publication_resource", "c_bootstrap_overlap"),
})
eqSlice(t, routes.CandidateFailed(guardFor(C, firstCarrier.Current.Revision, "c_bootstrap_sfu"), operation.WakeAtMs+3).FailedPeerIDs, []string{})
cEdge := edgeOf(t, routes, C)
eq(t, cEdge.ConnectionID, "c_from_host")
eq(t, cEdge.Usable, true)
eq(t, cEdge.PhysicalActive, true)
secondBootstrap := must(t, routes.Reconcile(operation.WakeAtMs+4).Operation)
eq(t, secondBootstrap.ChildPeerID, B)
eq(t, secondBootstrap.DemandPeerID, A)
eq(t, secondBootstrap.Reason, DemandSfuBootstrap)
secondCarrier := beginOperation(t, routes, BeginInput{
NowMs: operation.WakeAtMs + 5,
ConnectionID: "b_bootstrap_sfu",
PublicationGeneration: "b_bootstrap_publication",
PublicationConnectionID: "b_bootstrap_ingress",
Reservation: sfuCreateOverlap("b_bootstrap_subscription", "b_bootstrap_publication_resource", "b_bootstrap_overlap"),
})
eqSlice(t, routes.CandidateFailed(guardFor(B, secondCarrier.Current.Revision, "b_bootstrap_sfu"), operation.WakeAtMs+6).FailedPeerIDs, []string{A})
for _, seed := range [][2]string{{B, "b_from_host"}, {C, "c_from_host"}} {
edge := edgeOf(t, routes, seed[0])
eq(t, edge.ConnectionID, seed[1])
eq(t, edge.Usable, true)
eq(t, edge.PhysicalActive, true)
}
nextDemand := must(t, routes.Reconcile(operation.WakeAtMs+7).Operation)
eq(t, nextDemand.ChildPeerID, D)
eq(t, nextDemand.Reason, DemandJoin)
beginCandidate(t, routes, BeginInput{NowMs: operation.WakeAtMs + 8, ConnectionID: "d_direct", Reservation: direct()})
eqSlice(t, routes.OperationExpired(nextDemand.DeadlineAtMs).FailedPeerIDs, []string{D})
noOperation(t, routes.Reconcile(nextDemand.DeadlineAtMs+1).Operation)
}
// TS 3680: abandons bootstrap when a Host slot becomes available
func TestAbandonsBootstrapWhenHostSlotBecomesAvailable(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
eq(t, routes.SetEffectiveCapacity(HOST, "host_session", 1, nil), true)
addViewer(routes, B, 0, nil)
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
addViewer(routes, A, 0, nil)
bootstrap := must(t, routes.Reconcile(0).Operation)
eq(t, bootstrap.ChildPeerID, B)
eq(t, bootstrap.DemandPeerID, A)
eq(t, bootstrap.Reason, DemandSfuBootstrap)
prepared := beginOperation(t, routes, BeginInput{
NowMs: 1,
ConnectionID: "stale_bootstrap",
PublicationGeneration: "stale_publication",
PublicationConnectionID: "stale_ingress",
Reservation: sfuCreateOverlap("stale_subscription", "stale_publication_resource", "stale_overlap"),
})
eq(t, routes.SetEffectiveCapacity(HOST, "host_session", 2, nil), true)
stale := routes.CandidateReady(guardFor(B, prepared.Current.Revision, "stale_bootstrap"), 2, nil, CandidateProof{})
eq(t, stale.Accepted, false)
containsLabels(t, stale.Released, "stale_subscription", "stale_publication_resource", "stale_overlap")
bEdge := edgeOf(t, routes, B)
eq(t, bEdge.ConnectionID, "b_from_host")
eq(t, bEdge.Usable, true)
eq(t, bEdge.PhysicalActive, true)
join := must(t, routes.Reconcile(3).Operation)
eq(t, join.ChildPeerID, A)
eq(t, join.DemandPeerID, A)
eq(t, join.Reason, DemandJoin)
}
// TS 3731: tries SFU first after bootstrap and retains the direct suffix
func TestTriesSfuFirstAfterBootstrapAndRetainsDirectSuffix(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
for _, seed := range [][2]string{{B, "b_from_host"}, {C, "c_from_host"}} {
addViewer(routes, seed[0], 1, nil)
routes.hydrateEdge(seed[0], peerEdge(HOST, seed[1]))
}
addViewer(routes, A, 0, nil)
acquisition := must(t, routes.Reconcile(0).Operation)
beginCandidate(t, routes, BeginInput{NowMs: 1, ConnectionID: "a_direct_head_start", Reservation: direct()})
routes.OperationExpired(acquisition.WakeAtMs)
bootstrap := must(t, routes.Reconcile(acquisition.WakeAtMs+1).Operation)
carrier := beginOperation(t, routes, BeginInput{
NowMs: acquisition.WakeAtMs + 2,
ConnectionID: "bootstrap_sfu",
PublicationGeneration: "bootstrap_publication",
PublicationConnectionID: "bootstrap_ingress",
Reservation: sfuCreateOverlap("carrier_subscription", "bootstrap_publication_resource", "bootstrap_overlap"),
})
eq(t, routes.CandidateReady(CandidateGuard{
ChildPeerID: bootstrap.ChildPeerID,
ChildSessionID: bootstrap.ChildSessionID,
Revision: carrier.Current.Revision,
ConnectionID: "bootstrap_sfu",
}, acquisition.WakeAtMs+3, nil, CandidateProof{}).Accepted, true)
demand := must(t, routes.Reconcile(acquisition.WakeAtMs+4).Operation)
eq(t, demand.ChildPeerID, A)
eq(t, demand.DemandPeerID, A)
eq(t, demand.Reason, DemandJoin)
eq(t, demand.Candidates[0].Tuple, sfuTuple(PublicationReuse))
for _, tuple := range tuplesFrom(demand, 1) {
eq(t, tuple.Kind, UpstreamPeer)
}
subscription := beginOperation(t, routes, BeginInput{NowMs: acquisition.WakeAtMs + 5, ConnectionID: "a_sfu", Reservation: sfuReuse("a_subscription")})
eq(t, routes.CandidateFailed(guardFor(A, subscription.Current.Revision, "a_sfu"), acquisition.WakeAtMs+6).Accepted, true)
pending := must(t, routes.Snapshot().Operation)
eq(t, pending.Candidates[pending.Cursor].Tuple.Kind, UpstreamPeer)
directOp := beginOperation(t, routes, BeginInput{NowMs: acquisition.WakeAtMs + 7, ConnectionID: "a_deferred_direct", Reservation: direct()})
eq(t, routes.CandidateReady(guardFor(A, directOp.Current.Revision, "a_deferred_direct"), acquisition.WakeAtMs+8, nil, CandidateProof{}).Accepted, true)
edge := edgeOf(t, routes, A)
eq(t, edge.Kind, UpstreamPeer)
eq(t, edge.ConnectionID, "a_deferred_direct")
convergence := must(t, routes.Reconcile(acquisition.WakeAtMs+9).Operation)
eq(t, convergence.ChildPeerID, bootstrap.ChildPeerID)
eq(t, convergence.Reason, DemandDirectConvergence)
}
// TS 3819: replans when the current candidate parent fails
func TestReplansWhenCurrentCandidateParentFails(t *testing.T) {
routes := newController(3, Options{})
addViewer(routes, A, 1, nil)
addViewer(routes, B, 1, nil)
addViewer(routes, C, 1, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_old"))
routes.hydrateEdge(B, peerEdge(HOST, "b_old"))
routes.hydrateEdge(C, peerEdge(HOST, "c_old"))
revision := routes.Snapshot().Revision
eq(t, routes.InvalidateEdge(edgeGuard(A, "host_session", revision, "a_old"), nil), true)
routes.Reconcile(0)
current := beginOperation(t, routes, BeginInput{NowMs: 1, ConnectionID: "a_new", Reservation: direct()})
currentTuple := current.Current.Tuple
eq(t, currentTuple.Kind, UpstreamPeer)
failedParentPeerID := currentTuple.ParentPeerID
failedParentConnectionID := "c_old"
if failedParentPeerID == B {
failedParentConnectionID = "b_old"
}
eq(t, routes.InvalidateEdge(edgeGuard(failedParentPeerID, "host_session", revision, failedParentConnectionID), nil), true)
eq(t, routes.CandidateReady(guardFor(A, current.Current.Revision, "a_new"), 2, nil, CandidateProof{}).Accepted, false)
replacement := beginOperation(t, routes, BeginInput{NowMs: 2, ConnectionID: "a_new_2", Reservation: direct()})
eq(t, routes.CandidateReady(guardFor(A, replacement.Current.Revision, "a_new_2"), 3, nil, CandidateProof{}).Accepted, true)
eq(t, must(t, routes.Reconcile(3).Operation).ChildPeerID, failedParentPeerID)
}
// TS 3886: keeps an active candidate when an unrelated session is replaced
func TestKeepsActiveCandidateWhenUnrelatedSessionIsReplaced(t *testing.T) {
routes := newController(2, Options{})
for _, seed := range [][2]string{{B, "b_from_host"}, {C, "c_from_host"}} {
addViewer(routes, seed[0], 1, nil)
routes.hydrateEdge(seed[0], peerEdge(HOST, seed[1]))
}
addViewer(routes, A, 0, nil)
operation := must(t, routes.Reconcile(0).Operation)
active := beginOperation(t, routes, BeginInput{NowMs: 1, ConnectionID: "a_active_candidate", Reservation: direct()})
tuple := active.Current.Tuple
eq(t, tuple.Kind, UpstreamPeer)
unrelatedPeerID := B
if tuple.ParentPeerID == B {
unrelatedPeerID = C
}
eqLabels(t, routes.UpsertParticipant(viewerInput(unrelatedPeerID, unrelatedPeerID+"_replacement", 1), ms(2)))
current := must(t, routes.Snapshot().Operation).Current
if current == nil {
t.Fatal("expected the live attempt to survive")
}
eq(t, current.Revision, operation.BaseRevision+1)
eq(t, current.ConnectionID, "a_active_candidate")
}
// TS 3920: uses a freed Host slot for new demand without moving healthy descendants
func TestUsesFreedHostSlotForNewDemandWithoutMovingHealthyDescendants(t *testing.T) {
routes := newController(2, Options{})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 2, nil)
addViewer(routes, C, 0, nil)
addViewer(routes, D, 0, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_root"))
routes.hydrateEdge(B, peerEdge(HOST, "b_root"))
routes.hydrateEdge(C, peerEdge(A, "c_from_a"))
routes.hydrateEdge(D, peerEdge(B, "d_from_b"))
eq(t, routes.ConfirmDeparture(B, nil), true)
repair := must(t, routes.Reconcile(0).Operation)
eq(t, repair.ChildPeerID, D)
eq(t, repair.Candidates[0].Tuple, peerTuple(A))
commitCurrent(t, routes, 1, "d_from_a")
noOperation(t, routes.Reconcile(2).Operation)
eq(t, hasEdge(routes, B), false)
addViewer(routes, E, 0, nil)
join := must(t, routes.Reconcile(3).Operation)
eq(t, join.ChildPeerID, E)
eq(t, join.Candidates[0].Tuple, peerTuple(HOST))
commitCurrent(t, routes, 4, "e_from_host")
cEdge := edgeOf(t, routes, C)
eq(t, cEdge.ParentPeerID, A)
eq(t, cEdge.ConnectionID, "c_from_a")
dEdge := edgeOf(t, routes, D)
eq(t, dEdge.ParentPeerID, A)
eq(t, dEdge.ConnectionID, "d_from_a")
eEdge := edgeOf(t, routes, E)
eq(t, eEdge.ParentPeerID, HOST)
eq(t, eEdge.ConnectionID, "e_from_host")
}
// TS 3967: keeps grace media, then reparents a confirmed relay departure
func TestKeepsGraceMediaThenReparentsConfirmedRelayDeparture(t *testing.T) {
routes := newController(2, Options{})
addViewer(routes, A, 2, nil)
addViewer(routes, B, 1, nil)
addViewer(routes, C, 0, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_connection"))
routes.hydrateEdge(B, peerEdge(A, "b_connection"))
routes.hydrateEdge(C, peerEdge(B, "c_connection"))
eq(t, routes.DisconnectSession(A, sessionOf(A)), true)
noOperation(t, routes.Reconcile(0).Operation)
eq(t, routes.ConfirmDeparture(A, nil), true)
eq(t, must(t, routes.Reconcile(1).Operation).ChildPeerID, B)
commitCurrent(t, routes, 2, "b_reparented")
beforePrune := routes.Snapshot().Revision
after := routes.Reconcile(3)
if !slices.Contains(after.RemovedPeerIDs, A) {
t.Fatalf("removedPeerIds %v does not contain A", after.RemovedPeerIDs)
}
if routes.Snapshot().Revision <= beforePrune {
t.Fatalf("revision %d did not advance past %d", routes.Snapshot().Revision, beforePrune)
}
cEdge := edgeOf(t, routes, C)
eq(t, cEdge.ParentPeerID, B)
eq(t, cEdge.ConnectionID, "c_connection")
}
// TS 3991: moves newest overflow children and supports effective capacity zero
func TestMovesNewestOverflowChildrenAndSupportsEffectiveCapacityZero(t *testing.T) {
routes := newController(3, Options{})
addViewer(routes, A, 3, nil)
routes.hydrateEdge(A, peerEdge(HOST, "a_connection"))
for _, peerID := range []string{B, C, D} {
addViewer(routes, peerID, 0, nil)
routes.hydrateEdge(peerID, peerEdge(A, peerID+"_connection"))
}
eq(t, routes.SetEffectiveCapacity(A, sessionOf(A), 1, nil), true)
eq(t, must(t, routes.Reconcile(0).Operation).ChildPeerID, D)
eqLabels(t, routes.SetPaused(true, nil))
eq(t, routes.SetEffectiveCapacity(A, sessionOf(A), 0, nil), true)
eqLabels(t, routes.SetPaused(false, nil))
eq(t, must(t, routes.Reconcile(1).Operation).ChildPeerID, D)
blocked := newController(1, Options{})
addViewer(blocked, A, 0, nil)
addViewer(blocked, B, 0, nil)
blocked.hydrateEdge(A, peerEdge(HOST, "blocked_a"))
blocked.hydrateEdge(B, peerEdge(A, "blocked_b"))
noOperation(t, blocked.Reconcile(0).Operation)
noOperation(t, blocked.Reconcile(1).Operation)
}
// TS 4015: aborts pending SFU resources on pause and repairs publication
func TestAbortsPendingSfuResourcesOnPauseAndRepairsPublication(t *testing.T) {
routes := newController(1, Options{SfuEnabled: true})
addViewer(routes, A, 1, nil)
addViewer(routes, B, 0, nil)
routes.hydrateHostPublication("publication_1", res("publication_resource"), "")
routes.hydrateEdge(A, sfuEdge("publication_1", "a_sfu", "a_subscription"))
operation := must(t, routes.Reconcile(0).Operation)
eq(t, operation.Candidates[0].Tuple, peerTuple(A))
beginCandidate(t, routes, BeginInput{NowMs: 1, ConnectionID: "b_direct_head_start", Reservation: direct()})
routes.OperationExpired(operation.WakeAtMs)
beginCandidate(t, routes, BeginInput{NowMs: operation.WakeAtMs + 1, ConnectionID: "b_candidate", Reservation: sfuReuse("b_subscription")})
eqLabels(t, routes.SetPaused(true, nil), "b_subscription")
noOperation(t, routes.Snapshot().Operation)
eq(t, edgeOf(t, routes, A).ConnectionID, "a_sfu")
routes.SetPaused(false, nil)
eq(t, routes.InvalidateHostPublication(PublicationGuard{
HostSessionID: "host_session",
RouteRevision: routes.Snapshot().Revision,
Generation: "publication_1",
ConnectionID: "publication:publication_1",
}, nil), true)
routes.TouchExternalFacts()
repair := must(t, routes.Reconcile(2).Operation)
eq(t, repair.ChildPeerID, A)
eq(t, repair.Candidates[0].Tuple, peerTuple(HOST))
skipCandidate(t, routes, 3)
eq(t, must(t, routes.Snapshot().Operation).Candidates[1].Tuple, sfuTuple(PublicationReplace))
expectPanic(t, "endpoint overlap reservation", func() {
beginCandidate(t, routes, BeginInput{
NowMs: 3,
ConnectionID: "a_sfu_2",
PublicationGeneration: "publication_2",
PublicationConnectionID: "publication_connection_2",
Reservation: sfuCreate("a_subscription_2", "publication_resource_2"),
})
})
publicationRepair := beginOperation(t, routes, BeginInput{
NowMs: 3,
ConnectionID: "a_sfu_2",
PublicationGeneration: "publication_2",
PublicationConnectionID: "publication_connection_2",
Reservation: sfuCreateOverlap("a_subscription_2", "publication_resource_2", "publication_overlap"),
})
repaired := routes.CandidateReady(guardFor(A, publicationRepair.Current.Revision, "a_sfu_2"), 4, nil, CandidateProof{})
eq(t, repaired.Accepted, true)
eq(t, len(repaired.Released), 3)
containsLabels(t, repaired.Released, "publication_overlap", "a_subscription", "publication_resource")
publication := routes.Snapshot().HostPublication
if publication == nil {
t.Fatal("expected a repaired publication")
}
eq(t, publication.Generation, "publication_2")
eq(t, publication.Usable, true)
eq(t, publication.PhysicalActive, true)
bootstrap := newController(1, Options{SfuEnabled: true})
addViewer(bootstrap, A, 0, nil)
addViewer(bootstrap, B, 0, nil)
bootstrap.hydrateEdge(A, peerEdge(HOST, "bootstrap_a"))
bootstrapOperation := must(t, bootstrap.Reconcile(0).Operation)
eq(t, bootstrapOperation.ChildPeerID, A)
eq(t, bootstrapOperation.Candidates[0].Tuple, sfuTuple(PublicationCreate))
expectPanic(t, "endpoint overlap reservation", func() {
beginCandidate(t, bootstrap, BeginInput{
NowMs: 2,
ConnectionID: "bootstrap_sfu",
PublicationGeneration: "bootstrap_publication",
PublicationConnectionID: "bootstrap_ingress_connection",
Reservation: sfuCreate("bootstrap_subscription", "bootstrap_ingress"),
})
})
preparedBootstrap := beginOperation(t, bootstrap, BeginInput{
NowMs: 2,
ConnectionID: "bootstrap_sfu",
PublicationGeneration: "bootstrap_publication",
PublicationConnectionID: "bootstrap_ingress_connection",
Reservation: sfuCreateOverlap("bootstrap_subscription", "bootstrap_ingress", "bootstrap_overlap"),
})
eq(t, bootstrap.CandidateReady(guardFor(A, preparedBootstrap.Current.Revision, "bootstrap_sfu"), 3, nil, CandidateProof{}).Accepted, true)
eq(t, must(t, bootstrap.Reconcile(4).Operation).ChildPeerID, B)
}
// TS 4153: reopens SFU bootstrap after the physical publication retires
func TestReopensSfuBootstrapAfterPhysicalPublicationRetires(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
addViewer(routes, A, 0, nil)
routes.hydrateHostPublication("retired_publication", res("publication_resource"), "")
routes.hydrateEdge(A, sfuEdge("retired_publication", "a_from_sfu", "a_subscription"))
publication := routes.Snapshot().HostPublication
if publication == nil {
t.Fatal("expected a host publication")
}
containsLabels(t, routes.RetireHostPublication(PublicationGuard{
HostSessionID: publication.HostSessionID,
RouteRevision: routes.Snapshot().Revision,
Generation: publication.Generation,
ConnectionID: publication.ConnectionID,
}), "a_subscription", "publication_resource")
if routes.Snapshot().HostPublication != nil {
t.Fatal("expected no host publication")
}
for _, seed := range [][2]string{{B, "b_from_host"}, {C, "c_from_host"}} {
addViewer(routes, seed[0], 0, nil)
routes.hydrateEdge(seed[0], peerEdge(HOST, seed[1]))
}
reconciled := routes.Reconcile(1)
eqSlice(t, reconciled.FailedPeerIDs, []string{})
operation := must(t, reconciled.Operation)
eq(t, operation.DemandPeerID, A)
eq(t, operation.Reason, DemandSfuBootstrap)
eqSlice(t, tuples(operation), []CandidateTuple{sfuTuple(PublicationCreate)})
}
// TS 4199: binds reserve and skip results to the exact candidate cursor
func TestBindsReserveAndSkipResultsToExactCandidateCursor(t *testing.T) {
reserve := newController(2, Options{})
addViewer(reserve, A, 2, nil)
reserve.hydrateEdge(A, peerEdge(HOST, "a_active"))
addViewer(reserve, B, 0, nil)
first := must(t, reserve.Reconcile(0).Operation)
eq(t, first.Candidates[0].Tuple.Kind, UpstreamPeer)
eq(t, first.Candidates[0].Tuple.ParentPeerID, HOST)
reserve.SetEffectiveCapacity(HOST, "host_session", 0, nil)
staleReserve := reserve.BeginCurrentCandidate(BeginInput{
Guard: cursorGuard(first),
NowMs: 1,
ConnectionID: "reserved_for_old_cursor",
Reservation: directOverlap("stale_reservation"),
})
eq(t, staleReserve.Accepted, false)
eqLabels(t, staleReserve.Released, "stale_reservation")
rebuilt := must(t, reserve.Snapshot().Operation)
eq(t, rebuilt.Cursor, 0)
eqSlice(t, tuples(rebuilt), []CandidateTuple{peerTuple(A)})
if rebuilt.Current != nil {
t.Fatal("expected no live attempt")
}
skip := newController(2, Options{})
addViewer(skip, A, 2, nil)
skip.hydrateEdge(A, peerEdge(HOST, "a_active"))
addViewer(skip, B, 0, nil)
staleCursor := must(t, skip.Reconcile(0).Operation)
skip.SetEffectiveCapacity(HOST, "host_session", 0, nil)
eq(t, skip.SkipCurrentCandidate(cursorGuard(staleCursor), 1, RejectionStale).Accepted, false)
pending := must(t, skip.Snapshot().Operation)
eq(t, pending.Cursor, 0)
eq(t, pending.Candidates[0].Tuple.Kind, UpstreamPeer)
eq(t, pending.Candidates[0].Tuple.ParentPeerID, A)
}
// TS 4243: keeps only old SFU roots that still anchor peer descendants
func TestKeepsOnlyOldSfuRootsThatStillAnchorPeerDescendants(t *testing.T) {
routes := newController(1, Options{SfuEnabled: true})
addViewer(routes, A, 0, nil)
addViewer(routes, B, 1, nil)
addViewer(routes, C, 0, nil)
addViewer(routes, D, 0, nil)
routes.hydrateHostPublication("publication_1", res("publication_resource_1"), "")
for _, seed := range [][2]string{{A, "subscription_a"}, {B, "subscription_b"}, {D, "subscription_d"}} {
routes.hydrateEdge(seed[0], sfuEdge("publication_1", seed[0]+"_sfu_1", seed[1]))
}
routes.hydrateEdge(C, peerEdge(B, "c_from_b"))
eq(t, routes.InvalidateHostPublication(PublicationGuard{
HostSessionID: "host_session",
RouteRevision: 0,
Generation: "publication_1",
ConnectionID: "publication:publication_1",
}, ms(10)), true)
routes.TouchExternalFacts()
operation := must(t, routes.Reconcile(20).Operation)
eq(t, operation.Candidates[0].Tuple, sfuTuple(PublicationReplace))
expectTransition(t, operation.Candidates[0].EndpointTransition, TransitionOverlap, HOST)
prepared := beginOperation(t, routes, BeginInput{
NowMs: 21,
ConnectionID: "a_sfu_2",
PublicationGeneration: "publication_2",
PublicationConnectionID: "publication_connection_2",
Reservation: sfuCreateOverlap("subscription_a_2", "publication_resource_2", "publication_overlap"),
})
settled := routes.CandidateReady(guardFor(A, prepared.Current.Revision, "a_sfu_2"), 22, nil, CandidateProof{})
eq(t, settled.Accepted, true)
containsLabels(t, settled.Released, "publication_overlap", "subscription_a", "subscription_b", "subscription_d", "publication_resource_1")
eq(t, edgeOf(t, routes, A).PublicationGeneration, "publication_2")
bEdge := edgeOf(t, routes, B)
eq(t, bEdge.Kind, UpstreamSfu)
eq(t, bEdge.PublicationGeneration, "publication_1")
eq(t, bEdge.Usable, false)
eq(t, bEdge.PhysicalActive, false)
cEdge := edgeOf(t, routes, C)
eq(t, cEdge.Kind, UpstreamPeer)
eq(t, cEdge.ParentPeerID, B)
eq(t, hasEdge(routes, D), false)
child := routes.RouteDiagnosticSnapshot(22).Children[1]
eqDeep(t, child.DemandAgeMs, optInt(12))
eq(t, child.FinalRoute, "waiting")
next := must(t, routes.Reconcile(23).Operation)
eq(t, next.ChildPeerID, B)
eq(t, next.Reason, DemandEdgeUnavailable)
}
// TS 4328: rebinds direct and SFU media to replacement sessions
func TestRebindsDirectAndSfuMediaToReplacementSessions(t *testing.T) {
peerRoutes := newController(2, Options{})
addViewer(peerRoutes, A, 1, nil)
addViewer(peerRoutes, B, 0, nil)
peerRoutes.hydrateEdge(A, peerEdge(HOST, "a_direct"))
peerRoutes.hydrateEdge(B, peerEdge(A, "b_direct"))
eqLabels(t, peerRoutes.UpsertParticipant(viewerInput(A, "a_replacement", 1), nil))
aEdge := edgeOf(t, peerRoutes, A)
eq(t, aEdge.ChildSessionID, "a_replacement")
eq(t, aEdge.Usable, true)
eq(t, aEdge.PhysicalActive, true)
eq(t, edgeOf(t, peerRoutes, B).ParentSessionID, "a_replacement")
eqLabels(t, peerRoutes.UpsertParticipant(viewerInput(B, "b_replacement", 0), nil))
bEdge := edgeOf(t, peerRoutes, B)
eq(t, bEdge.ChildSessionID, "b_replacement")
eq(t, bEdge.Usable, true)
eq(t, bEdge.PhysicalActive, true)
sfuRoutes := newController(1, Options{SfuEnabled: true})
addViewer(sfuRoutes, A, 0, nil)
sfuRoutes.hydrateHostPublication("publication", res("publication_resource"), "publication_connection")
sfuRoutes.hydrateEdge(A, sfuEdge("publication", "subscription_connection", "subscription_resource"))
eqLabels(t, sfuRoutes.UpsertParticipant(hostInput("host_replacement", 1), nil))
publication := sfuRoutes.Snapshot().HostPublication
if publication == nil {
t.Fatal("expected a host publication")
}
eq(t, publication.HostSessionID, "host_replacement")
eq(t, publication.Usable, true)
eq(t, publication.PhysicalActive, true)
eqLabels(t, sfuRoutes.UpsertParticipant(viewerInput(A, "a_replacement", 0), nil))
sfuEdgeA := edgeOf(t, sfuRoutes, A)
eq(t, sfuEdgeA.ChildSessionID, "a_replacement")
eq(t, sfuEdgeA.PhysicalActive, true)
}
// TS 4399: retries a bootstrap carrier only when its transition improves
func TestRetriesBootstrapCarrierOnlyWhenTransitionImproves(t *testing.T) {
routes := newController(1, Options{SfuEnabled: true})
addViewer(routes, A, 0, nil)
routes.hydrateEdge(A, peerEdge(HOST, "failed_direct"))
eq(t, routes.InvalidateEdge(edgeGuard(A, "host_session", 0, "failed_direct"), nil), true)
operation := must(t, routes.Reconcile(0).Operation)
prepared := beginOperation(t, routes, BeginInput{
NowMs: 1,
ConnectionID: "failed_sfu",
PublicationGeneration: "failed_publication",
PublicationConnectionID: "failed_publication_connection",
Reservation: sfuCreateOverlap("failed_subscription_resource", "failed_publication_resource", "failed_overlap"),
})
failed := routes.CandidateFailed(CandidateGuard{
ChildPeerID: operation.ChildPeerID,
ChildSessionID: operation.ChildSessionID,
Revision: prepared.Current.Revision,
ConnectionID: "failed_sfu",
}, 2)
eq(t, failed.Accepted, true)
eqSlice(t, failed.FailedPeerIDs, []string{A})
improved := must(t, routes.Reconcile(3).Operation)
eq(t, improved.Candidates[0].Tuple, sfuTuple(PublicationCreate))
expectTransition(t, improved.Candidates[0].EndpointTransition, TransitionNone, HOST)
improvedAttempt := beginOperation(t, routes, BeginInput{
NowMs: 4,
ConnectionID: "improved_sfu",
PublicationGeneration: "improved_publication",
PublicationConnectionID: "improved_publication_connection",
Reservation: sfuCreate("improved_subscription_resource", "improved_publication_resource"),
})
failedAgain := routes.CandidateFailed(CandidateGuard{
ChildPeerID: improved.ChildPeerID,
ChildSessionID: improved.ChildSessionID,
Revision: improvedAttempt.Current.Revision,
ConnectionID: "improved_sfu",
}, 5)
eq(t, failedAgain.Accepted, true)
eqSlice(t, failedAgain.FailedPeerIDs, []string{A})
factVersion := routes.Snapshot().FactVersion
noOperation(t, routes.Reconcile(6).Operation)
eq(t, routes.SetEffectiveCapacity(A, sessionOf(A), 0, nil), true)
eq(t, routes.Snapshot().FactVersion, factVersion)
noOperation(t, routes.Reconcile(7).Operation)
eq(t, routes.InvalidateEdge(edgeGuard(A, "host_session", routes.Snapshot().Revision, "failed_direct"), nil), true)
eq(t, routes.Snapshot().FactVersion, factVersion)
noOperation(t, routes.Reconcile(8).Operation)
addViewer(routes, B, 0, ms(9))
eq(t, must(t, routes.Reconcile(9).Operation).ChildPeerID, B)
// TS: `while (routes.snapshot().operation) skipCandidate(routes, 10);` bounded here so a broken port cannot hang.
for rounds := 0; routes.Snapshot().Operation != nil; rounds++ {
if rounds > 100 {
t.Fatal("skipping never exhausted the operation")
}
skipCandidate(t, routes, 10)
}
noOperation(t, routes.Reconcile(11).Operation)
}
// TS 4472: requires one overlap slot for an SFU-to-Host transition at C=%i
func TestRequiresOneOverlapSlotForSfuToHostTransition(t *testing.T) {
for _, capacity := range []int{1, 2} {
t.Run(fmt.Sprintf("C=%d", capacity), func(t *testing.T) {
routes := newController(capacity, Options{SfuEnabled: true})
addViewer(routes, A, 0, nil)
if capacity == 2 {
addViewer(routes, E, 0, nil)
routes.hydrateEdge(E, peerEdge(HOST, "e_direct"))
}
routes.hydrateHostPublication("publication_1", res("publication_resource_1"), "")
routes.hydrateEdge(A, sfuEdge("publication_1", "a_sfu", "a_subscription"))
routes.InvalidateEdge(edgeGuard(A, "", 0, "a_sfu"), nil)
operation := must(t, routes.Reconcile(0).Operation)
eq(t, operation.Candidates[0].Tuple, peerTuple(HOST))
expectTransition(t, operation.Candidates[0].EndpointTransition, TransitionOverlap, HOST)
expectPanic(t, "endpoint overlap reservation", func() {
beginCandidate(t, routes, BeginInput{NowMs: 1, ConnectionID: "a_host_direct", Reservation: direct()})
})
})
}
}
// TS 4510: uses one bounded gap instead of a fourth Host copy at C=3
func TestUsesOneBoundedGapInsteadOfFourthHostCopyAtCapacityThree(t *testing.T) {
routes := newController(3, Options{SfuEnabled: true})
addViewer(routes, A, 0, nil)
for _, peerID := range []string{E, F} {
addViewer(routes, peerID, 0, nil)
routes.hydrateEdge(peerID, peerEdge(HOST, peerID+"_direct"))
}
routes.hydrateHostPublication("publication_1", res("publication_resource_1"), "")
routes.hydrateEdge(A, sfuEdge("publication_1", "a_sfu", "a_subscription"))
routes.InvalidateEdge(edgeGuard(A, "", 0, "a_sfu"), nil)
operation := must(t, routes.Reconcile(10).Operation)
deadline := operation.DeadlineAtMs
eq(t, operation.Candidates[0].Tuple, peerTuple(HOST))
transition := operation.Candidates[0].EndpointTransition
expectTransition(t, transition, TransitionBoundedGap, HOST)
if transition.Retire == nil {
t.Fatal("expected a retirement")
}
eq(t, transition.Retire.Kind, RetirePublication)
eq(t, transition.Retire.Generation, "publication_1")
gap := routes.RetireCurrentCandidateProducer(cursorGuard(operation), 11)
eq(t, gap.Accepted, true)
eqLabels(t, gap.Released, "a_subscription", "publication_resource_1")
replanned := must(t, gap.Operation)
eq(t, replanned.DeadlineAtMs, deadline)
eq(t, len(replanned.Candidates), 1)
expectTransition(t, replanned.Candidates[0].EndpointTransition, TransitionNone, HOST)
prepared := beginOperation(t, routes, BeginInput{NowMs: 12, ConnectionID: "a_host_direct", Reservation: direct()})
eq(t, routes.CandidateReady(guardFor(A, prepared.Current.Revision, "a_host_direct"), 13, nil, CandidateProof{}).Accepted, true)
if routes.Snapshot().HostPublication != nil {
t.Fatal("expected no host publication")
}
}
// TS 4564: does not reopen consumed none transitions after capacity reduction
func TestDoesNotReopenConsumedNoneTransitionsAfterCapacityReduction(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
addViewer(routes, A, 0, nil)
routes.hydrateHostPublication("publication", res("publication_resource"), "")
routes.hydrateEdge(A, sfuEdge("publication", "a_from_sfu", "a_subscription"))
eq(t, routes.InvalidateEdge(edgeGuard(A, "", 0, "a_from_sfu"), nil), true)
operation := must(t, routes.Reconcile(0).Operation)
eq(t, operation.Candidates[0].Tuple, peerTuple(HOST))
expectTransition(t, operation.Candidates[0].EndpointTransition, TransitionNone, HOST)
eq(t, routes.SetEffectiveCapacity(HOST, "host_session", 1, nil), true)
eqSlice(t, routes.OperationExpired(operation.DeadlineAtMs).FailedPeerIDs, []string{A})
noOperation(t, routes.Reconcile(operation.DeadlineAtMs+1).Operation)
}
// TS 4596: does not unblock an exhausted demand for an unrelated join
func TestDoesNotUnblockExhaustedDemandForUnrelatedJoin(t *testing.T) {
routes := newController(1, Options{})
addViewer(routes, B, 0, nil)
routes.hydrateEdge(B, peerEdge(HOST, "b_from_host"))
addViewer(routes, A, 0, ms(0))
eqSlice(t, routes.Reconcile(0).FailedPeerIDs, []string{A})
eq(t, routes.SetEffectiveCapacity(A, sessionOf(A), 1, nil), true)
noOperation(t, routes.Reconcile(1).Operation)
addViewer(routes, C, 0, ms(2))
eqSlice(t, routes.Reconcile(2).FailedPeerIDs, []string{C})
noOperation(t, routes.Snapshot().Operation)
eq(t, routes.SetEffectiveCapacity(B, sessionOf(B), 1, nil), true)
operation := must(t, routes.Reconcile(3).Operation)
eq(t, operation.ChildPeerID, A)
eqSlice(t, tuples(operation), []CandidateTuple{peerTuple(B)})
}
// TS 4621: does not revive an exact failed Host tuple when a new parent joins
func TestDoesNotReviveExactFailedHostTupleWhenNewParentJoins(t *testing.T) {
routes := newController(1, Options{})
addViewer(routes, A, 0, nil)
routes.hydrateEdge(A, peerEdge(HOST, "failed_direct"))
eq(t, routes.InvalidateEdge(edgeGuard(A, "host_session", 0, "failed_direct"), nil), true)
noOperation(t, routes.Reconcile(0).Operation)
addViewer(routes, B, 1, ms(1))
join := must(t, routes.Reconcile(2).Operation)
eq(t, join.ChildPeerID, B)
eq(t, join.Reason, DemandJoin)
commitCurrent(t, routes, 2, "b_from_host")
retry := must(t, routes.Reconcile(4).Operation)
eq(t, retry.ChildPeerID, A)
eqSlice(t, tuples(retry), []CandidateTuple{peerTuple(B)})
eq(t, hasTuple(retry, func(tuple CandidateTuple) bool { return isPeer(tuple) && tuple.ParentPeerID == HOST }), false)
}
// TS 4660: does not unblock an exhausted Viewer when its candidate set shrinks
func TestDoesNotUnblockExhaustedViewerWhenCandidateSetShrinks(t *testing.T) {
routes := newController(1, Options{})
addViewer(routes, A, 0, nil)
routes.hydrateEdge(A, peerEdge(HOST, "failed_host"))
eq(t, routes.InvalidateEdge(edgeGuard(A, "host_session", 0, "failed_host"), nil), true)
noOperation(t, routes.Reconcile(0).Operation)
addViewer(routes, B, 1, ms(1))
routes.Reconcile(1)
commitCurrent(t, routes, 2, "b_from_host")
candidate := must(t, routes.Reconcile(3).Operation)
attempt := beginOperation(t, routes, BeginInput{NowMs: 4, ConnectionID: "a_from_b", Reservation: direct()})
eqSlice(t, routes.CandidateFailed(guardFor(A, attempt.Current.Revision, "a_from_b"), 5).FailedPeerIDs, []string{A})
eq(t, len(candidate.Candidates), 1)
eq(t, routes.DisconnectSession(B, sessionOf(B)), true)
noOperation(t, routes.Reconcile(6).Operation)
}
// TS 4694: treats a replacement child session as a new exact opportunity
func TestTreatsReplacementChildSessionAsNewExactOpportunity(t *testing.T) {
routes := newController(1, Options{})
addViewer(routes, A, 0, nil)
routes.hydrateEdge(A, peerEdge(HOST, "failed_host"))
eq(t, routes.InvalidateEdge(edgeGuard(A, "host_session", 0, "failed_host"), nil), true)
noOperation(t, routes.Reconcile(0).Operation)
routes.UpsertParticipant(viewerInput(A, "replacement_session", 0), nil)
operation := must(t, routes.Reconcile(1).Operation)
eq(t, operation.ChildPeerID, A)
eq(t, operation.ChildSessionID, "replacement_session")
eqSlice(t, tuples(operation), []CandidateTuple{peerTuple(HOST)})
}
// TS 4724: does not retry a failed bootstrap carrier when another carrier leaves
func TestDoesNotRetryFailedBootstrapCarrierWhenAnotherCarrierLeaves(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
for _, seed := range [][2]string{{B, "b_from_host"}, {C, "c_from_host"}} {
addViewer(routes, seed[0], 1, nil)
routes.hydrateEdge(seed[0], peerEdge(HOST, seed[1]))
}
addViewer(routes, A, 0, ms(0))
acquisition := must(t, routes.Reconcile(0).Operation)
beginCandidate(t, routes, BeginInput{NowMs: 1, ConnectionID: "a_head_start", Reservation: direct()})
routes.OperationExpired(acquisition.WakeAtMs)
bootstrap := must(t, routes.Reconcile(acquisition.WakeAtMs+1).Operation)
eq(t, bootstrap.Reason, DemandSfuBootstrap)
failedCarrier := bootstrap.ChildPeerID
otherCarrier := B
if failedCarrier == B {
otherCarrier = C
}
attempt := beginOperation(t, routes, BeginInput{
NowMs: acquisition.WakeAtMs + 2,
ConnectionID: "failed_bootstrap",
PublicationGeneration: "failed_publication",
PublicationConnectionID: "failed_publication_connection",
Reservation: sfuCreateOverlap("failed_subscription", "failed_publication_resource", "failed_overlap"),
})
eq(t, routes.CandidateFailed(guardFor(failedCarrier, attempt.Current.Revision, "failed_bootstrap"), acquisition.WakeAtMs+3).Accepted, true)
eq(t, routes.DisconnectSession(otherCarrier, sessionOf(otherCarrier)), true)
next := routes.Reconcile(acquisition.WakeAtMs + 4).Operation
if next != nil && next.Reason == DemandSfuBootstrap && next.ChildPeerID == failedCarrier {
t.Fatalf("retried the failed bootstrap carrier %q", failedCarrier)
}
}
// TS 4775: does not bootstrap a new SFU generation after the last reuse opportunity expires
func TestDoesNotBootstrapNewSfuGenerationAfterLastReuseOpportunityExpires(t *testing.T) {
routes, nowMs := retireLastSfuPublicationAfterReuseDeadline(t)
reconciled := routes.Reconcile(nowMs)
noOperation(t, reconciled.Operation)
eqSlice(t, reconciled.FailedPeerIDs, []string{})
}
// TS 4783: reopens the exhausted SFU opportunity for a new %s authority
func TestReopensExhaustedSfuOpportunityForNewAuthority(t *testing.T) {
for _, authority := range []string{"child-session", "host-session", "external-resource"} {
t.Run(authority, func(t *testing.T) {
routes, nowMs := retireLastSfuPublicationAfterReuseDeadline(t)
switch authority {
case "child-session":
routes.UpsertParticipant(viewerInput(A, "replacement_child_session", 0), nil)
case "host-session":
routes.UpsertParticipant(hostInput("replacement_host_session", 2), nil)
default:
routes.TouchExternalFacts()
}
operation := must(t, routes.Reconcile(nowMs).Operation)
eq(t, operation.DemandPeerID, A)
eq(t, operation.Reason, DemandSfuBootstrap)
eqSlice(t, tuples(operation), []CandidateTuple{sfuTuple(PublicationCreate)})
})
}
}
// TS 4817: consumes the canonical SFU opportunity when an active edge fails
func TestConsumesCanonicalSfuOpportunityWhenActiveEdgeFails(t *testing.T) {
routes := invalidateActiveSfuEdge(t)
// TS: `operation?.candidates.some(sfu)` toBe(false) needs an operation.
operation := must(t, routes.Reconcile(0).Operation)
eq(t, hasTuple(operation, isSfu), false)
}
// TS 4826: reopens an active-edge SFU opportunity for a new %s authority
func TestReopensActiveEdgeSfuOpportunityForNewAuthority(t *testing.T) {
for _, authority := range []string{"child-session", "host-session", "external-resource"} {
t.Run(authority, func(t *testing.T) {
routes := invalidateActiveSfuEdge(t)
switch authority {
case "child-session":
routes.UpsertParticipant(viewerInput(A, "active_edge_child_replacement", 0), nil)
case "host-session":
routes.UpsertParticipant(hostInput("active_edge_host_replacement", 1), nil)
default:
routes.TouchExternalFacts()
}
operation := must(t, routes.Reconcile(0).Operation)
eq(t, hasTuple(operation, func(tuple CandidateTuple) bool {
return isSfu(tuple) && tuple.Publication == PublicationReuse
}), true)
})
}
}
// TS 4860: does not turn a failed same-rank SFU replace into a create retry
func TestDoesNotTurnFailedSameRankSfuReplaceIntoCreateRetry(t *testing.T) {
routes := newController(2, Options{SfuEnabled: true})
routes.hydrateHostPublication("failed_publication", res("failed_publication_resource"), "")
eq(t, routes.InvalidateHostPublication(PublicationGuard{
HostSessionID: "host_session",
RouteRevision: routes.Snapshot().Revision,
Generation: "failed_publication",
ConnectionID: "publication:failed_publication",
}, nil), true)
addViewer(routes, A, 0, nil)
operation := must(t, routes.Reconcile(0).Operation)
eq(t, operation.Candidates[0].Tuple.Kind, UpstreamPeer)
skipCandidate(t, routes, 1)
replaceCandidate := must(t, routes.Snapshot().Operation).Candidates[1]
eq(t, replaceCandidate.Tuple, sfuTuple(PublicationReplace))
expectTransition(t, replaceCandidate.EndpointTransition, TransitionNone, HOST)
replace := beginOperation(t, routes, BeginInput{
NowMs: 2,
ConnectionID: "failed_replace",
PublicationGeneration: "replacement_publication",
PublicationConnectionID: "replacement_ingress",
Reservation: sfuCreate("replacement_subscription", "replacement_publication_resource"),
})
eqSlice(t, routes.CandidateFailed(guardFor(A, replace.Current.Revision, "failed_replace"), 3).FailedPeerIDs, []string{A})
addViewer(routes, B, 0, nil)
eq(t, routes.ConfirmDeparture(B, nil), true)
reconciled := routes.Reconcile(4)
if routes.Snapshot().HostPublication != nil {
t.Fatal("expected no host publication")
}
noOperation(t, reconciled.Operation)
eqSlice(t, reconciled.FailedPeerIDs, []string{})
}
// TS 4913: reopens only SFU opportunities for a new external fact
func TestReopensOnlySfuOpportunitiesForNewExternalFact(t *testing.T) {
routes := newController(1, Options{SfuEnabled: true})
addViewer(routes, A, 0, ms(0))
routes.Reconcile(0)
directAttempt := beginOperation(t, routes, BeginInput{NowMs: 1, ConnectionID: "failed_direct", Reservation: direct()})
eq(t, routes.CandidateFailed(guardFor(A, directAttempt.Current.Revision, "failed_direct"), 2).Accepted, true)
sfuAttempt := beginOperation(t, routes, BeginInput{
NowMs: 3,
ConnectionID: "failed_sfu",
PublicationGeneration: "failed_publication",
PublicationConnectionID: "failed_publication_connection",
Reservation: sfuCreate("failed_subscription", "failed_publication_resource"),
})
eq(t, routes.CandidateFailed(guardFor(A, sfuAttempt.Current.Revision, "failed_sfu"), 4).Accepted, true)
noOperation(t, routes.Snapshot().Operation)
noOperation(t, routes.Reconcile(5).Operation)
routes.TouchExternalFacts()
retried := must(t, routes.Reconcile(6).Operation)
eq(t, len(retried.Candidates), 1)
eq(t, retried.Candidates[0].Tuple, sfuTuple(PublicationCreate))
expectTransition(t, retried.Candidates[0].EndpointTransition, TransitionNone, HOST)
eq(t, hasTuple(retried, isPeer), false)
}
// TS 4963: fails all waiting demands without cutting capacity-three Host children
func TestFailsAllWaitingDemandsWithoutCuttingCapacityThreeHostChildren(t *testing.T) {
routes := newController(3, Options{SfuEnabled: true})
seeds := [][2]string{{A, "a_direct"}, {B, "b_direct"}, {C, "c_direct"}}
for _, seed := range seeds {
addViewer(routes, seed[0], 0, nil)
routes.hydrateEdge(seed[0], peerEdge(HOST, seed[1]))
}
addViewer(routes, D, 0, nil)
addViewer(routes, E, 0, nil)
reconciled := routes.Reconcile(0)
noOperation(t, reconciled.Operation)
eqSlice(t, reconciled.FailedPeerIDs, []string{D, E})
for _, seed := range seeds {
edge := edgeOf(t, routes, seed[0])
eq(t, edge.ConnectionID, seed[1])
eq(t, edge.Usable, true)
eq(t, edge.PhysicalActive, true)
}
}
// TS 4991: does not promote when synchronous admission commit fails
func TestDoesNotPromoteWhenSynchronousAdmissionCommitFails(t *testing.T) {
routes := newController(1, Options{})
addViewer(routes, A, 0, nil)
operation := must(t, routes.Reconcile(0).Operation)
prepared := beginOperation(t, routes, BeginInput{NowMs: 1, ConnectionID: "candidate_connection", Reservation: direct()})
settled := routes.CandidateReady(CandidateGuard{
ChildPeerID: operation.ChildPeerID,
ChildSessionID: operation.ChildSessionID,
Revision: prepared.Current.Revision,
ConnectionID: "candidate_connection",
}, 2, func(CandidateReservation) bool { return false }, CandidateProof{})
eq(t, settled.Accepted, false)
if settled.ActiveRevision <= prepared.Current.Revision {
t.Fatalf("activeRevision %d did not advance past %d", settled.ActiveRevision, prepared.Current.Revision)
}
eq(t, hasEdge(routes, A), false)
}
// --- describe("bounded NAT candidate acquisition") -------------------------
// TS 5033: runs three real generations on %s, then exhausts until a new session
func TestRunsThreeRealGenerationsThenExhaustsUntilNewSession(t *testing.T) {
for _, failure := range []string{"failed", "timeout"} {
t.Run(failure, func(t *testing.T) {
routes := newController(2, Options{NatPredictionEnabled: true, OperationTimeoutMs: 20_000})
addViewer(routes, A, 0, nil)
nowMs := int64(0)
var previous *CandidateGuard
for attempt := 1; attempt <= 3; attempt++ {
operation, prepared, guard := preparePeer(t, routes, nowMs, fmt.Sprintf("nat_%d", attempt))
eqDeep(t, prepared.Current.ConnectionAttempt, &ConnectionAttemptProgress{Current: attempt, Total: 3})
eq(t, operation.DeadlineAtMs-nowMs, 20_000)
eq(t, prepared.WakeAtMs, operation.DeadlineAtMs)
if previous != nil {
eq(t, routes.CandidateFailed(*previous, nowMs).Accepted, false)
}
eq(t, must(t, routes.Snapshot().Operation).Current.ConnectionID, fmt.Sprintf("nat_%d", attempt))
var result SettleResult
if failure == "timeout" {
nowMs = operation.DeadlineAtMs
result = routes.OperationExpired(nowMs)
} else {
nowMs++
result = routes.CandidateFailed(guard, nowMs)
}
if attempt == 3 {
eqSlice(t, result.FailedPeerIDs, []string{A})
} else {
eqSlice(t, result.FailedPeerIDs, []string{})
}
eq(t, routes.CandidateFailed(guard, nowMs).Accepted, false)
previous = &guard
nowMs++
}
noOperation(t, routes.Reconcile(nowMs).Operation)
routes.UpsertParticipant(viewerInput(A, "replacement_session", 0), ms(nowMs))
_, prepared, guard := preparePeer(t, routes, nowMs+1, "new_session")
eq(t, attemptCurrent(t, prepared), 1)
eq(t, routes.CandidateReady(guard, nowMs+2, nil, CandidateProof{}).Committed, true)
})
}
}
// TS 5067: gives waiting viewers their first opportunity before another viewer retries
func TestGivesWaitingViewersFirstOpportunityBeforeAnotherViewerRetries(t *testing.T) {
routes := newController(2, Options{NatPredictionEnabled: true})
addViewer(routes, A, 0, nil)
addViewer(routes, B, 0, nil)
for index, childPeerID := range []string{A, B, A, B, A, B} {
_, prepared, guard := preparePeer(t, routes, int64(index*2), fmt.Sprintf("fair_%d", index))
eq(t, prepared.ChildPeerID, childPeerID)
eq(t, attemptCurrent(t, prepared), index/2+1)
routes.CandidateFailed(guard, int64(index*2+1))
}
noOperation(t, routes.Reconcile(20).Operation)
}
// TS 5080: tries a newly usable parent before spending another attempt on an old parent
func TestTriesNewlyUsableParentBeforeSpendingAnotherAttemptOnOldParent(t *testing.T) {
routes := newController(2, Options{NatPredictionEnabled: true})
addViewer(routes, B, 2, nil)
routes.hydrateEdge(B, peerEdge(HOST, "existing_b"))
addViewer(routes, A, 0, nil)
_, firstPrepared, firstGuard := preparePeer(t, routes, 0, "first_host")
eq(t, firstPrepared.Current.Tuple.ParentPeerID, HOST)
routes.CandidateFailed(firstGuard, 1)
_, secondPrepared, secondGuard := preparePeer(t, routes, 2, "first_b")
eq(t, secondPrepared.Current.Tuple.ParentPeerID, B)
routes.CandidateFailed(secondGuard, 3)
addViewer(routes, C, 2, ms(4))
routes.hydrateEdge(C, peerEdge(HOST, "existing_c"))
_, thirdPrepared, _ := preparePeer(t, routes, 5, "first_c")
eq(t, thirdPrepared.Current.Tuple.ParentPeerID, C)
eq(t, attemptCurrent(t, thirdPrepared), 1)
}
// TS 5098: keeps SFU's foreground timing and shares the attempt budget with round-robin direct continuations
func TestKeepsSfuForegroundTimingAndSharesAttemptBudgetWithDirectContinuations(t *testing.T) {
routes := newController(2, Options{NatPredictionEnabled: true, SfuEnabled: true, OperationTimeoutMs: 20_000})
nowMs := int64(0)
for _, childPeerID := range []string{A, B} {
addViewer(routes, childPeerID, 0, ms(nowMs))
_, directPrepared, _ := preparePeer(t, routes, nowMs, childPeerID+"_first")
eq(t, directPrepared.WakeAtMs, nowMs+5_000)
nowMs += 5_000
eqSlice(t, routes.OperationExpired(nowMs).FailedPeerIDs, []string{})
reservation := sfuReuse("subscription_b")
if childPeerID == A {
reservation = sfuCreate("subscription_a", "publication")
}
sfu := beginOperation(t, routes, BeginInput{
NowMs: nowMs,
ConnectionID: childPeerID + "_sfu",
PublicationGeneration: "sfu_generation",
PublicationConnectionID: "sfu_publisher",
Reservation: reservation,
})
if sfu.Current.ConnectionAttempt != nil {
t.Fatal("SFU attempts carry no NAT connection attempt")
}
nowMs++
routes.CandidateReady(CandidateGuard{
ChildPeerID: childPeerID,
ChildSessionID: sfu.ChildSessionID,
Revision: sfu.Current.Revision,
ConnectionID: childPeerID + "_sfu",
}, nowMs, nil, CandidateProof{})
nowMs++
}
for index, childPeerID := range []string{A, B, A, B} {
_, retry, guard := preparePeer(t, routes, nowMs, fmt.Sprintf("background_%d", index))
eq(t, retry.ChildPeerID, childPeerID)
eq(t, retry.Reason, DemandDirectConvergence)
eqDeep(t, retry.Current.ConnectionAttempt, &ConnectionAttemptProgress{Current: index/2 + 2, Total: 3})
eq(t, retry.WakeAtMs-nowMs, 20_000)
nowMs++
eqSlice(t, routes.CandidateFailed(guard, nowMs).FailedPeerIDs, []string{})
edge := edgeOf(t, routes, childPeerID)
eq(t, edge.Kind, UpstreamSfu)
eq(t, edge.Usable, true)
nowMs++
}
noOperation(t, routes.Reconcile(nowMs).Operation)
}
// TS 5139: does not retry without NAT policy or when no candidate was created
func TestDoesNotRetryWithoutNatPolicyOrWhenNoCandidateWasCreated(t *testing.T) {
ordinary := newController(2, Options{})
addViewer(ordinary, A, 0, nil)
_, firstPrepared, firstGuard := preparePeer(t, ordinary, 0, "ordinary")
if firstPrepared.Current.ConnectionAttempt != nil {
t.Fatal("expected no connection attempt without NAT prediction")
}
eqSlice(t, ordinary.CandidateFailed(firstGuard, 1).FailedPeerIDs, []string{A})
noOperation(t, ordinary.Reconcile(2).Operation)
routes := newController(2, Options{NatPredictionEnabled: true})
addViewer(routes, A, 0, nil)
operation := must(t, routes.Reconcile(0).Operation)
routes.SkipCurrentCandidate(cursorGuard(operation), 1, RejectionCandidateFailed)
noOperation(t, routes.Reconcile(2).Operation)
}