Files
Piik/internal/server/signal/router_test.go
T
AAA 083c9273a7 fix(routing): retain connected candidates during delayed viewer feedback
When Viewer progress and first-frame confirmations are delayed, an already connected P2P candidate can be discarded at the five-second scheduling boundary if another candidate is available. This can turn a working initial connection into another connection attempt.

Host and relay parents now report transport progress through the existing room router. The router checks the exact participant/session, revision and connection; only the candidate Viewer can confirm first-frame readiness. The total operation deadline and the routing algorithm are unchanged. Browser and Native senders use their existing prepared-child owners, and discarded preparations cannot report progress.

The loopback benchmark now identifies Peer and SFU connections from room signaling, excluding local codec probes and encoding-pool transports from media-edge counts. Its optional old Viewer-only quality canary remains unsuitable for the current sender-owned quality contract and is recorded in TODO.

Validation:

- `npm run check`: type checks, 1,000 tests and production Web build passed.
- `npm run check:go-core`: formatting, Go tests, vet and supported Windows/Linux builds passed; Darwin capture remains a native-runner check.
- Documentation and project-state checks passed.
- Released v1.5.0 versus this candidate, delaying Viewer transport/ready feedback by six seconds: the old candidate connection closed after the scheduling boundary despite 79 decoded frames; the fixed connection stayed intact. Browser Host, Native Host and relay-parent paths were exercised, with another Viewer continuing to play.
- Three-Viewer loopback topology and relay-departure recovery passed; Host and relay copy limits remained valid.
- Actual App public-link signaling, with isolated 1/8/20-second packet loss on one Browser Viewer: playback recovered in all three cases and the control Viewer continued. Every test process, port and isolated profile was cleaned up.

The original reporter's browser/network and paired diagnostics are unavailable. These bounded tests establish the repaired mechanism, not the cause of every brief connection or a representative NAT success rate.

## Release notes

### 中文

这是一次连接稳定性修复,改善进入房间时的 P2P 连接过程。

- 修复观众的连接确认反馈较慢时,已接通的线路可能被提前拆除,造成重复连接或延迟出画的问题。修复覆盖 App 分享、浏览器分享和观众中继。
- 使用 App 公网邀请或本地模式时,房主更新 App 后重新分享即可。使用已部署站点时,请管理员更新 Server,并刷新页面。

### English

This patch improves P2P connection stability when joining a room.

- Fixed an established connection being discarded prematurely when Viewer confirmations arrive late, causing repeated connection attempts or delayed playback. The fix covers App sharing, browser sharing and Viewer relays.
- For App public invitations or Local mode, the Host should update the App and start sharing again. For a deployed site, update the Server and refresh the page.
2026-09-17 15:18:08 +08:00

2212 lines
81 KiB
Go

package signal
// Ported from tests/hybrid-media-router.test.ts (baseline b20fd88). The
// harness mirrors harness(...) there: a memory room.Store, a real
// sfu.Admission, sfutest.FakeMedia behind a thin spy (the vi.spyOn
// cases), a fake token issuer returning "token-<peerId>", recorded hooks and
// the shared mutex every router call and every assertion takes.
//
// Timing model. Node ran each TS test segment between two awaits as one
// synchronous tick with the router's continuations (the pump, token issues,
// drains) suspended; they ran at the next microtask checkpoint. Here every
// such segment is one h.locked block, so the router's goroutines (which take
// the same mutex) cannot interleave with it, and the TS await points map to
// waitFor/advance, which let those goroutines settle (router.inflight == 0).
// The seven fake-timer cases reproduce vitest 4.1 exactly: vi.waitFor ticks
// the clock 50 ms before every check and gives up after 20 checks,
// advanceTimersByTimeAsync fires due timers in order at their due time and
// flushes between them, and setSystemTime jumps the clock while shifting
// every pending timer by the same amount.
import (
"context"
"errors"
"fmt"
"io"
"log/slog"
"reflect"
"runtime"
"slices"
"sort"
"strings"
"sync"
"testing"
"time"
"github.com/TNTcraftHIM/Piik/internal/server/ordered"
"github.com/TNTcraftHIM/Piik/internal/server/protocol"
"github.com/TNTcraftHIM/Piik/internal/server/room"
"github.com/TNTcraftHIM/Piik/internal/server/route"
"github.com/TNTcraftHIM/Piik/internal/server/sfu"
"github.com/TNTcraftHIM/Piik/internal/server/sfu/sfutest"
)
const testShareGeneration = "share_generation_12345678"
// ---------------------------------------------------------------------------
// harness
// ---------------------------------------------------------------------------
type routerHarnessOptions struct {
capacity int
withSfu bool
prepareTimeoutMs int64
sfuIngressCapacity int
fakeTimers bool
}
type routerHarness struct {
t *testing.T
mu sync.Mutex
store *room.Store
router *router
admission *sfu.Admission
media *spyMedia
clock *manualClock
sent map[string][]protocol.ServerMessage
routesChanged []string
}
func newRouterHarness(t *testing.T, options routerHarnessOptions) *routerHarness {
t.Helper()
h := &routerHarness{t: t, sent: map[string][]protocol.ServerMessage{}}
storeOptions := room.Options{MaxRooms: 4, MaxViewersPerRoom: 20}
var now func() int64
var afterFunc func(time.Duration, func()) func() bool
if options.fakeTimers {
h.clock = &manualClock{settle: h.settle}
now = h.clock.now
afterFunc = h.clock.afterFunc
}
store, err := room.New(storeOptions)
if err != nil {
t.Fatalf("room.New: %v", err)
}
h.store = store
var fallback *sfuFallback
if options.withSfu {
ingress := options.sfuIngressCapacity
if ingress == 0 {
ingress = 2
}
h.admission = sfu.NewAdmission(sfu.AdmissionOptions{IngressCapacity: ingress, EgressCapacity: 20})
h.media = &spyMedia{FakeMedia: sfutest.New()}
fallback = &sfuFallback{
admission: h.admission,
media: h.media,
prepareTimeoutMs: options.prepareTimeoutMs,
}
}
h.router = newRouter(routerOptions{
mu: &h.mu,
store: store,
endpointMediaCopyCapacity: options.capacity,
sfu: fallback,
hooks: routerHooks{
sendToSession: func(sessionID string, message protocol.ServerMessage) {
h.sent[sessionID] = append(h.sent[sessionID], message)
},
shareGeneration: func(string) string { return testShareGeneration },
routesChanged: func(roomID string) { h.routesChanged = append(h.routesChanged, roomID) },
},
now: now,
afterFunc: afterFunc,
})
t.Cleanup(func() { _ = h.close() })
return h
}
// locked runs fn as one synchronous TS segment.
func (h *routerHarness) locked(fn func()) {
h.mu.Lock()
defer h.mu.Unlock()
fn()
}
// settle waits for the router's pending continuations (the TS microtask
// backlog) to finish.
func (h *routerHarness) settle() {
h.t.Helper()
deadline := time.Now().Add(5 * time.Second)
for {
var idle bool
h.locked(func() { idle = h.router.inflight == 0 })
if idle {
return
}
if time.Now().After(deadline) {
h.t.Fatal("timed out waiting for the router to settle")
}
time.Sleep(time.Millisecond)
}
}
// close is the signaling server's `await this.hybridMediaRouter.close()`:
// entered with mu held, released only around the drain waits.
func (h *routerHarness) close() error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var err error
h.locked(func() { err = h.router.close(ctx) })
return err
}
func (h *routerHarness) createRoom() room.CreatedRoom {
h.t.Helper()
var created room.CreatedRoom
var err error
h.locked(func() {
err = h.store.BeginCreateRoom()
if err == nil {
created, err = h.store.CreateRoom(protocol.CodeEntryOpen, nil, nil, "")
}
})
if err != nil {
h.t.Fatalf("createRoom: %v", err)
}
return created
}
// connectHost ports connectHost: the store side only, like the TS helper.
func (h *routerHarness) connectHost(created room.CreatedRoom, policy *protocol.RoutePolicy, clientID, sessionID string) authenticatedRouteParticipant {
h.t.Helper()
if clientID == "" {
clientID = "host_client_12345678"
}
if sessionID == "" {
sessionID = "host_session_12345678"
}
var connected room.ConnectedParticipant
var err error
h.locked(func() {
connected, err = h.store.ConnectParticipant(room.ConnectParticipantInput{
RoomID: created.RoomID,
Role: protocol.RoleHost,
Token: created.HostToken,
ClientID: clientID,
SessionID: sessionID,
})
})
if err != nil {
h.t.Fatalf("connect host: %v", err)
}
return authenticatedRouteParticipant{
roomID: created.RoomID,
role: protocol.RoleHost,
peerID: connected.PeerID,
sessionID: sessionID,
routePolicy: policy,
}
}
// connectViewer ports connectViewer: the store side only.
func (h *routerHarness) connectViewer(created room.CreatedRoom, suffix string) authenticatedRouteParticipant {
h.t.Helper()
sessionID := "viewer_session_" + suffix + "_12345678"
var connected room.ConnectedParticipant
var err error
h.locked(func() {
connected, err = h.store.ConnectParticipant(room.ConnectParticipantInput{
RoomID: created.RoomID,
Role: protocol.RoleViewer,
ViewerGrant: created.ViewerGrant,
ClientID: "viewer_client_" + suffix + "_12345678",
SessionID: sessionID,
})
})
if err != nil {
h.t.Fatalf("connect viewer %s: %v", suffix, err)
}
return authenticatedRouteParticipant{
roomID: created.RoomID,
role: protocol.RoleViewer,
peerID: connected.PeerID,
sessionID: sessionID,
}
}
// The do* helpers are the TS router calls; they expect the harness lock to be
// held by an enclosing locked block. The lock-taking wrappers below serve the
// single-call segments.
func (h *routerHarness) doComplete(participant authenticatedRouteParticipant) hybridAuthenticationState {
state := h.router.connectParticipant(participant)
h.router.completeAuthentication(participant, state)
return state
}
func readyMessage(revision int64) protocol.RouteReadyMessage {
return protocol.RouteReadyMessage{Type: "route-ready", Revision: protocol.Int(revision), Phase: "prepare"}
}
func (h *routerHarness) doReady(participant authenticatedRouteParticipant, revision int64) {
h.router.handleRouteReady(participant, readyMessage(revision))
}
func (h *routerHarness) doReadyApproved(participant authenticatedRouteParticipant, revision int64) {
message := readyMessage(revision)
message.QualityApproved = true
h.router.handleRouteReady(participant, message)
}
func routeFailedMessage(revision int64, phase, connectionID string) protocol.RouteFailedMessage {
message := protocol.RouteFailedMessage{Type: "route-failed", Revision: protocol.Int(revision), Phase: phase}
if connectionID != "" {
message.ConnectionID = &connectionID
}
return message
}
func (h *routerHarness) doFailed(participant authenticatedRouteParticipant, message protocol.RouteFailedMessage) {
h.router.handleRouteFailed(participant, message)
}
func (h *routerHarness) doRelay(participant authenticatedRouteParticipant, downstreamEdges int) {
h.router.setViewerRelayCapacity(participant, downstreamEdges)
}
// doDisconnect is the store.disconnectParticipant + router.disconnectParticipant pair.
func (h *routerHarness) doDisconnect(participant authenticatedRouteParticipant) {
h.t.Helper()
if _, err := h.store.DisconnectParticipant(participant.roomID, participant.peerID, participant.sessionID); err != nil {
h.t.Fatalf("DisconnectParticipant: %v", err)
}
h.router.disconnectParticipant(participant.roomID, participant.peerID, participant.sessionID)
}
func (h *routerHarness) doEdge(roomID, peerID string) (activeViewerMediaEdge, bool) {
return h.router.resolveActiveViewerMediaEdge(roomID, peerID)
}
func (h *routerHarness) doMustEdge(roomID, peerID string) activeViewerMediaEdge {
h.t.Helper()
edge, ok := h.doEdge(roomID, peerID)
if !ok {
h.t.Fatal("expected an active viewer media edge")
}
return edge
}
func (h *routerHarness) complete(participant authenticatedRouteParticipant) hybridAuthenticationState {
var state hybridAuthenticationState
h.locked(func() { state = h.doComplete(participant) })
return state
}
func (h *routerHarness) routeReady(participant authenticatedRouteParticipant, revision int64) {
h.locked(func() { h.doReady(participant, revision) })
}
func (h *routerHarness) routeFailed(participant authenticatedRouteParticipant, message protocol.RouteFailedMessage) {
h.locked(func() { h.doFailed(participant, message) })
}
func (h *routerHarness) disconnect(participant authenticatedRouteParticipant) {
h.t.Helper()
h.locked(func() { h.doDisconnect(participant) })
}
func (h *routerHarness) activeEdge(roomID, peerID string) (activeViewerMediaEdge, bool) {
var edge activeViewerMediaEdge
var ok bool
h.locked(func() { edge, ok = h.doEdge(roomID, peerID) })
return edge, ok
}
func (h *routerHarness) mustActiveEdge(roomID, peerID string) activeViewerMediaEdge {
h.t.Helper()
edge, ok := h.activeEdge(roomID, peerID)
if !ok {
h.t.Fatal("expected an active viewer media edge")
}
return edge
}
func (h *routerHarness) messages(sessionID string) []protocol.ServerMessage {
var copied []protocol.ServerMessage
h.locked(func() { copied = slices.Clone(h.sent[sessionID]) })
return copied
}
func (h *routerHarness) allMessages() []protocol.ServerMessage {
var all []protocol.ServerMessage
h.locked(func() {
for _, messages := range h.sent {
all = append(all, messages...)
}
})
return all
}
// preparedFor ports preparedFor: the last phase "prepare" route-update.
func (h *routerHarness) preparedFor(sessionID string) (protocol.RouteUpdatePrepareMessage, bool) {
return lastPrepare(h.messages(sessionID), "")
}
// ownPrepare is the test-local ownPrepare: the last prepare whose candidate
// child is the participant.
func (h *routerHarness) ownPrepare(participant authenticatedRouteParticipant) (protocol.RouteUpdatePrepareMessage, bool) {
return lastPrepare(h.messages(participant.sessionID), participant.peerID)
}
func lastPrepare(messages []protocol.ServerMessage, childPeerID string) (protocol.RouteUpdatePrepareMessage, bool) {
for index := len(messages) - 1; index >= 0; index-- {
if prepared, ok := messages[index].(protocol.RouteUpdatePrepareMessage); ok &&
(childPeerID == "" || prepared.Candidate.ChildPeerID == childPeerID) {
return prepared, true
}
}
return protocol.RouteUpdatePrepareMessage{}, false
}
// activeAfter ports activeAfter.
func (h *routerHarness) activeAfter(sessionID string, revision int64) (protocol.RouteUpdateActiveMessage, bool) {
messages := h.messages(sessionID)
for index := len(messages) - 1; index >= 0; index-- {
if active, ok := messages[index].(protocol.RouteUpdateActiveMessage); ok && int64(active.Revision) > revision {
return active, true
}
}
return protocol.RouteUpdateActiveMessage{}, false
}
func (h *routerHarness) lastRouteStatus(sessionID string) (protocol.RouteStatusMessage, bool) {
messages := h.messages(sessionID)
for index := len(messages) - 1; index >= 0; index-- {
if status, ok := messages[index].(protocol.RouteStatusMessage); ok {
return status, true
}
}
return protocol.RouteStatusMessage{}, false
}
func (h *routerHarness) hasRouteStatus(sessionID, state string) bool {
return h.countRouteStatus(sessionID, state) > 0
}
func (h *routerHarness) countRouteStatus(sessionID, state string) int {
count := 0
for _, message := range h.messages(sessionID) {
if status, ok := message.(protocol.RouteStatusMessage); ok && status.State == state {
count++
}
}
return count
}
func sfuConfigCount(messages []protocol.ServerMessage) int {
count := 0
for _, message := range messages {
if _, ok := message.(protocol.SfuConfigMessage); ok {
count++
}
}
return count
}
// waitFor ports vi.waitFor. With fake timers every check first ticks the
// clock by the 50 ms poll interval and the 1 s timeout allows 20 checks, as
// vitest 4.1 does; with real timers it polls.
func (h *routerHarness) waitFor(description string, condition func() bool) {
h.t.Helper()
if h.clock != nil {
for check := 0; check < 20; check++ {
h.clock.advance(50)
if condition() {
return
}
}
h.t.Fatalf("timed out waiting for %s", description)
}
deadline := time.Now().Add(5 * time.Second)
for {
if condition() {
return
}
if time.Now().After(deadline) {
h.t.Fatalf("timed out waiting for %s", description)
}
time.Sleep(time.Millisecond)
}
}
func (h *routerHarness) waitPrepared(sessionID string) protocol.RouteUpdatePrepareMessage {
h.t.Helper()
return h.waitPreparedMatching(sessionID, "a prepare", func(protocol.RouteUpdatePrepareMessage) bool { return true })
}
func (h *routerHarness) waitPreparedTransport(sessionID, transport string) protocol.RouteUpdatePrepareMessage {
h.t.Helper()
return h.waitPreparedMatching(sessionID, transport+" prepare", func(prepared protocol.RouteUpdatePrepareMessage) bool {
return prepared.Candidate.Transport == transport
})
}
func (h *routerHarness) waitPreparedUpstream(sessionID string, upstream protocol.MediaRouteUpstream) protocol.RouteUpdatePrepareMessage {
h.t.Helper()
return h.waitPreparedMatching(sessionID, "a prepare with upstream "+upstream.Kind, func(prepared protocol.RouteUpdatePrepareMessage) bool {
return prepared.Assignment.Upstream == upstream
})
}
func (h *routerHarness) waitPreparedAbove(sessionID string, revision int64) protocol.RouteUpdatePrepareMessage {
h.t.Helper()
return h.waitPreparedMatching(sessionID, "a newer prepare", func(prepared protocol.RouteUpdatePrepareMessage) bool {
return int64(prepared.Revision) > revision
})
}
func (h *routerHarness) waitPreparedMatching(sessionID, description string, matches func(protocol.RouteUpdatePrepareMessage) bool) protocol.RouteUpdatePrepareMessage {
h.t.Helper()
var prepared protocol.RouteUpdatePrepareMessage
h.waitFor(description+" for "+sessionID, func() bool {
candidate, ok := h.preparedFor(sessionID)
if ok && matches(candidate) {
prepared = candidate
return true
}
return false
})
return prepared
}
func (h *routerHarness) waitFailed(sessionID string) {
h.t.Helper()
h.waitFor("route-exhausted for "+sessionID, func() bool {
status, ok := h.lastRouteStatus(sessionID)
return ok && status.State == "failed" && status.Reason == "route-exhausted"
})
}
func (h *routerHarness) waitUsage(want sfu.Usage) {
h.t.Helper()
h.waitFor("SFU usage to reach the expected value", func() bool { return h.usage() == want })
}
func (h *routerHarness) diagnostic(roomID string) protocol.RouteDiagnosticSnapshot {
var snapshot protocol.RouteDiagnosticSnapshot
h.locked(func() { snapshot = h.router.routeDiagnosticSnapshot(roomID) })
return snapshot
}
func (h *routerHarness) usage() sfu.Usage {
var usage sfu.Usage
h.locked(func() { usage = h.admission.Usage() })
return usage
}
// establishSfuRoom ports establishSfuRoom: a Host, a first Viewer moved to
// the SFU and a second Viewer joining it.
func (h *routerHarness) establishSfuRoom(created room.CreatedRoom) (host, first, second authenticatedRouteParticipant) {
h.t.Helper()
host = h.connectHost(created, nil, "", "")
first = h.connectViewer(created, "sfu-root-"+created.RoomID)
h.locked(func() {
h.doComplete(host)
h.doComplete(first)
})
firstDirect := h.waitPrepared(first.sessionID)
second = h.connectViewer(created, "sfu-leaf-"+created.RoomID)
h.locked(func() {
h.doReady(first, int64(firstDirect.Revision))
h.doComplete(second)
})
firstSfu := h.waitPreparedTransport(first.sessionID, "sfu")
h.routeReady(first, int64(firstSfu.Revision))
secondSfu := h.waitPreparedTransport(second.sessionID, "sfu")
h.routeReady(second, int64(secondSfu.Revision))
return host, first, second
}
// senderDiagnostics ports senderDiagnostics(state).
func senderDiagnostics(state string) protocol.SenderQualityDiagnostics {
var reason *string
switch state {
case "healthy":
value := "none"
reason = &value
case "degraded":
value := "bandwidth"
reason = &value
}
return protocol.SenderQualityDiagnostics{NatTraversalPath: "unknown", Reason: reason}
}
func senderEvidenceMessage(childPeerID, connectionID, rtpStatsID, trackIdentifier string, sampleTimestampMs float64, routeRevision int64, state string) protocol.SenderQualityEvidenceMessage {
timestamp := protocol.Num(sampleTimestampMs)
return protocol.SenderQualityEvidenceMessage{
Type: "sender-quality-evidence",
ChildPeerID: childPeerID,
ConnectionID: connectionID,
RtpStatsID: &rtpStatsID,
TrackIdentifier: &trackIdentifier,
SampleTimestampMs: &timestamp,
RouteRevision: protocol.Int(routeRevision),
State: state,
Diagnostics: senderDiagnostics(state),
}
}
type spyMedia struct {
*sfutest.FakeMedia
mu sync.Mutex
deleteCalls int
closePublication func(sfu.ResourceFence) error
}
func (media *spyMedia) ClosePublication(fence sfu.ResourceFence) error {
media.mu.Lock()
media.deleteCalls++
override := media.closePublication
media.mu.Unlock()
if override != nil {
return override(fence)
}
return media.FakeMedia.ClosePublication(fence)
}
func (media *spyMedia) counts() (int, int, int) {
media.mu.Lock()
defer media.mu.Unlock()
return media.deleteCalls, 0, len(media.SubscriptionDrainAttempts())
}
func (media *spyMedia) setClosePublication(closePublication func(sfu.ResourceFence) error) {
media.mu.Lock()
defer media.mu.Unlock()
media.closePublication = closePublication
}
func hasViewer(fences []sfu.SubscriptionFence, viewerPeerID string) bool {
for _, fence := range fences {
if fence.ViewerPeerID == viewerPeerID {
return true
}
}
return false
}
// manualClock is vi.useFakeTimers + vi.setSystemTime over sinon fake-timers:
// advance fires due timers in due order at their due time and lets the
// router settle after each (tickAsync); set jumps the clock and shifts every
// pending timer by the jump so none fires early (setSystemTime).
type manualClock struct {
mu sync.Mutex
nowMs int64
seq uint64
timers []*manualTimer
settle func()
}
type manualTimer struct {
dueMs int64
seq uint64
fn func()
settled bool
}
func (c *manualClock) now() int64 {
c.mu.Lock()
defer c.mu.Unlock()
return c.nowMs
}
func (c *manualClock) set(ms int64) {
c.mu.Lock()
defer c.mu.Unlock()
difference := ms - c.nowMs
c.nowMs = ms
for _, timer := range c.timers {
timer.dueMs += difference
}
}
func (c *manualClock) afterFunc(d time.Duration, fn func()) func() bool {
c.mu.Lock()
defer c.mu.Unlock()
timer := &manualTimer{dueMs: c.nowMs + d.Milliseconds(), seq: c.seq, fn: fn}
c.seq++
c.timers = append(c.timers, timer)
return func() bool {
c.mu.Lock()
defer c.mu.Unlock()
if timer.settled {
return false
}
timer.settled = true
c.timers = slices.DeleteFunc(c.timers, func(candidate *manualTimer) bool { return candidate == timer })
return true
}
}
func (c *manualClock) advance(ms int64) {
c.settle()
c.mu.Lock()
target := c.nowMs + ms
c.mu.Unlock()
for {
c.mu.Lock()
var next *manualTimer
for _, timer := range c.timers {
if timer.dueMs <= target && (next == nil || timer.dueMs < next.dueMs ||
(timer.dueMs == next.dueMs && timer.seq < next.seq)) {
next = timer
}
}
if next == nil {
c.nowMs = target
c.mu.Unlock()
c.settle()
return
}
next.settled = true
c.timers = slices.DeleteFunc(c.timers, func(candidate *manualTimer) bool { return candidate == next })
c.nowMs = max(c.nowMs, next.dueMs)
c.mu.Unlock()
next.fn()
c.settle()
}
}
// debugRecorder is vi.spyOn(router, "debug"): a slog handler that records
// the sanitised route events while the debug flag is forced on.
type debugRecord struct {
roomID string
event string
details map[string]any
}
type debugRecorder struct {
mu sync.Mutex
records []debugRecord
}
func captureRouteDebug(t *testing.T) *debugRecorder {
t.Helper()
recorder := &debugRecorder{}
previousLogger := slog.Default()
t.Setenv("PIIK_DEBUG", "route")
slog.SetDefault(slog.New(recorder))
t.Cleanup(func() {
slog.SetDefault(previousLogger)
})
return recorder
}
func (d *debugRecorder) Enabled(context.Context, slog.Level) bool { return true }
func (d *debugRecorder) Handle(_ context.Context, record slog.Record) error {
if record.Message != "piik-route" {
return nil
}
details := map[string]any{}
record.Attrs(func(attr slog.Attr) bool {
details[attr.Key] = normaliseAttr(attr.Value)
return true
})
roomID, _ := details["roomId"].(string)
event, _ := details["event"].(string)
d.mu.Lock()
defer d.mu.Unlock()
d.records = append(d.records, debugRecord{roomID: roomID, event: event, details: details})
return nil
}
func (d *debugRecorder) WithAttrs([]slog.Attr) slog.Handler { return d }
func (d *debugRecorder) WithGroup(string) slog.Handler { return d }
// calledWith is expect(debug).toHaveBeenCalledWith(roomId, event,
// expect.objectContaining(expected)).
func (d *debugRecorder) calledWith(roomID, event string, expected map[string]any) bool {
d.mu.Lock()
defer d.mu.Unlock()
for _, record := range d.records {
if record.roomID != roomID || record.event != event {
continue
}
matches := true
for key, want := range expected {
if got, ok := record.details[key]; !ok || !reflect.DeepEqual(got, want) {
matches = false
break
}
}
if matches {
return true
}
}
return false
}
// normaliseAttr flattens slog values to string/int64/float64/bool/nil so the
// expectations can use plain literals.
func normaliseAttr(value slog.Value) any {
switch value.Kind() {
case slog.KindString:
return value.String()
case slog.KindInt64:
return value.Int64()
case slog.KindUint64:
return int64(value.Uint64())
case slog.KindFloat64:
return value.Float64()
case slog.KindBool:
return value.Bool()
case slog.KindAny:
raw := value.Any()
if raw == nil {
return nil
}
reflected := reflect.ValueOf(raw)
switch reflected.Kind() {
case reflect.String:
return reflected.String()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return reflected.Int()
case reflect.Float32, reflect.Float64:
return reflected.Float()
case reflect.Bool:
return reflected.Bool()
}
return raw
}
return value.Any()
}
var qualityPolicy = &protocol.RoutePolicy{PeerOnly: false, TopologyOptimization: true, NatPrediction: false}
// prepareQualityConvergence ports the shared prefix of "commits P2P quality
// convergence after client relative approval": two Viewers on the Host, the
// second advertising relay capacity, and enough sender evidence for the
// controller to prepare a quality-convergence candidate for the first.
func prepareQualityConvergence(h *routerHarness, debug *debugRecorder) (created room.CreatedRoom, host, first, second authenticatedRouteParticipant, firstEdge activeViewerMediaEdge, qualityPrepare protocol.RouteUpdatePrepareMessage) {
t := h.t
t.Helper()
created = h.createRoom()
host = h.connectHost(created, qualityPolicy, "", "")
first = h.connectViewer(created, "quality-first")
h.locked(func() {
h.doComplete(host)
h.doComplete(first)
})
firstPrepare := h.waitPrepared(first.sessionID)
if firstPrepare.Candidate.QualityProbe {
t.Fatal("the join candidate must not be a quality probe")
}
second = h.connectViewer(created, "quality-second")
h.locked(func() {
h.doReady(first, int64(firstPrepare.Revision))
h.doComplete(second)
})
secondPrepare := h.waitPrepared(second.sessionID)
h.locked(func() {
h.doReady(second, int64(secondPrepare.Revision))
h.doRelay(second, 3)
firstEdge = h.doMustEdge(created.RoomID, first.peerID)
secondEdge := h.doMustEdge(created.RoomID, second.peerID)
if firstEdge.upstream != protocol.PeerUpstream(host.peerID) || secondEdge.upstream != protocol.PeerUpstream(host.peerID) {
t.Fatalf("both viewers must start on the host: %v %v", firstEdge.upstream, secondEdge.upstream)
}
if !h.router.observeSenderQualityEvidence(host, senderEvidenceMessage(second.peerID, secondEdge.connectionID, "second-rtp", "track", 100, secondEdge.revision, "healthy")) {
t.Fatal("healthy evidence for the second viewer must be accepted")
}
if debug != nil && !debug.calledWith(created.RoomID, "sender-quality-evidence", map[string]any{
"routeRevision": secondEdge.revision, "committedCopies": int64(2), "candidateReservedCopies": int64(0),
"possibleCopies": int64(2), "endpointCapacity": int64(2), "operationReason": nil,
"candidateTransition": nil, "sampleRole": "active",
}) {
t.Fatalf("active sender evidence debug context missing: %+v", debug.records)
}
if !h.router.observeSenderQualityEvidence(host, senderEvidenceMessage(first.peerID, firstEdge.connectionID, "first-rtp", "track", 100, firstEdge.revision, "healthy")) {
t.Fatal("healthy evidence for the first viewer must be accepted")
}
for window := 0; window < 3; window++ {
if !h.router.observeSenderQualityEvidence(host, senderEvidenceMessage(first.peerID, firstEdge.connectionID, "first-rtp", "track", float64(101+window), firstEdge.revision, "degraded")) {
t.Fatal("degraded evidence must be accepted")
}
}
})
qualityPrepare = h.waitPreparedAbove(first.sessionID, firstEdge.revision)
if !qualityPrepare.Candidate.QualityProbe {
t.Fatal("the convergence candidate must be a quality probe")
}
h.locked(func() {
if !h.router.observeSenderQualityEvidence(second, senderEvidenceMessage(first.peerID, qualityPrepare.Candidate.ConnectionID, "candidate-rtp", "track", 200, int64(qualityPrepare.Revision), "healthy")) {
t.Fatal("candidate evidence must be accepted")
}
if debug != nil && !debug.calledWith(created.RoomID, "sender-quality-evidence", map[string]any{
"routeRevision": int64(qualityPrepare.Revision), "committedCopies": int64(0), "candidateReservedCopies": int64(1),
"possibleCopies": int64(1), "endpointCapacity": int64(2), "operationReason": "quality-convergence",
"candidateTransition": "none", "sampleRole": "candidate",
}) {
t.Fatalf("candidate sender evidence debug context missing: %+v", debug.records)
}
})
return created, host, first, second, firstEdge, qualityPrepare
}
// ---------------------------------------------------------------------------
// tests, in tests/hybrid-media-router.test.ts order
// ---------------------------------------------------------------------------
// TS: it.each([undefined, false, true]) with candidateReady mocked. Go cannot
// monkeypatch the controller and SettleResult.Committed collapses undefined
// and false, so the two representable outcomes come from a real
// quality-convergence candidate: ready without relative approval settles
// accepted-but-uncommitted, ready with approval commits.
func TestRouterPublishesConnectionIdentityOnlyForExplicitCommit(t *testing.T) {
for _, committed := range []bool{false, true} {
t.Run(map[bool]string{false: "uncommitted", true: "committed"}[committed], func(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2})
created, _, first, _, firstEdge, qualityPrepare := prepareQualityConvergence(h, nil)
var got string
h.locked(func() {
if committed {
h.doReadyApproved(first, int64(qualityPrepare.Revision))
} else {
h.doReady(first, int64(qualityPrepare.Revision))
}
edge, _ := h.doEdge(created.RoomID, first.peerID)
got = edge.connectionID
})
want := firstEdge.connectionID
if committed {
want = qualityPrepare.Candidate.ConnectionID
}
if got != want {
t.Fatalf("connection identity = %q, want %q", got, want)
}
})
}
}
func TestRouterLogsOnlyAggregateCandidateOriginForRouteSignals(t *testing.T) {
debug := captureRouteDebug(t)
h := newRouterHarness(t, routerHarnessOptions{capacity: 2})
h.locked(func() {
h.router.debugPeerSignal(peerSignalDebugInput{
roomID: "1234",
sourcePeerID: "source_12345678",
targetPeerID: "target_12345678",
signalKind: "candidate",
candidateOrigin: protocol.CandidateOriginPredicted,
authorization: signalAuthorizationProbe,
})
})
if !debug.calledWith("1234", "peer-signal", map[string]any{
"candidateOrigin": "predicted", "signalKind": "candidate", "authorization": "probe",
}) {
t.Fatalf("peer-signal debug event missing: %+v", debug.records)
}
for _, record := range debug.records {
for _, value := range record.details {
if text, ok := value.(string); ok && (strings.Contains(text, "source_12345678") || strings.Contains(text, "target_12345678")) {
t.Fatalf("raw peer id leaked into debug output: %+v", record.details)
}
}
}
}
func TestRouterProjectsExactEndpointCopyContextFromCurrentSnapshot(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2})
edge := func(parentPeerID, connectionID string) route.CommittedEdge {
return route.CommittedEdge{
Kind: route.UpstreamPeer, ChildSessionID: "child-session", ParentPeerID: parentPeerID,
ParentSessionID: "parent-session", Transport: route.TransportDirect, ConnectionID: connectionID,
Usable: true, PhysicalActive: true,
}
}
edges := func(entries ...[3]string) *ordered.Map[string, route.CommittedEdge] {
upstream := &ordered.Map[string, route.CommittedEdge]{}
for _, entry := range entries {
upstream.Set(entry[0], edge(entry[1], entry[2]))
}
return upstream
}
operation := func(tuple route.CandidateTuple, transition route.EndpointTransition) *route.OperationSnapshot {
return &route.OperationSnapshot{
ChildPeerID: "candidate", ChildSessionID: "candidate-session", DemandPeerID: "candidate",
Reason: route.DemandQualityConvergence, BaseRevision: 7, FactVersion: 1,
Candidates: []route.CandidatePlan{{Tuple: tuple, EndpointTransition: transition}},
Cursor: 0, DeadlineAtMs: 20_000, WakeAtMs: 20_000,
Current: &route.CurrentAttempt{Tuple: tuple, Revision: 8, ConnectionID: "candidate-connection"},
}
}
snapshot := func(upstream *ordered.Map[string, route.CommittedEdge], op *route.OperationSnapshot, publication *route.HostPublication) route.RouteSnapshot {
return route.RouteSnapshot{Revision: 7, FactVersion: 1, UpstreamByViewer: upstream, HostPublication: publication, Operation: op}
}
project := func(state route.RouteSnapshot, observedPeerID string, endpointCapacity int, sample qualitySample) qualityCopyContext {
var context qualityCopyContext
h.locked(func() { context = h.router.qualityCopyContext(state, "host", observedPeerID, endpointCapacity, sample) })
return context
}
hostProducer := "host"
overlap := route.EndpointTransition{Kind: route.TransitionOverlap, ProducerPeerID: &hostProducer}
none := route.EndpointTransition{Kind: route.TransitionNone}
hostEdges := edges([3]string{"a", "host", "a-connection"}, [3]string{"b", "host", "b-connection"})
peerCandidate := route.CandidateTuple{Kind: route.UpstreamPeer, ParentPeerID: "host", Transport: route.TransportDirect}
sfuSample := qualitySample{kind: route.UpstreamSfu, routeRevision: 8, publicationGeneration: "publication", hostSessionID: "host-session"}
publication := &route.HostPublication{
Generation: "publication", HostSessionID: "host-session", ConnectionID: "publication-connection",
Usable: true, PhysicalActive: true, Resource: &route.Resource{},
}
got := project(snapshot(hostEdges, operation(peerCandidate, overlap), nil), "host", 2,
qualitySample{kind: route.UpstreamPeer, childPeerID: "candidate", routeRevision: 8, connectionID: "candidate-connection"})
want := qualityCopyContext{
committedCopies: 2, candidateReservedCopies: 1, possibleCopies: 3, endpointCapacity: 2,
operationReason: route.DemandQualityConvergence, candidateTransition: route.TransitionOverlap, sampleRole: "candidate",
}
if got != want {
t.Fatalf("peer candidate context = %+v, want %+v", got, want)
}
oneHostEdge := edges([3]string{"a", "host", "a-connection"})
sfuCases := []struct {
state route.RouteSnapshot
expected qualityCopyContext
}{
{
state: snapshot(hostEdges, operation(route.CandidateTuple{Kind: route.UpstreamSfu, Publication: route.PublicationCreate}, overlap), nil),
expected: qualityCopyContext{committedCopies: 2, candidateReservedCopies: 1, possibleCopies: 3,
candidateTransition: route.TransitionOverlap},
},
{
state: snapshot(oneHostEdge, operation(route.CandidateTuple{Kind: route.UpstreamSfu, Publication: route.PublicationReuse}, none), publication),
expected: qualityCopyContext{committedCopies: 2, candidateReservedCopies: 0, possibleCopies: 2,
candidateTransition: route.TransitionNone},
},
}
for index, testCase := range sfuCases {
got := project(testCase.state, "host", 2, sfuSample)
if got.committedCopies != testCase.expected.committedCopies ||
got.candidateReservedCopies != testCase.expected.candidateReservedCopies ||
got.possibleCopies != testCase.expected.possibleCopies ||
got.candidateTransition != testCase.expected.candidateTransition ||
got.sampleRole != "candidate" {
t.Fatalf("sfu case %d = %+v, want %+v (candidate)", index, got, testCase.expected)
}
}
relayEdges := edges([3]string{"a", "relay", "relay-a"}, [3]string{"b", "relay", "relay-b"})
relaySample := qualitySample{kind: route.UpstreamPeer, childPeerID: "a", routeRevision: 7, connectionID: "relay-a"}
got = project(snapshot(relayEdges, nil, nil), "relay", 3, relaySample)
if got.committedCopies != 2 || got.candidateReservedCopies != 0 || got.possibleCopies != 2 ||
got.endpointCapacity != 3 || got.sampleRole != "active" {
t.Fatalf("relay context = %+v", got)
}
stale := relaySample
stale.routeRevision = 6
if got := project(snapshot(relayEdges, nil, nil), "relay", 3, stale); got.sampleRole != "unknown" {
t.Fatalf("stale relay sample role = %q, want unknown", got.sampleRole)
}
}
func TestRouterKeepsControllerDiagnosticLabelUntilDepartedRelayIsPruned(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
parent := h.connectViewer(created, "diagnostic-label-parent")
h.locked(func() {
h.doComplete(host)
h.doComplete(parent)
})
parentPrepare := h.waitPrepared(parent.sessionID)
temporaryRoot := h.connectViewer(created, "diagnostic-label-temporary-root")
h.locked(func() {
h.doReady(parent, int64(parentPrepare.Revision))
h.doRelay(parent, 1)
h.doComplete(temporaryRoot)
})
temporaryPrepare := h.waitPreparedUpstream(temporaryRoot.sessionID, protocol.PeerUpstream(host.peerID))
child := h.connectViewer(created, "diagnostic-label-child")
h.locked(func() {
h.doReady(temporaryRoot, int64(temporaryPrepare.Revision))
h.doComplete(child)
})
childPrepare := h.waitPreparedUpstream(child.sessionID, protocol.PeerUpstream(parent.peerID))
h.locked(func() {
h.doReady(child, int64(childPrepare.Revision))
h.doDisconnect(temporaryRoot)
h.router.removeViewer(created.RoomID, temporaryRoot.peerID)
})
h.waitFor("the temporary root to lose its edge", func() bool {
_, ok := h.activeEdge(created.RoomID, temporaryRoot.peerID)
return !ok
})
debugPeer := func(peerID string) string {
var label string
h.locked(func() { label = h.router.debugPeer(created.RoomID, peerID) })
return label
}
h.locked(func() {
label := h.router.debugPeer(created.RoomID, parent.peerID)
h.doDisconnect(parent)
h.router.removeViewer(created.RoomID, parent.peerID)
if got := h.router.debugPeer(created.RoomID, parent.peerID); got != label {
t.Fatalf("label after departure = %q, want %q", got, label)
}
})
recovery := h.waitPreparedMatching(child.sessionID, "a recovery onto the host", func(prepared protocol.RouteUpdatePrepareMessage) bool {
return int64(prepared.Revision) > int64(childPrepare.Revision) &&
prepared.Assignment.Upstream == protocol.PeerUpstream(host.peerID)
})
h.routeReady(child, int64(recovery.Revision))
h.waitFor("the departed relay to be pruned", func() bool { return debugPeer(parent.peerID) == "viewer-unknown" })
}
func TestRouterMapsExactHostActiveFailureToCommittedDirectEdge(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
viewer := h.connectViewer(created, "host-edge-failure")
h.locked(func() {
h.doComplete(host)
h.doComplete(viewer)
})
prepared := h.waitPrepared(viewer.sessionID)
h.locked(func() {
h.doReady(viewer, int64(prepared.Revision))
active := h.doMustEdge(created.RoomID, viewer.peerID)
h.doFailed(host, routeFailedMessage(active.revision, "active", active.connectionID))
})
h.waitFailed(viewer.sessionID)
}
func TestRouterRebindsReplacementHostAndClearsOfflineViewerMediaOwnership(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2})
created := h.createRoom()
previousHost := h.connectHost(created, nil, "", "")
viewer := h.connectViewer(created, "host-rebind-offline-viewer")
h.locked(func() {
h.doComplete(previousHost)
h.doComplete(viewer)
})
prepared := h.waitPrepared(viewer.sessionID)
h.locked(func() {
h.doReady(viewer, int64(prepared.Revision))
h.doRelay(viewer, 1)
})
h.waitFor("the viewer connection identity", func() bool {
_, ok := h.activeEdge(created.RoomID, viewer.peerID)
return ok
})
h.locked(func() {
h.doDisconnect(viewer)
h.doDisconnect(previousHost)
})
resumedHost := h.connectHost(created, nil, "replacement-host-client", "replacement-host-session")
if resumedHost.peerID == previousHost.peerID {
t.Fatal("a replacement client must receive a new host peer id")
}
h.locked(func() {
h.doComplete(resumedHost)
if _, ok := h.doEdge(created.RoomID, viewer.peerID); ok {
t.Fatal("the offline viewer must lose its media ownership")
}
})
reconnectedViewer := h.connectViewer(created, "host-rebind-offline-viewer")
if reconnectedViewer.peerID != viewer.peerID {
t.Fatal("the same client must keep its viewer peer id")
}
h.locked(func() {
h.router.connectParticipant(reconnectedViewer)
found := false
for _, child := range h.router.routeDiagnosticSnapshot(created.RoomID).Children {
if child.EffectiveCapacity == 1 {
found = true
}
}
if !found {
t.Fatal("the reconnected viewer must keep its advertised capacity")
}
})
}
func TestRouterAcceptsActiveFailureForDirectConnectionAdoptedDuringRebuild(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
viewer := h.connectViewer(created, "rebuilt-edge-failure")
h.locked(func() {
h.doComplete(host)
h.doComplete(viewer)
})
prepared := h.waitPrepared(viewer.sessionID)
h.locked(func() {
h.doReady(viewer, int64(prepared.Revision))
active := h.doMustEdge(created.RoomID, viewer.peerID)
rebuiltConnectionID := "rebuilt_connection_12345678"
authorization := h.router.peerSignalAuthorization(peerSignalInput{
roomID: created.RoomID, sourcePeerID: host.peerID, sourceSessionID: host.sessionID,
targetPeerID: viewer.peerID, targetSessionID: viewer.sessionID, connectionID: rebuiltConnectionID,
signalKind: "description", descriptionType: "offer",
})
if authorization != signalAuthorizationAllowed {
t.Fatalf("adoption authorization = %v, want allowed", authorization)
}
adopted := h.doMustEdge(created.RoomID, viewer.peerID)
if adopted.revision != active.revision || adopted.connectionID != rebuiltConnectionID {
t.Fatalf("adopted edge = %+v", adopted)
}
h.doFailed(viewer, routeFailedMessage(active.revision, "active", rebuiltConnectionID))
})
h.waitFailed(viewer.sessionID)
}
func TestRouterKeepsCommittedEdgeSignalingAuthorizedDuringAnotherPeerCandidate(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
h.complete(host)
roots := []authenticatedRouteParticipant{
h.connectViewer(created, "stable-root-a"),
h.connectViewer(created, "stable-root-b"),
}
// TS: for each root, complete then await; the ready + capacity of one
// root share the tick with the next root's complete.
var pending func()
for _, root := range roots {
h.locked(func() {
if pending != nil {
pending()
}
h.doComplete(root)
})
prepared := h.waitPrepared(root.sessionID)
pending = func() {
h.doReady(root, int64(prepared.Revision))
h.doRelay(root, 1)
}
}
candidateChild := h.connectViewer(created, "candidate-child")
h.locked(func() {
pending()
h.doComplete(candidateChild)
})
prepared := h.waitPreparedMatching(candidateChild.sessionID, "a peer candidate", func(prepared protocol.RouteUpdatePrepareMessage) bool {
return prepared.Assignment.Upstream.Kind == "peer"
})
var candidateParent, unrelatedRoot authenticatedRouteParticipant
for _, root := range roots {
if root.peerID == prepared.Assignment.Upstream.PeerID {
candidateParent = root
} else {
unrelatedRoot = root
}
}
h.locked(func() {
activeParentEdge := h.doMustEdge(created.RoomID, candidateParent.peerID)
authorize := func(input peerSignalInput) signalAuthorization {
input.roomID = created.RoomID
input.signalKind = "description"
input.descriptionType = "offer"
return h.router.peerSignalAuthorization(input)
}
if got := authorize(peerSignalInput{
sourcePeerID: host.peerID, sourceSessionID: host.sessionID,
targetPeerID: candidateParent.peerID, targetSessionID: candidateParent.sessionID,
connectionID: activeParentEdge.connectionID,
}); got != signalAuthorizationAllowed {
t.Fatalf("committed edge authorization = %v, want allowed", got)
}
if got := authorize(peerSignalInput{
sourcePeerID: candidateParent.peerID, sourceSessionID: candidateParent.sessionID,
targetPeerID: candidateChild.peerID, targetSessionID: candidateChild.sessionID,
connectionID: prepared.Candidate.ConnectionID,
}); got != signalAuthorizationProbe {
t.Fatalf("candidate authorization = %v, want probe", got)
}
if got := authorize(peerSignalInput{
sourcePeerID: unrelatedRoot.peerID, sourceSessionID: unrelatedRoot.sessionID,
targetPeerID: candidateChild.peerID, targetSessionID: candidateChild.sessionID,
connectionID: prepared.Candidate.ConnectionID,
}); got != signalAuthorizationUnassigned {
t.Fatalf("unrelated authorization = %v, want unassigned", got)
}
})
}
func TestRouterCommitsP2PQualityConvergenceAfterClientRelativeApproval(t *testing.T) {
debug := captureRouteDebug(t)
h := newRouterHarness(t, routerHarnessOptions{capacity: 2})
created, host, first, second, _, qualityPrepare := prepareQualityConvergence(h, debug)
h.locked(func() {
h.doReady(first, int64(qualityPrepare.Revision))
if edge := h.doMustEdge(created.RoomID, first.peerID); edge.upstream != protocol.PeerUpstream(host.peerID) {
t.Fatalf("upstream before approval = %v, want the host", edge.upstream)
}
h.doReadyApproved(first, int64(qualityPrepare.Revision))
if edge := h.doMustEdge(created.RoomID, first.peerID); edge.upstream != protocol.PeerUpstream(second.peerID) {
t.Fatalf("upstream after approval = %v, want the second viewer", edge.upstream)
}
})
}
func TestRouterPreparesSameParentConnectionRegenerationAfterPersistentDegradation(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2})
created := h.createRoom()
host := h.connectHost(created, qualityPolicy, "", "")
viewer := h.connectViewer(created, "quality-regeneration")
h.locked(func() {
h.doComplete(host)
h.doComplete(viewer)
})
initial := h.waitPrepared(viewer.sessionID)
if initial.Assignment.Upstream != protocol.PeerUpstream(host.peerID) {
t.Fatalf("initial upstream = %v", initial.Assignment.Upstream)
}
var edge activeViewerMediaEdge
h.locked(func() {
h.doReady(viewer, int64(initial.Revision))
edge = h.doMustEdge(created.RoomID, viewer.peerID)
h.router.observeSenderQualityEvidence(host, senderEvidenceMessage(viewer.peerID, edge.connectionID, "old-sender-rtp", "old-track", 100, edge.revision, "healthy"))
for _, sampleTimestampMs := range []float64{101, 102, 103} {
h.router.observeSenderQualityEvidence(host, senderEvidenceMessage(viewer.peerID, edge.connectionID, "old-sender-rtp", "old-track", sampleTimestampMs, edge.revision, "degraded"))
}
})
regeneration := h.waitPreparedAbove(viewer.sessionID, edge.revision)
if regeneration.Candidate.Transport != "direct" || !regeneration.Candidate.QualityProbe {
t.Fatalf("regeneration candidate = %+v", regeneration.Candidate)
}
if regeneration.Assignment.Upstream != protocol.PeerUpstream(host.peerID) {
t.Fatalf("regeneration upstream = %v", regeneration.Assignment.Upstream)
}
if hostPrepare, ok := h.preparedFor(host.sessionID); !ok || hostPrepare.Candidate.ConnectionID != regeneration.Candidate.ConnectionID {
t.Fatal("the host must receive the same regeneration candidate")
}
if active := h.mustActiveEdge(created.RoomID, viewer.peerID); active.connectionID != edge.connectionID ||
active.upstream != protocol.PeerUpstream(host.peerID) {
t.Fatalf("active edge during regeneration = %+v", active)
}
}
func TestRouterCreatesNatRetriesThroughRealPrepareMessages(t *testing.T) {
for _, withSfu := range []bool{false, true} {
t.Run(map[bool]string{false: "sfu=false", true: "sfu=true"}[withSfu], func(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2, withSfu: withSfu})
created := h.createRoom()
host := h.connectHost(created, &protocol.RoutePolicy{PeerOnly: true, TopologyOptimization: false, NatPrediction: true}, "", "")
viewer := h.connectViewer(created, "nat-retry")
h.locked(func() {
h.doComplete(host)
h.doComplete(viewer)
})
connectionIDs := map[string]struct{}{}
var staleFailure *protocol.RouteFailedMessage
for current := 1; current <= 3; current++ {
prepared := h.waitPreparedMatching(viewer.sessionID, "connection attempt", func(prepared protocol.RouteUpdatePrepareMessage) bool {
return prepared.Candidate.ConnectionAttempt != nil &&
*prepared.Candidate.ConnectionAttempt == protocol.ConnectionAttempt{Current: protocol.Int(current), Total: 3}
})
connectionIDs[prepared.Candidate.ConnectionID] = struct{}{}
h.locked(func() {
if hostPrepare, ok := lastPrepare(h.sent[host.sessionID], ""); !ok || !reflect.DeepEqual(hostPrepare.Candidate, prepared.Candidate) {
t.Fatal("the host must see the same candidate")
}
if staleFailure != nil {
h.doFailed(viewer, *staleFailure)
}
if again, _ := lastPrepare(h.sent[viewer.sessionID], ""); again.Candidate.ConnectionID != prepared.Candidate.ConnectionID {
t.Fatal("a stale failure must not disturb the current candidate")
}
if current == 3 {
h.doReady(viewer, int64(prepared.Revision))
if active := h.doMustEdge(created.RoomID, viewer.peerID); active.connectionID != prepared.Candidate.ConnectionID {
t.Fatalf("active edge = %+v", active)
}
} else {
failure := routeFailedMessage(int64(prepared.Revision), "prepare", prepared.Candidate.ConnectionID)
staleFailure = &failure
h.doFailed(viewer, failure)
h.doFailed(host, failure)
}
})
}
if len(connectionIDs) != 3 {
t.Fatalf("distinct connection ids = %d, want 3", len(connectionIDs))
}
for _, message := range h.allMessages() {
switch typed := message.(type) {
case protocol.SfuConfigMessage:
t.Fatal("peer-only media must not receive an sfu-config")
case protocol.RouteStatusMessage:
if typed.State == "failed" {
t.Fatal("NAT retries must not report failure")
}
}
}
})
}
}
func TestRouterKeepsPeerOnlySharesOutOfConfiguredSfuFallback(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true})
created := h.createRoom()
host := h.connectHost(created, &protocol.RoutePolicy{PeerOnly: true}, "", "")
first := h.connectViewer(created, "peer-only-first")
h.locked(func() {
h.doComplete(host)
h.doComplete(first)
})
direct := h.waitPreparedTransport(first.sessionID, "direct")
blocked := h.connectViewer(created, "peer-only-blocked")
h.locked(func() {
h.doReady(first, int64(direct.Revision))
h.doComplete(blocked)
})
h.waitFailed(blocked.sessionID)
if sfuConfigCount(h.allMessages()) != 0 {
t.Fatal("peer-only shares must never receive an sfu-config")
}
}
func TestRouterSilentlyKeepsWorkingMediaWhenQualityOnlySfuAdmissionIsDenied(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true, sfuIngressCapacity: 1})
occupied := sfu.ResourceFence{RoomID: "9001", ShareGeneration: "occupied_share_12345678", PublicationGeneration: "occupied_publication_12345678"}
created := h.createRoom()
host := h.connectHost(created, qualityPolicy, "", "")
viewer := h.connectViewer(created, "quality-sfu-denied")
h.locked(func() {
if !h.admission.ReservePublication(occupied) {
t.Fatal("the occupying publication must be reserved")
}
h.doComplete(host)
h.doComplete(viewer)
})
prepared := h.waitPrepared(viewer.sessionID)
var active activeViewerMediaEdge
h.locked(func() {
h.doReady(viewer, int64(prepared.Revision))
active = h.doMustEdge(created.RoomID, viewer.peerID)
for index, state := range []string{"healthy", "degraded"} {
if !h.router.observeSenderQualityEvidence(host, senderEvidenceMessage(viewer.peerID, active.connectionID, "viewer-rtp", "track", float64(100+index), active.revision, state)) {
t.Fatalf("%s evidence must be accepted", state)
}
}
})
h.waitFor("the quality operation to settle", func() bool { return h.diagnostic(created.RoomID).Operation == nil })
for _, message := range h.messages(viewer.sessionID) {
if _, ok := message.(protocol.RouteStatusMessage); ok {
t.Fatal("a denied quality-only candidate must stay silent")
}
}
if edge := h.mustActiveEdge(created.RoomID, viewer.peerID); edge.connectionID != active.connectionID ||
edge.upstream != protocol.PeerUpstream(host.peerID) {
t.Fatalf("working media changed: %+v", edge)
}
h.locked(func() {
h.admission.BeginDrain(occupied)
h.admission.CompleteDrain(occupied)
})
}
func TestRouterNotifiesPresenceOwnerAfterCommittedGraphChanges(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true})
created := h.createRoom()
h.establishSfuRoom(created)
h.waitFor("the last routesChanged call", func() bool {
var last string
h.locked(func() {
if len(h.routesChanged) > 0 {
last = h.routesChanged[len(h.routesChanged)-1]
}
})
return last == created.RoomID
})
}
func TestRouterWakesAtDerivedDirectBoundaryAndPreparesSfuAutomatically(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2, withSfu: true, prepareTimeoutMs: 300, fakeTimers: true})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
viewer := h.connectViewer(created, "staged-deadline")
h.locked(func() {
h.doComplete(host)
h.doComplete(viewer)
})
h.waitPreparedTransport(viewer.sessionID, "direct")
h.clock.advance(150)
h.waitPreparedTransport(viewer.sessionID, "sfu")
if h.hasRouteStatus(viewer.sessionID, "failed") {
t.Fatal("the direct boundary must not report failure")
}
}
func TestRouterRelayPreparationFailureAdvancesWithoutWaitingForDeadline(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2, withSfu: true, fakeTimers: true})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
relay := h.connectViewer(created, "relay")
h.locked(func() {
h.doComplete(host)
h.doComplete(relay)
})
h.settle()
root, ok := h.preparedFor(relay.sessionID)
if !ok {
t.Fatal("relay must prepare its Host upstream")
}
h.locked(func() {
h.doReady(relay, int64(root.Revision))
h.doRelay(relay, 2)
})
child := h.connectViewer(created, "child")
h.complete(child)
h.settle()
direct, ok := h.preparedFor(child.sessionID)
if !ok || direct.Assignment.Upstream != protocol.PeerUpstream(host.peerID) {
t.Fatal("child must try the Host first")
}
h.routeFailed(host, routeFailedMessage(int64(direct.Revision), "prepare", direct.Candidate.ConnectionID))
h.settle()
prepared, ok := h.preparedFor(child.sessionID)
if !ok || prepared.Assignment.Upstream != protocol.PeerUpstream(relay.peerID) {
t.Fatal("child must next try the relay")
}
failure := routeFailedMessage(int64(prepared.Revision), "prepare", prepared.Candidate.ConnectionID)
h.routeFailed(relay, failure)
h.settle()
fallback, ok := h.preparedFor(child.sessionID)
if !ok || fallback.Candidate.Transport != "sfu" || fallback.Revision <= prepared.Revision {
t.Fatalf("relay rejection must prepare SFU immediately: %+v", fallback)
}
h.routeFailed(relay, failure)
h.settle()
current, _ := h.preparedFor(child.sessionID)
if current.Candidate.ConnectionID != fallback.Candidate.ConnectionID {
t.Fatal("duplicate relay failure disturbed the next candidate")
}
if h.clock.now() != 0 {
t.Fatal("preparation failure must advance without a timer wake")
}
if retained := h.mustActiveEdge(created.RoomID, relay.peerID); retained.connectionID != root.Candidate.ConnectionID {
t.Fatal("relay preparation failure disturbed its active upstream")
}
}
func TestRouterKeepsExactTransportConnectedDirectCandidatePastBoundary(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2, withSfu: true, prepareTimeoutMs: 300, fakeTimers: true})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
viewer := h.connectViewer(created, "transport-connected")
h.locked(func() {
h.doComplete(host)
h.doComplete(viewer)
})
direct := h.waitPreparedTransport(viewer.sessionID, "direct")
h.locked(func() {
h.router.handleRouteTransportConnected(viewer, protocol.RouteTransportConnectedMessage{
Type: "route-transport-connected", Revision: direct.Revision, ConnectionID: direct.Candidate.ConnectionID})
})
h.clock.advance(150)
if prepared, _ := h.preparedFor(viewer.sessionID); !reflect.DeepEqual(prepared, direct) {
t.Fatalf("prepare after the boundary = %+v, want the direct candidate", prepared)
}
h.routeReady(viewer, int64(direct.Revision))
h.waitFor("the active direct edge", func() bool {
active, ok := h.activeAfter(viewer.sessionID, int64(direct.Revision)-1)
return ok && active.Assignment.Upstream.Kind == "peer"
})
}
func TestRouterAcceptsExactParentProgressWithoutGrantingReadiness(t *testing.T) {
for _, relayParent := range []bool{false, true} {
t.Run(fmt.Sprintf("relay=%v", relayParent), func(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2, withSfu: true, prepareTimeoutMs: 300, fakeTimers: true})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
h.locked(func() { h.doComplete(host) })
parents := map[string]authenticatedRouteParticipant{host.peerID: host}
if relayParent {
for _, name := range []string{"root-a", "root-b"} {
root := h.connectViewer(created, name)
h.locked(func() { h.doComplete(root) })
prepared := h.waitPreparedTransport(root.sessionID, "direct")
h.routeReady(root, int64(prepared.Revision))
h.locked(func() { h.doRelay(root, 2) })
parents[root.peerID] = root
}
}
viewer := h.connectViewer(created, "delayed-feedback")
h.locked(func() { h.doComplete(viewer) })
direct := h.waitPreparedTransport(viewer.sessionID, "direct")
h.locked(func() {
rm, _ := h.router.rooms.Get(created.RoomID)
before := rm.controller.Operation()
parent := parents[before.Current.Tuple.ParentPeerID]
if (parent.role == protocol.RoleViewer) != relayParent {
t.Fatal("unexpected preparing parent")
}
message := protocol.RouteTransportConnectedMessage{Type: "route-transport-connected", Revision: direct.Revision, ConnectionID: direct.Candidate.ConnectionID}
stale := parent
stale.sessionID = "retired-session"
h.router.handleRouteTransportConnected(stale, message)
foreign := parent
foreign.peerID = "unrelated-peer"
h.router.handleRouteTransportConnected(foreign, message)
wrong := message
wrong.ConnectionID = "retired-connection"
h.router.handleRouteTransportConnected(parent, wrong)
wrong = message
wrong.Revision++
h.router.handleRouteTransportConnected(parent, wrong)
if rm.controller.Operation().WakeAtMs != before.WakeAtMs {
t.Fatal("unrelated progress changed candidate timing")
}
h.router.handleRouteTransportConnected(parent, message)
after := rm.controller.Operation()
if after == nil || after.DeadlineAtMs != before.DeadlineAtMs || after.WakeAtMs != before.DeadlineAtMs {
t.Fatal("parent progress did not retain the original total deadline")
}
h.doReady(parent, int64(direct.Revision))
if rm.controller.Operation() == nil {
t.Fatal("parent committed a route without Viewer readiness")
}
})
h.clock.advance(150)
if prepared, _ := h.preparedFor(viewer.sessionID); !reflect.DeepEqual(prepared, direct) {
t.Fatal("connected candidate was replaced before delayed Viewer readiness")
}
h.routeReady(viewer, int64(direct.Revision))
h.waitFor("the active peer edge", func() bool {
active, ok := h.activeAfter(viewer.sessionID, int64(direct.Revision)-1)
return ok && active.Assignment.Upstream.Kind == "peer"
})
})
}
}
func TestRouterUsesDirectBoundaryToBootstrapSfuWhenEveryHostSlotIsFull(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2, withSfu: true, prepareTimeoutMs: 300, fakeTimers: true})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
firstRoot := h.connectViewer(created, "first-root")
h.locked(func() {
h.doComplete(host)
h.doComplete(firstRoot)
})
firstPrepare := h.waitPreparedTransport(firstRoot.sessionID, "direct")
secondRoot := h.connectViewer(created, "second-root")
h.locked(func() {
h.doReady(firstRoot, int64(firstPrepare.Revision))
h.doRelay(firstRoot, 2)
h.doComplete(secondRoot)
})
secondPrepare := h.waitPreparedTransport(secondRoot.sessionID, "direct")
waiting := h.connectViewer(created, "waiting")
h.locked(func() {
h.doReady(secondRoot, int64(secondPrepare.Revision))
h.doComplete(waiting)
})
h.waitPreparedTransport(waiting.sessionID, "direct")
h.clock.advance(150)
secondBootstrap := h.waitPreparedTransport(secondRoot.sessionID, "sfu")
h.routeFailed(secondRoot, routeFailedMessage(int64(secondBootstrap.Revision), "prepare", secondBootstrap.Candidate.ConnectionID))
firstBootstrap := h.waitPreparedTransport(firstRoot.sessionID, "sfu")
if h.hasRouteStatus(secondRoot.sessionID, "failed") {
t.Fatal("a failed carrier bootstrap must not fail the carrier")
}
h.routeFailed(firstRoot, routeFailedMessage(int64(firstBootstrap.Revision), "prepare", firstBootstrap.Candidate.ConnectionID))
h.waitFailed(waiting.sessionID)
for _, root := range []authenticatedRouteParticipant{firstRoot, secondRoot} {
if h.hasRouteStatus(root.sessionID, "failed") {
t.Fatal("roots must not be told about the waiting viewer's exhaustion")
}
}
if !h.hasRouteStatus(waiting.sessionID, "failed") {
t.Fatal("the waiting viewer must be told")
}
}
func TestRouterDoesNotRecreateSfuAfterExhaustedReuseAndCarrierTeardown(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2, withSfu: true, prepareTimeoutMs: 300, fakeTimers: true})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
firstRoot := h.connectViewer(created, "reuse-loop-first-root")
h.locked(func() {
h.doComplete(host)
h.doComplete(firstRoot)
})
firstDirect := h.waitPreparedTransport(firstRoot.sessionID, "direct")
secondRoot := h.connectViewer(created, "reuse-loop-second-root")
h.locked(func() {
h.doReady(firstRoot, int64(firstDirect.Revision))
h.doRelay(firstRoot, 1)
h.doComplete(secondRoot)
})
secondDirect := h.waitPreparedTransport(secondRoot.sessionID, "direct")
departedParent := h.connectViewer(created, "reuse-loop-departed-parent")
h.locked(func() {
h.doReady(secondRoot, int64(secondDirect.Revision))
h.doComplete(departedParent)
})
parentDirect := h.waitPreparedUpstream(departedParent.sessionID, protocol.PeerUpstream(firstRoot.peerID))
demand := h.connectViewer(created, "reuse-loop-demand")
h.locked(func() {
h.doReady(departedParent, int64(parentDirect.Revision))
h.doRelay(departedParent, 1)
h.doComplete(demand)
})
demandDirect := h.waitPreparedUpstream(demand.sessionID, protocol.PeerUpstream(departedParent.peerID))
var departureAt int64
h.locked(func() {
h.doReady(demand, int64(demandDirect.Revision))
departureAt = h.clock.now()
h.doDisconnect(departedParent)
h.router.removeViewer(created.RoomID, departedParent.peerID)
h.doRelay(secondRoot, 1)
})
h.waitFor("a direct recovery for the demand", func() bool {
recovery, ok := h.ownPrepare(demand)
return ok && int64(recovery.Revision) > int64(demandDirect.Revision) &&
recovery.Candidate.ChildPeerID == demand.peerID && recovery.Candidate.Transport == "direct"
})
h.clock.advance(max(int64(0), departureAt+150-h.clock.now()))
var carrier authenticatedRouteParticipant
h.waitFor("a carrier bootstrap", func() bool {
for _, viewer := range []authenticatedRouteParticipant{firstRoot, secondRoot} {
if prepared, ok := h.ownPrepare(viewer); ok && prepared.Candidate.Transport == "sfu" {
carrier = viewer
return true
}
}
return false
})
bootstrap, _ := h.ownPrepare(carrier)
h.routeReady(carrier, int64(bootstrap.Revision))
h.waitPreparedTransport(demand.sessionID, "sfu")
if h.usage().Ingress != 1 {
t.Fatal("expected one admitted publication")
}
configCountAfterBootstrap := sfuConfigCount(h.allMessages())
if configCountAfterBootstrap < 3 {
t.Fatalf("sfu-config count after bootstrap = %d, want >= 3", configCountAfterBootstrap)
}
h.clock.advance(max(int64(0), departureAt+300-h.clock.now()))
h.waitFor("the carrier direct convergence", func() bool {
prepared, ok := h.ownPrepare(carrier)
return ok && prepared.Candidate.Transport == "direct"
})
if operation := h.diagnostic(created.RoomID).Operation; operation == nil || operation.Reason != "direct-convergence" {
t.Fatalf("operation = %+v, want direct-convergence", operation)
}
carrierDirect, _ := h.ownPrepare(carrier)
h.routeReady(carrier, int64(carrierDirect.Revision))
h.waitFor("the carrier back on the host", func() bool {
edge, ok := h.activeEdge(created.RoomID, carrier.peerID)
return ok && edge.upstream == protocol.PeerUpstream(host.peerID)
})
h.waitFor("the publication drain", func() bool { return len(h.media.Deleted()) == 1 })
h.clock.advance(300)
if h.usage().Ingress != 0 {
t.Fatal("retired publication remained admitted")
}
if count := sfuConfigCount(h.allMessages()); count != configCountAfterBootstrap {
t.Fatalf("sfu-config count = %d, want %d", count, configCountAfterBootstrap)
}
if usage := h.usage(); usage != (sfu.Usage{}) {
t.Fatalf("usage = %+v, want empty", usage)
}
}
func TestRouterReportsLateFinalSfuBootstrapReadyOnlyToWaitingDemand(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2, withSfu: true, prepareTimeoutMs: 300, fakeTimers: true})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
firstRoot := h.connectViewer(created, "late-ready-first-root")
h.locked(func() {
h.doComplete(host)
h.doComplete(firstRoot)
})
firstPrepare := h.waitPreparedTransport(firstRoot.sessionID, "direct")
secondRoot := h.connectViewer(created, "late-ready-second-root")
h.locked(func() {
h.doReady(firstRoot, int64(firstPrepare.Revision))
h.doRelay(firstRoot, 2)
h.doComplete(secondRoot)
})
secondPrepare := h.waitPreparedTransport(secondRoot.sessionID, "direct")
waiting := h.connectViewer(created, "late-ready-waiting")
h.locked(func() {
h.doReady(secondRoot, int64(secondPrepare.Revision))
h.doComplete(waiting)
})
h.waitPreparedTransport(waiting.sessionID, "direct")
h.clock.advance(150)
firstCarrier := h.waitPreparedTransport(secondRoot.sessionID, "sfu")
h.routeFailed(secondRoot, routeFailedMessage(int64(firstCarrier.Revision), "prepare", firstCarrier.Candidate.ConnectionID))
finalCarrier := h.waitPreparedTransport(firstRoot.sessionID, "sfu")
h.clock.set(451)
h.routeReady(firstRoot, int64(finalCarrier.Revision))
h.waitFor("exactly one exhaustion for the waiting viewer", func() bool {
count := 0
for _, message := range h.messages(waiting.sessionID) {
if status, ok := message.(protocol.RouteStatusMessage); ok && status.State == "failed" && status.Reason == "route-exhausted" {
count++
}
}
return count == 1
})
for _, carrier := range []authenticatedRouteParticipant{firstRoot, secondRoot} {
if h.hasRouteStatus(carrier.sessionID, "failed") {
t.Fatal("carriers must not be told about the waiting viewer's exhaustion")
}
}
}
func TestRouterReportsBoundedRouteExhaustionWithoutRawErrorText(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2, withSfu: true, prepareTimeoutMs: 300, fakeTimers: true})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
viewer := h.connectViewer(created, "route-exhausted")
h.locked(func() {
h.doComplete(host)
h.doComplete(viewer)
})
h.waitPreparedTransport(viewer.sessionID, "direct")
h.clock.advance(150)
h.waitPreparedTransport(viewer.sessionID, "sfu")
h.clock.advance(150)
h.waitFailed(viewer.sessionID)
for _, message := range h.messages(viewer.sessionID) {
if failure, ok := message.(protocol.ErrorMessage); ok && strings.Contains(failure.Message, "media route") {
t.Fatal("raw error text must not reach the viewer")
}
}
}
func TestRouterReportsTypedExhaustionWhenExactCandidateFails(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 1})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
viewer := h.connectViewer(created, "candidate-failed")
h.locked(func() {
h.doComplete(host)
h.doComplete(viewer)
})
prepared := h.waitPrepared(viewer.sessionID)
h.routeFailed(viewer, routeFailedMessage(int64(prepared.Revision), "prepare", prepared.Candidate.ConnectionID))
h.waitFailed(viewer.sessionID)
}
func noCandidateExhaustion(t *testing.T) {
t.Helper()
h := newRouterHarness(t, routerHarnessOptions{capacity: 1})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
first := h.connectViewer(created, "only-slot")
h.locked(func() {
h.doComplete(host)
h.doComplete(first)
})
firstPrepare := h.waitPrepared(first.sessionID)
blocked := h.connectViewer(created, "no-candidate")
h.locked(func() {
h.doReady(first, int64(firstPrepare.Revision))
h.doComplete(blocked)
})
h.waitFailed(blocked.sessionID)
}
func TestRouterReportsTypedExhaustionWhenReconciliationHasNoCandidate(t *testing.T) {
noCandidateExhaustion(t)
}
func TestRouterReportsOnlySfuAdmissionWaitingWhileBoundedCapacityIsUnavailable(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2, withSfu: true, prepareTimeoutMs: 300, fakeTimers: true})
capacityFences := []sfu.ResourceFence{
{RoomID: "9001", ShareGeneration: "capacity_share_a", PublicationGeneration: "capacity_publication_a"},
{RoomID: "9002", ShareGeneration: "capacity_share_b", PublicationGeneration: "capacity_publication_b"},
}
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
viewer := h.connectViewer(created, "sfu-admission-wait")
h.locked(func() {
for _, fence := range capacityFences {
if !h.admission.ReservePublication(fence) {
t.Fatal("the capacity fence must be reserved")
}
}
h.doComplete(host)
h.doComplete(viewer)
})
h.waitPreparedTransport(viewer.sessionID, "direct")
h.clock.advance(150)
h.waitFor("the sfu-admission waiting status", func() bool {
status, ok := h.lastRouteStatus(viewer.sessionID)
return ok && status.State == "waiting" && status.Reason == "sfu-admission"
})
h.locked(func() {
for _, fence := range capacityFences {
h.admission.BeginDrain(fence)
h.admission.CompleteDrain(fence)
}
})
}
// TS asserted admission.reservePublication was not called again through a
// spy; here the observable is the waiting status count, which a woken
// waiter would re-send (map 7.2 step 9c), plus the unchanged usage.
func TestRouterReleasesExactSubscriptionChargeAfterPhysicalClose(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true, sfuIngressCapacity: 2})
activeRoom := h.createRoom()
_, first, _ := h.establishSfuRoom(activeRoom)
h.waitUsage(sfu.Usage{Ingress: 1, Egress: 2})
waitingRoom := h.createRoom()
waitingHost := h.connectHost(waitingRoom, nil, "", "")
waitingViewer := h.connectViewer(waitingRoom, "waiter")
h.locked(func() {
if !h.admission.ReservePublication(sfu.ResourceFence{RoomID: "9001", ShareGeneration: "waiter_share_12345678", PublicationGeneration: "waiter_publication_12345678"}) {
t.Fatal("the occupying publication must be reserved")
}
h.doComplete(waitingHost)
h.doComplete(waitingViewer)
})
direct := h.waitPrepared(waitingViewer.sessionID)
h.routeFailed(waitingViewer, routeFailedMessage(int64(direct.Revision), "prepare", direct.Candidate.ConnectionID))
h.waitFor("the waiting status", func() bool {
status, ok := h.lastRouteStatus(waitingViewer.sessionID)
return ok && status.State == "waiting" && status.Reason == "sfu-admission"
})
drainGate := make(chan struct{})
h.media.SetSubscriptionDrainBarrier(drainGate)
h.locked(func() {
h.doDisconnect(first)
h.router.removeViewer(activeRoom.RoomID, first.peerID)
})
h.waitFor("one diagnostic child", func() bool { return len(h.diagnostic(activeRoom.RoomID).Children) == 1 })
h.waitFor("the subscription drain attempt", func() bool {
return hasViewer(h.media.SubscriptionDrainAttempts(), first.peerID)
})
if usage := h.usage(); usage != (sfu.Usage{Ingress: 2, Egress: 2}) {
t.Fatalf("usage during drain = %+v", usage)
}
close(drainGate)
h.waitFor("the drained subscription", func() bool {
return hasViewer(h.media.DrainedSubscriptions(), first.peerID)
})
h.waitUsage(sfu.Usage{Ingress: 2, Egress: 1})
if err := h.close(); err != nil {
t.Fatalf("close: %v", err)
}
if usage := h.usage(); usage != (sfu.Usage{}) {
t.Fatalf("usage after close = %+v", usage)
}
}
func TestRouterOrdersExactPrepareCommitsOnlyChildProofAndRollsBackAboveP(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 2})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
first := h.connectViewer(created, "first")
h.locked(func() {
h.doComplete(host)
h.doComplete(first)
})
firstPrepare := h.waitPrepared(first.sessionID)
if firstPrepare.Candidate.ChildPeerID != first.peerID || firstPrepare.Candidate.Transport != "direct" {
t.Fatalf("first candidate = %+v", firstPrepare.Candidate)
}
h.locked(func() {
if hostPrepare, ok := lastPrepare(h.sent[host.sessionID], ""); !ok || !reflect.DeepEqual(hostPrepare.Candidate, firstPrepare.Candidate) {
t.Fatal("the host must receive the same candidate")
}
h.doReady(host, int64(firstPrepare.Revision))
for _, message := range h.sent[first.sessionID] {
if active, ok := message.(protocol.RouteUpdateActiveMessage); ok && int64(active.Revision) > int64(firstPrepare.Revision) {
t.Fatal("host proof must not commit the candidate")
}
}
h.doReady(first, int64(firstPrepare.Revision))
})
h.waitFor("the active route at the prepare revision", func() bool {
for _, message := range h.messages(first.sessionID) {
if active, ok := message.(protocol.RouteUpdateActiveMessage); ok && active.Revision == firstPrepare.Revision {
return true
}
}
return false
})
second := h.connectViewer(created, "second")
h.complete(second)
secondPrepare := h.waitPrepared(second.sessionID)
h.routeFailed(second, routeFailedMessage(int64(secondPrepare.Revision), "prepare", secondPrepare.Candidate.ConnectionID))
h.waitFor("the rollback broadcast", func() bool {
_, ok := h.activeAfter(second.sessionID, int64(secondPrepare.Revision))
return ok
})
if active, _ := h.activeAfter(second.sessionID, int64(secondPrepare.Revision)); int64(active.Revision) <= int64(secondPrepare.Revision) {
t.Fatalf("rollback revision = %d", active.Revision)
}
}
func TestRouterCreatesOneHostPublicationAndReusesExactViewerSubscriptions(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
first := h.connectViewer(created, "sfu_first")
h.locked(func() {
h.doComplete(host)
h.doComplete(first)
})
firstDirect := h.waitPrepared(first.sessionID)
second := h.connectViewer(created, "sfu_second")
h.locked(func() {
h.doReady(first, int64(firstDirect.Revision))
h.doComplete(second)
})
firstSfu := h.waitPreparedTransport(first.sessionID, "sfu")
h.routeReady(first, int64(firstSfu.Revision))
secondSfu := h.waitPreparedTransport(second.sessionID, "sfu")
h.routeReady(second, int64(secondSfu.Revision))
h.waitUsage(sfu.Usage{Ingress: 1, Egress: 2})
if count := sfuConfigCount(h.messages(host.sessionID)); count != 1 {
t.Fatalf("host sfu-config count = %d, want 1", count)
}
}
func TestRouterResolvesExactDirectAndPeerRelayedViewerEvidenceSources(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 1})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
root := h.connectViewer(created, "evidence-root")
h.locked(func() {
h.doComplete(host)
h.doComplete(root)
})
rootPrepare := h.waitPrepared(root.sessionID)
h.locked(func() {
if _, ok := h.doEdge(created.RoomID, root.peerID); ok {
t.Fatal("a prepared candidate is not an active edge")
}
h.doReady(root, int64(rootPrepare.Revision))
})
h.waitFor("the root active edge", func() bool {
edge, ok := h.activeEdge(created.RoomID, root.peerID)
return ok && edge == activeViewerMediaEdge{
revision: int64(rootPrepare.Revision), connectionID: rootPrepare.Candidate.ConnectionID,
upstream: protocol.PeerUpstream(host.peerID),
}
})
child := h.connectViewer(created, "evidence-child")
h.locked(func() {
h.doRelay(root, 1)
h.doComplete(child)
})
childPrepare := h.waitPrepared(child.sessionID)
if childPrepare.Candidate.Transport != "direct" {
t.Fatalf("child transport = %q", childPrepare.Candidate.Transport)
}
h.routeReady(child, int64(childPrepare.Revision))
h.waitFor("the child active edge", func() bool {
edge, ok := h.activeEdge(created.RoomID, child.peerID)
return ok && edge == activeViewerMediaEdge{
revision: int64(childPrepare.Revision), connectionID: childPrepare.Candidate.ConnectionID,
upstream: protocol.PeerUpstream(root.peerID),
}
})
}
func TestRouterDrainsEveryPendingSfuPublicationDuringCloseEvenWhenOneDrainRejects(t *testing.T) {
before := runtime.NumGoroutine()
h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true, prepareTimeoutMs: 1_000, sfuIngressCapacity: 2_000})
roomA := h.createRoom()
h.establishSfuRoom(roomA)
roomB := h.createRoom()
h.establishSfuRoom(roomB)
h.media.SetFailDelete(true)
h.locked(func() {
h.router.stopRoom(roomA.RoomID)
h.router.stopRoom(roomB.RoomID)
})
h.waitFor("two delete attempts", func() bool {
deletes, _, _ := h.media.counts()
return deletes == 2
})
h.media.SetFailDelete(false)
fake := h.media.FakeMedia
h.media.setClosePublication(func(fence sfu.ResourceFence) error {
if fence.RoomID == roomA.RoomID {
return errors.New("room deletion failed")
}
return fake.ClosePublication(fence)
})
err := h.close()
if err == nil || !strings.Contains(err.Error(), "SFU drain failed during shutdown") {
t.Fatalf("close error = %v, want the shutdown drain failure", err)
}
var deleted []string
for _, fence := range h.media.Deleted() {
deleted = append(deleted, fence.RoomID)
}
if !slices.Equal(deleted, []string{roomB.RoomID}) {
t.Fatalf("deleted rooms = %v, want [%s]", deleted, roomB.RoomID)
}
expectGoroutinesSettled(t, before)
}
func TestRouterSerializesFreshSfuConfigurationWithoutMakingRefreshOneShot(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true})
created := h.createRoom()
_, first, _ := h.establishSfuRoom(created)
configCount := func() int { return sfuConfigCount(h.messages(first.sessionID)) }
var active activeViewerMediaEdge
var initialCount int
h.locked(func() {
active = h.doMustEdge(created.RoomID, first.peerID)
if active.upstream != protocol.SfuUpstream() {
t.Fatalf("upstream = %v", active.upstream)
}
initialCount = sfuConfigCount(h.sent[first.sessionID])
h.router.refreshSfu(first, 0)
})
h.waitFor("the first refresh", func() bool { return configCount() == initialCount+1 })
h.locked(func() { h.router.refreshSfu(first, active.revision) })
h.waitFor("the second refresh", func() bool { return configCount() == initialCount+2 })
}
// hybrid-media-router.ts:1129-1132: a throw from media or tokenIssuer
// rejected the pump promise and its .catch told the Host. In Go the throw is a
// panic raised while mu is released for the token issue, so the pump must come
// back holding mu; otherwise drivePump's deferred Unlock aborts the process.
func TestRouterEmitsTypedExhaustionAndClearsDiagnosticsOnDepartureAndRoomDeletion(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 1})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
viewer := h.connectViewer(created, "diagnostic")
h.locked(func() {
h.doComplete(host)
h.doComplete(viewer)
})
prepared := h.waitPrepared(viewer.sessionID)
h.locked(func() {
if len(h.router.routeDiagnosticSnapshot(created.RoomID).Children) != 1 {
t.Fatal("one diagnostic child expected")
}
h.doFailed(viewer, routeFailedMessage(int64(prepared.Revision), "prepare", prepared.Candidate.ConnectionID))
})
h.waitFailed(viewer.sessionID)
replacement := h.connectViewer(created, "diagnostic-replacement")
h.locked(func() {
emptySnapshot := func() bool {
snapshot := h.router.routeDiagnosticSnapshot(created.RoomID)
return len(snapshot.Children) == 0 && snapshot.Operation == nil
}
h.router.removeViewer(created.RoomID, viewer.peerID)
if !emptySnapshot() {
t.Fatal("departure must clear the diagnostics")
}
h.doComplete(replacement)
if len(h.router.routeDiagnosticSnapshot(created.RoomID).Children) != 1 {
t.Fatal("the replacement must appear in the diagnostics")
}
h.router.stopRoom(created.RoomID)
if !emptySnapshot() {
t.Fatal("stopRoom must clear the diagnostics")
}
h.doComplete(host)
if len(h.router.routeDiagnosticSnapshot(created.RoomID).Children) == 0 {
t.Fatal("a restarted share must rebuild the diagnostics")
}
h.router.deleteRoom(created.RoomID)
if !emptySnapshot() {
t.Fatal("deleteRoom must clear the diagnostics")
}
})
}
func TestRouterPreservesConnectedViewerCapacityAcrossSharingGenerations(t *testing.T) {
h := newRouterHarness(t, routerHarnessOptions{capacity: 1})
created := h.createRoom()
host := h.connectHost(created, nil, "", "")
root := h.connectViewer(created, "retained-capacity-root")
h.locked(func() {
h.doComplete(host)
h.doComplete(root)
})
initialRoot := h.waitPrepared(root.sessionID)
h.locked(func() {
h.doReady(root, int64(initialRoot.Revision))
h.doRelay(root, 1)
h.router.stopRoom(created.RoomID)
h.sent[root.sessionID] = nil
h.doComplete(host)
})
restartedRoot := h.waitPrepared(root.sessionID)
child := h.connectViewer(created, "retained-capacity-child")
h.locked(func() {
h.doReady(root, int64(restartedRoot.Revision))
h.doComplete(child)
})
childPrepare := h.waitPrepared(child.sessionID)
if childPrepare.Assignment.Upstream != protocol.PeerUpstream(root.peerID) {
t.Fatalf("child upstream = %v, want the retained root", childPrepare.Assignment.Upstream)
}
}
// TS declared this scenario twice under two names; both are kept.
func TestRouterEmitsTypedExhaustionWhenReconciliationHasNoCandidate(t *testing.T) {
noCandidateExhaustion(t)
}
// ---------------------------------------------------------------------------
// Go-only coverage of the concurrency seams the TS runtime did not have:
// the per-room-name FIFO keeps LiveKit calls in decision order.
// ---------------------------------------------------------------------------
func TestRouterDebugFlagParsesLikeNodeDebug(t *testing.T) {
previous := slog.Default()
t.Cleanup(func() { slog.SetDefault(previous) })
slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil)))
cases := map[string]bool{"": false, "route": true, "http,route": true, " route ": true, "router": false}
keys := make([]string, 0, len(cases))
for key := range cases {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
t.Setenv("PIIK_DEBUG", key)
if got := routeDebugFlag(key); got != cases[key] {
t.Fatalf("routeDebugFlag(%q) = %v, want %v", key, got, cases[key])
}
if got := routeDebugEnabled(); got != cases[key] {
t.Fatalf("routeDebugEnabled with %q = %v, want %v", key, got, cases[key])
}
}
t.Setenv("PIIK_DEBUG", "")
slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: slog.LevelDebug})))
if !routeDebugEnabled() {
t.Fatal("the active diagnostic logger did not enable route events")
}
}